snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/multi_master/join_test.go

@@ -0,0 +1,212 @@
+package multi_master
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+var raftImplementations = []struct {
+	name          string
+	raftHashicorp bool
+}{
+	{"goraft", false},
+	{"hashicorp", true},
+}
+
+// TestFreshClusterFormsWithoutALeader covers the other half of the bootstrap
+// decision: with no leader anywhere, three masters starting together still have
+// to mint a cluster between them.
+func TestFreshClusterFormsWithoutALeader(t *testing.T) {
+	for _, impl := range raftImplementations {
+		t.Run(impl.name, func(t *testing.T) {
+			mc := NewMasterCluster(t, impl.raftHashicorp)
+			for i := range 3 {
+				mc.StartNode(i)
+			}
+
+			if _, err := waitForCommonLeader(mc, waitTimeout); err != nil {
+				mc.DumpLogs()
+				t.Fatalf("fresh cluster did not converge: %v", err)
+			}
+			for i := range 3 {
+				if err := waitForPeerCount(mc, i, 2, waitTimeout); err != nil {
+					mc.DumpLogs()
+					t.Fatalf("master %d does not see the full cluster: %v", i, err)
+				}
+			}
+		})
+	}
+}
+
+// TestScaleUpOntoExistingLeader mirrors a Kubernetes master StatefulSet whose
+// replica count goes back from one to three: master 0 keeps running as the
+// leader of a single-peer cluster while two fresh masters come up pointing at
+// all three. The newcomers start with an empty raft log, so neither raft
+// implementation lets them campaign — the sitting leader has to admit them.
+func TestScaleUpOntoExistingLeader(t *testing.T) {
+	for _, impl := range raftImplementations {
+		t.Run(impl.name, func(t *testing.T) {
+			mc := NewMasterCluster(t, impl.raftHashicorp)
+
+			// Master 0 is alone in its peer list, the way the operator renders
+			// -peers when spec.master.replicas is 1.
+			mc.SetNodePeers(0, mc.NodeAddress(0))
+			mc.StartNode(0)
+			if err := mc.WaitForLeader(waitTimeout); err != nil {
+				mc.DumpLogs()
+				t.Fatalf("single master did not become leader: %v", err)
+			}
+
+			// Scale up. These two carry the full peer list; master 0 still
+			// runs with the old one and has never heard of them.
+			mc.StartNode(1)
+			mc.StartNode(2)
+
+			leader, err := waitForCommonLeader(mc, waitTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatalf("masters did not converge after scaling up: %v", err)
+			}
+			if leader != mc.NodeAddress(0) {
+				t.Fatalf("leader moved to %s, want the sitting leader %s", leader, mc.NodeAddress(0))
+			}
+
+			if err := waitForPeerCount(mc, 0, 2, waitTimeout); err != nil {
+				mc.DumpLogs()
+				t.Fatalf("leader did not admit both new masters: %v", err)
+			}
+		})
+	}
+}
+
+// waitForCommonLeader waits until every running master names the same leader,
+// and returns it.
+func waitForCommonLeader(mc *MasterCluster, timeout time.Duration) (string, error) {
+	var lastErr error
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		leader, err := commonLeader(mc)
+		if err == nil {
+			return leader, nil
+		}
+		lastErr = err
+		time.Sleep(waitTick)
+	}
+	return "", lastErr
+}
+
+func commonLeader(mc *MasterCluster) (string, error) {
+	agreed := ""
+	for i := range 3 {
+		if !mc.IsNodeRunning(i) {
+			continue
+		}
+		cs, err := mc.GetClusterStatus(i)
+		if err != nil {
+			return "", err
+		}
+		leader := pb.ServerAddress(cs.Leader).ToHttpAddress()
+		if leader == "" {
+			return "", fmt.Errorf("master %d has no leader", i)
+		}
+		if agreed == "" {
+			agreed = leader
+		} else if agreed != leader {
+			return "", fmt.Errorf("masters disagree on the leader: %s and %s", agreed, leader)
+		}
+	}
+	if agreed == "" {
+		return "", fmt.Errorf("no master is running")
+	}
+	return agreed, nil
+}
+
+// waitForPeerCount waits until node i reports the given number of raft peers.
+// The count excludes the node itself.
+func waitForPeerCount(mc *MasterCluster, i, want int, timeout time.Duration) error {
+	got := -1
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		cs, err := mc.GetClusterStatus(i)
+		if err == nil {
+			got = peerCountExcludingSelf(cs.Peers, mc.NodeAddress(i))
+			if got == want {
+				return nil
+			}
+		}
+		time.Sleep(waitTick)
+	}
+	return fmt.Errorf("master %d reports %d peers, want %d", i, got, want)
+}
+
+func peerCountExcludingSelf(peers []string, self string) int {
+	count := 0
+	for _, peer := range peers {
+		if pb.ServerAddress(peer).ToHttpAddress() != self {
+			count++
+		}
+	}
+	return count
+}
+
+// TestRaftBootstrapKeepsExistingCluster covers a master restarting under
+// -raftBootstrap, the way the helm chart renders it on every master on every
+// roll. Bootstrapping is genesis: seeding a second cluster over committed raft
+// state mints a rival TopologyId, and the split-brain guard then Fatals every
+// master that still holds the first one.
+func TestRaftBootstrapKeepsExistingCluster(t *testing.T) {
+	for _, impl := range raftImplementations {
+		t.Run(impl.name, func(t *testing.T) {
+			mc := NewMasterCluster(t, impl.raftHashicorp)
+			for i := range 3 {
+				mc.SetRaftBootstrap(i)
+				mc.StartNode(i)
+			}
+			before, err := mc.WaitForTopologyId(waitTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatalf("cluster did not mint a TopologyId: %v", err)
+			}
+
+			for i := range 3 {
+				mc.StopNode(i)
+			}
+			for i := range 3 {
+				mc.StartNode(i)
+			}
+
+			after, err := mc.WaitForTopologyId(waitTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatalf("cluster did not come back after a restart: %v", err)
+			}
+			if after != before {
+				mc.DumpLogs()
+				t.Fatalf("-raftBootstrap re-seeded the cluster: TopologyId %s became %s", before, after)
+			}
+
+			// The leader answers for the whole cluster, so a follower that
+			// forked is only visible in its own log.
+			seen, err := mc.WaitForNodeTopologyIds(waitTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatal(err)
+			}
+			for i, ids := range seen {
+				for _, id := range ids {
+					if id != before {
+						mc.DumpLogs()
+						t.Fatalf("master %d saw TopologyId %s, want %s", i, id, before)
+					}
+				}
+				if mc.LogContains(i, "Split-brain detected") {
+					mc.DumpLogs()
+					t.Fatalf("master %d hit the split-brain guard", i)
+				}
+			}
+		})
+	}
+}