snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/multi_master/chaos_test.go

@@ -0,0 +1,186 @@
+package multi_master
+
+import (
+	"fmt"
+	"math/rand/v2"
+	"os"
+	"strconv"
+	"testing"
+	"time"
+)
+
+// chaosRounds is one stop or start per round, enough to walk in and out of
+// quorum several times without turning this into a soak test.
+const chaosRounds = 12
+
+// TestRandomStartStopElection bounces masters at random and holds the election
+// to the two things it must never get wrong: two masters claiming leadership at
+// once, and a quorum that comes back without agreeing on one. The cluster's
+// identity has to survive the whole walk — a master that re-mints a TopologyId
+// here is the split brain SetTopologyId kills its peers over.
+func TestRandomStartStopElection(t *testing.T) {
+	for _, impl := range raftImplementations {
+		t.Run(impl.name, func(t *testing.T) {
+			seed := chaosSeed(t)
+			t.Logf("seed %d — replay this walk with MULTI_MASTER_IT_SEED=%d", seed, seed)
+			rng := rand.New(rand.NewPCG(seed, seed))
+
+			mc := NewMasterCluster(t, impl.raftHashicorp)
+			for i := range 3 {
+				mc.StartNode(i)
+			}
+			if err := mc.WaitForLeader(waitTimeout); err != nil {
+				mc.DumpLogs()
+				t.Fatalf("cluster did not elect a leader: %v", err)
+			}
+			topologyId, err := mc.WaitForTopologyId(waitTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatalf("no initial TopologyId: %v", err)
+			}
+
+			for round := 1; round <= chaosRounds; round++ {
+				target := rng.IntN(3)
+				if mc.IsNodeRunning(target) {
+					mc.StopNode(target)
+					t.Logf("round %d: stopped master %d, %d left running", round, target, runningMasters(mc))
+				} else {
+					mc.StartNode(target)
+					t.Logf("round %d: started master %d, %d now running", round, target, runningMasters(mc))
+				}
+
+				if err := waitForSettledElection(mc, leaderElectionTimeout); err != nil {
+					mc.DumpLogs()
+					t.Fatalf("round %d (seed %d): %v", round, seed, err)
+				}
+
+				// /dir/status proxies to the leader, so this reads the value the
+				// cluster as a whole is carrying, not any one master's copy.
+				if runningMasters(mc) >= 2 {
+					id, err := mc.WaitForTopologyId(leaderElectionTimeout)
+					if err != nil {
+						mc.DumpLogs()
+						t.Fatalf("round %d (seed %d): %v", round, seed, err)
+					}
+					if id != topologyId {
+						mc.DumpLogs()
+						t.Fatalf("round %d (seed %d): TopologyId changed from %s to %s", round, seed, topologyId, id)
+					}
+				}
+			}
+
+			// Everything back up, so the walk ends on a full cluster.
+			for i := range 3 {
+				mc.StartNode(i)
+			}
+			if _, err := waitForCommonLeader(mc, leaderElectionTimeout); err != nil {
+				mc.DumpLogs()
+				t.Fatalf("cluster did not recover after the walk (seed %d): %v", seed, err)
+			}
+			for i := range 3 {
+				if err := waitForPeerCount(mc, i, 2, leaderElectionTimeout); err != nil {
+					mc.DumpLogs()
+					t.Fatalf("master %d does not see the full cluster (seed %d): %v", i, seed, err)
+				}
+			}
+			id, err := mc.WaitForTopologyId(leaderElectionTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatalf("no TopologyId after the walk (seed %d): %v", seed, err)
+			}
+			if id != topologyId {
+				mc.DumpLogs()
+				t.Fatalf("TopologyId changed from %s to %s over the walk (seed %d)", topologyId, id, seed)
+			}
+		})
+	}
+}
+
+// waitForSettledElection waits for a quorum to agree on one leader, and fails
+// if two masters claim leadership across consecutive polls — anything shorter
+// than that is a master on its way down.
+//
+// Below a quorum there is nothing to wait for: the walk moves on. A master that
+// has lost its quorum can keep claiming leadership for tens of seconds under
+// goraft, and it cannot commit anything in that window, so stepping down is a
+// liveness question rather than a safety one. TestTwoMastersDownAndRestart
+// holds that direction to account.
+func waitForSettledElection(mc *MasterCluster, timeout time.Duration) error {
+	if runningMasters(mc) < 2 {
+		return nil
+	}
+
+	var lastErr error
+	splitPolls := 0
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		claims, err := leaderClaims(mc)
+		if err != nil {
+			lastErr = err
+		}
+		if len(claims) > 1 {
+			splitPolls++
+			if splitPolls > 1 {
+				return fmt.Errorf("masters %v all claim leadership", claims)
+			}
+			time.Sleep(waitTick)
+			continue
+		}
+		splitPolls = 0
+
+		if _, err := commonLeader(mc); err == nil {
+			return nil
+		} else {
+			lastErr = err
+		}
+		time.Sleep(waitTick)
+	}
+	if lastErr == nil {
+		lastErr = fmt.Errorf("cluster did not settle within %v", timeout)
+	}
+	return lastErr
+}
+
+// leaderClaims returns the running masters that call themselves leader. The
+// error reports masters that did not answer at all, which is a reason to keep
+// waiting rather than a verdict.
+func leaderClaims(mc *MasterCluster) (claims []int, err error) {
+	for i := range 3 {
+		if !mc.IsNodeRunning(i) {
+			continue
+		}
+		cs, statusErr := mc.GetClusterStatus(i)
+		if statusErr != nil {
+			err = fmt.Errorf("master %d: %w", i, statusErr)
+			continue
+		}
+		if cs.IsLeader {
+			claims = append(claims, i)
+		}
+	}
+	return claims, err
+}
+
+func runningMasters(mc *MasterCluster) int {
+	count := 0
+	for i := range 3 {
+		if mc.IsNodeRunning(i) {
+			count++
+		}
+	}
+	return count
+}
+
+// chaosSeed picks the walk. It is random by default and always logged, so a
+// failure names the seed that reproduces it.
+func chaosSeed(t *testing.T) uint64 {
+	t.Helper()
+	if v := os.Getenv("MULTI_MASTER_IT_SEED"); v != "" {
+		seed, err := strconv.ParseUint(v, 10, 64)
+		if err != nil {
+			t.Fatalf("MULTI_MASTER_IT_SEED %q: %v", v, err)
+		}
+		return seed
+	}
+	return uint64(time.Now().UnixNano())
+}