snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/multi_master/chaos_test.go

@@ -0,0 +1,186 @@
+package multi_master
+
+import (
+	"fmt"
+	"math/rand/v2"
+	"os"
+	"strconv"
+	"testing"
+	"time"
+)
+
+// chaosRounds is one stop or start per round, enough to walk in and out of
+// quorum several times without turning this into a soak test.
+const chaosRounds = 12
+
+// TestRandomStartStopElection bounces masters at random and holds the election
+// to the two things it must never get wrong: two masters claiming leadership at
+// once, and a quorum that comes back without agreeing on one. The cluster's
+// identity has to survive the whole walk — a master that re-mints a TopologyId
+// here is the split brain SetTopologyId kills its peers over.
+func TestRandomStartStopElection(t *testing.T) {
+	for _, impl := range raftImplementations {
+		t.Run(impl.name, func(t *testing.T) {
+			seed := chaosSeed(t)
+			t.Logf("seed %d — replay this walk with MULTI_MASTER_IT_SEED=%d", seed, seed)
+			rng := rand.New(rand.NewPCG(seed, seed))
+
+			mc := NewMasterCluster(t, impl.raftHashicorp)
+			for i := range 3 {
+				mc.StartNode(i)
+			}
+			if err := mc.WaitForLeader(waitTimeout); err != nil {
+				mc.DumpLogs()
+				t.Fatalf("cluster did not elect a leader: %v", err)
+			}
+			topologyId, err := mc.WaitForTopologyId(waitTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatalf("no initial TopologyId: %v", err)
+			}
+
+			for round := 1; round <= chaosRounds; round++ {
+				target := rng.IntN(3)
+				if mc.IsNodeRunning(target) {
+					mc.StopNode(target)
+					t.Logf("round %d: stopped master %d, %d left running", round, target, runningMasters(mc))
+				} else {
+					mc.StartNode(target)
+					t.Logf("round %d: started master %d, %d now running", round, target, runningMasters(mc))
+				}
+
+				if err := waitForSettledElection(mc, leaderElectionTimeout); err != nil {
+					mc.DumpLogs()
+					t.Fatalf("round %d (seed %d): %v", round, seed, err)
+				}
+
+				// /dir/status proxies to the leader, so this reads the value the
+				// cluster as a whole is carrying, not any one master's copy.
+				if runningMasters(mc) >= 2 {
+					id, err := mc.WaitForTopologyId(leaderElectionTimeout)
+					if err != nil {
+						mc.DumpLogs()
+						t.Fatalf("round %d (seed %d): %v", round, seed, err)
+					}
+					if id != topologyId {
+						mc.DumpLogs()
+						t.Fatalf("round %d (seed %d): TopologyId changed from %s to %s", round, seed, topologyId, id)
+					}
+				}
+			}
+
+			// Everything back up, so the walk ends on a full cluster.
+			for i := range 3 {
+				mc.StartNode(i)
+			}
+			if _, err := waitForCommonLeader(mc, leaderElectionTimeout); err != nil {
+				mc.DumpLogs()
+				t.Fatalf("cluster did not recover after the walk (seed %d): %v", seed, err)
+			}
+			for i := range 3 {
+				if err := waitForPeerCount(mc, i, 2, leaderElectionTimeout); err != nil {
+					mc.DumpLogs()
+					t.Fatalf("master %d does not see the full cluster (seed %d): %v", i, seed, err)
+				}
+			}
+			id, err := mc.WaitForTopologyId(leaderElectionTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatalf("no TopologyId after the walk (seed %d): %v", seed, err)
+			}
+			if id != topologyId {
+				mc.DumpLogs()
+				t.Fatalf("TopologyId changed from %s to %s over the walk (seed %d)", topologyId, id, seed)
+			}
+		})
+	}
+}
+
+// waitForSettledElection waits for a quorum to agree on one leader, and fails
+// if two masters claim leadership across consecutive polls — anything shorter
+// than that is a master on its way down.
+//
+// Below a quorum there is nothing to wait for: the walk moves on. A master that
+// has lost its quorum can keep claiming leadership for tens of seconds under
+// goraft, and it cannot commit anything in that window, so stepping down is a
+// liveness question rather than a safety one. TestTwoMastersDownAndRestart
+// holds that direction to account.
+func waitForSettledElection(mc *MasterCluster, timeout time.Duration) error {
+	if runningMasters(mc) < 2 {
+		return nil
+	}
+
+	var lastErr error
+	splitPolls := 0
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		claims, err := leaderClaims(mc)
+		if err != nil {
+			lastErr = err
+		}
+		if len(claims) > 1 {
+			splitPolls++
+			if splitPolls > 1 {
+				return fmt.Errorf("masters %v all claim leadership", claims)
+			}
+			time.Sleep(waitTick)
+			continue
+		}
+		splitPolls = 0
+
+		if _, err := commonLeader(mc); err == nil {
+			return nil
+		} else {
+			lastErr = err
+		}
+		time.Sleep(waitTick)
+	}
+	if lastErr == nil {
+		lastErr = fmt.Errorf("cluster did not settle within %v", timeout)
+	}
+	return lastErr
+}
+
+// leaderClaims returns the running masters that call themselves leader. The
+// error reports masters that did not answer at all, which is a reason to keep
+// waiting rather than a verdict.
+func leaderClaims(mc *MasterCluster) (claims []int, err error) {
+	for i := range 3 {
+		if !mc.IsNodeRunning(i) {
+			continue
+		}
+		cs, statusErr := mc.GetClusterStatus(i)
+		if statusErr != nil {
+			err = fmt.Errorf("master %d: %w", i, statusErr)
+			continue
+		}
+		if cs.IsLeader {
+			claims = append(claims, i)
+		}
+	}
+	return claims, err
+}
+
+func runningMasters(mc *MasterCluster) int {
+	count := 0
+	for i := range 3 {
+		if mc.IsNodeRunning(i) {
+			count++
+		}
+	}
+	return count
+}
+
+// chaosSeed picks the walk. It is random by default and always logged, so a
+// failure names the seed that reproduces it.
+func chaosSeed(t *testing.T) uint64 {
+	t.Helper()
+	if v := os.Getenv("MULTI_MASTER_IT_SEED"); v != "" {
+		seed, err := strconv.ParseUint(v, 10, 64)
+		if err != nil {
+			t.Fatalf("MULTI_MASTER_IT_SEED %q: %v", v, err)
+		}
+		return seed
+	}
+	return uint64(time.Now().UnixNano())
+}

test/multi_master/cluster.go

@@ -0,0 +1,519 @@
+package multi_master
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+)
+
+const (
+	waitTimeout = 30 * time.Second
+	waitTick    = 200 * time.Millisecond
+)
+
+// masterNode represents a single master process in the cluster.
+type masterNode struct {
+	port     int
+	grpcPort int
+	dataDir  string
+	cmd      *exec.Cmd
+	logFile  string
+	stopped  bool
+	// peersStr overrides the cluster-wide peer list for this node, so a test
+	// can start a master that only knows about a subset of the cluster.
+	peersStr string
+	// raftBootstrap starts the master with -raftBootstrap, the way the helm
+	// chart renders master.raftBootstrap on every master, every restart.
+	raftBootstrap bool
+}
+
+// MasterCluster manages a 3-node master raft cluster for integration tests.
+type MasterCluster struct {
+	t          testing.TB
+	weedBinary string
+	baseDir    string
+	logsDir    string
+	keepLogs   bool
+
+	nodes [3]*masterNode
+	mu    sync.Mutex
+
+	// peers string shared by all nodes, e.g. "127.0.0.1:9333,127.0.0.1:9334,127.0.0.1:9335"
+	peersStr string
+
+	// raftHashicorp starts the masters with -raftHashicorp
+	raftHashicorp bool
+}
+
+// clusterStatus is the JSON returned by /cluster/status.
+type clusterStatus struct {
+	IsLeader bool     `json:"IsLeader"`
+	Leader   string   `json:"Leader"`
+	Peers    []string `json:"Peers"`
+}
+
+// StartMasterCluster boots a 3-node master raft cluster and waits for a leader.
+func StartMasterCluster(t testing.TB) *MasterCluster {
+	t.Helper()
+
+	mc := NewMasterCluster(t, false)
+	for i := range 3 {
+		mc.StartNode(i)
+	}
+
+	if err := mc.WaitForLeader(waitTimeout); err != nil {
+		mc.DumpLogs()
+		mc.StopAll()
+		t.Fatalf("cluster did not elect a leader: %v", err)
+	}
+
+	// Wait for TopologyId to be generated and propagated. This is async
+	// after leader election, and we need it committed before tests can
+	// reliably stop/restart nodes.
+	if _, err := mc.WaitForTopologyId(waitTimeout); err != nil {
+		mc.DumpLogs()
+		mc.StopAll()
+		t.Fatalf("TopologyId not generated: %v", err)
+	}
+
+	return mc
+}
+
+// NewMasterCluster allocates ports and data directories for a 3-node master
+// cluster without starting anything, so a test can choose what each node comes
+// up with.
+func NewMasterCluster(t testing.TB, raftHashicorp bool) *MasterCluster {
+	t.Helper()
+
+	weedBinary, err := findOrBuildWeedBinary()
+	if err != nil {
+		t.Fatalf("resolve weed binary: %v", err)
+	}
+
+	keepLogs := os.Getenv("MULTI_MASTER_IT_KEEP_LOGS") == "1"
+	baseDir, err := os.MkdirTemp("", "seaweedfs_multi_master_it_")
+	if err != nil {
+		t.Fatalf("create temp dir: %v", err)
+	}
+	logsDir := filepath.Join(baseDir, "logs")
+	os.MkdirAll(logsDir, 0o755)
+
+	// Allocate 3 mini-safe ports (each guarantees port+10000 is also free).
+	httpPorts, err := testutil.AllocateMiniPorts(3)
+	if err != nil {
+		t.Fatalf("allocate ports: %v", err)
+	}
+	var nodes [3]*masterNode
+	var peerParts []string
+	for i, hp := range httpPorts {
+		dataDir := filepath.Join(baseDir, fmt.Sprintf("m%d", i))
+		os.MkdirAll(dataDir, 0o755)
+		nodes[i] = &masterNode{
+			port:     hp,
+			grpcPort: hp + testutil.GrpcPortOffset,
+			dataDir:  dataDir,
+			logFile:  filepath.Join(logsDir, fmt.Sprintf("master%d.log", i)),
+		}
+		peerParts = append(peerParts, fmt.Sprintf("127.0.0.1:%d", hp))
+	}
+
+	mc := &MasterCluster{
+		t:             t,
+		weedBinary:    weedBinary,
+		baseDir:       baseDir,
+		logsDir:       logsDir,
+		keepLogs:      keepLogs,
+		nodes:         nodes,
+		peersStr:      strings.Join(peerParts, ","),
+		raftHashicorp: raftHashicorp,
+	}
+
+	t.Cleanup(func() {
+		mc.StopAll()
+	})
+	return mc
+}
+
+// SetNodePeers narrows the peer list node i starts with, mirroring a
+// StatefulSet whose replica count changed under a running master.
+func (mc *MasterCluster) SetNodePeers(i int, peers string) {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+	mc.nodes[i].peersStr = peers
+}
+
+// SetRaftBootstrap makes node i start with -raftBootstrap.
+func (mc *MasterCluster) SetRaftBootstrap(i int) {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+	mc.nodes[i].raftBootstrap = true
+}
+
+// StartNode starts the master process at the given index (0–2).
+func (mc *MasterCluster) StartNode(i int) {
+	mc.t.Helper()
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+
+	n := mc.nodes[i]
+	if n.cmd != nil && !n.stopped {
+		return // already running
+	}
+
+	logFile, err := os.OpenFile(n.logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		mc.t.Fatalf("create log for node %d: %v", i, err)
+	}
+
+	peersStr := n.peersStr
+	if peersStr == "" {
+		peersStr = mc.peersStr
+	}
+	args := []string{
+		"master",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(n.port),
+		"-port.grpc=" + strconv.Itoa(n.grpcPort),
+		"-mdir=" + n.dataDir,
+		"-peers=" + peersStr,
+		"-electionTimeout=3s",
+		"-volumeSizeLimitMB=32",
+		"-defaultReplication=000",
+	}
+	if mc.raftHashicorp {
+		args = append(args, "-raftHashicorp")
+	}
+	if n.raftBootstrap {
+		args = append(args, "-raftBootstrap")
+	}
+
+	n.cmd = exec.Command(mc.weedBinary, args...)
+	n.cmd.Dir = mc.baseDir
+	n.cmd.Stdout = logFile
+	n.cmd.Stderr = logFile
+	n.stopped = false
+	if err := n.cmd.Start(); err != nil {
+		mc.t.Fatalf("start node %d: %v", i, err)
+	}
+}
+
+// StopNode gracefully stops the master at the given index.
+func (mc *MasterCluster) StopNode(i int) {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+	mc.stopNodeLocked(i)
+}
+
+func (mc *MasterCluster) stopNodeLocked(i int) {
+	n := mc.nodes[i]
+	if n.cmd == nil || n.stopped {
+		return
+	}
+	n.stopped = true
+	_ = n.cmd.Process.Signal(os.Interrupt)
+	done := make(chan error, 1)
+	go func() { done <- n.cmd.Wait() }()
+	select {
+	case <-time.After(10 * time.Second):
+		_ = n.cmd.Process.Kill()
+		<-done
+	case <-done:
+	}
+}
+
+// StopAll stops all running master nodes.
+func (mc *MasterCluster) StopAll() {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+	for i := range 3 {
+		mc.stopNodeLocked(i)
+	}
+	if !mc.keepLogs && !mc.t.Failed() {
+		os.RemoveAll(mc.baseDir)
+	} else if mc.baseDir != "" {
+		mc.t.Logf("multi-master logs kept at %s", mc.baseDir)
+	}
+}
+
+// NodeURL returns the HTTP URL for node i.
+func (mc *MasterCluster) NodeURL(i int) string {
+	return fmt.Sprintf("http://127.0.0.1:%d", mc.nodes[i].port)
+}
+
+// NodeAddress returns "127.0.0.1:port" for node i.
+func (mc *MasterCluster) NodeAddress(i int) string {
+	return fmt.Sprintf("127.0.0.1:%d", mc.nodes[i].port)
+}
+
+// NodeGRPCAddress returns "127.0.0.1:grpcPort" for node i.
+func (mc *MasterCluster) NodeGRPCAddress(i int) string {
+	return fmt.Sprintf("127.0.0.1:%d", mc.nodes[i].grpcPort)
+}
+
+// IsNodeRunning returns true if the node at index i has a live process.
+func (mc *MasterCluster) IsNodeRunning(i int) bool {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+	n := mc.nodes[i]
+	return n.cmd != nil && !n.stopped
+}
+
+// GetClusterStatus fetches /cluster/status from node i.
+func (mc *MasterCluster) GetClusterStatus(i int) (*clusterStatus, error) {
+	client := &http.Client{Timeout: 2 * time.Second}
+	resp, err := client.Get(mc.NodeURL(i) + "/cluster/status")
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(resp.Body)
+	var cs clusterStatus
+	if err := json.Unmarshal(body, &cs); err != nil {
+		return nil, fmt.Errorf("parse cluster/status: %w (body: %s)", err, string(body))
+	}
+	return &cs, nil
+}
+
+// GetTopologyId fetches the TopologyId from /dir/status on node i.
+func (mc *MasterCluster) GetTopologyId(i int) (string, error) {
+	client := &http.Client{Timeout: 2 * time.Second}
+	resp, err := client.Get(mc.NodeURL(i) + "/dir/status")
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(resp.Body)
+	var raw map[string]any
+	if err := json.Unmarshal(body, &raw); err != nil {
+		return "", err
+	}
+	if id, ok := raw["TopologyId"].(string); ok {
+		return id, nil
+	}
+	return "", nil
+}
+
+// FindLeader returns the index of the leader node and its address.
+// Returns -1 if no leader is found.
+func (mc *MasterCluster) FindLeader() (int, string) {
+	for i := range 3 {
+		if !mc.IsNodeRunning(i) {
+			continue
+		}
+		cs, err := mc.GetClusterStatus(i)
+		if err != nil {
+			continue
+		}
+		if cs.IsLeader {
+			return i, mc.NodeAddress(i)
+		}
+	}
+	return -1, ""
+}
+
+// WaitForLeader polls until a leader is elected or timeout.
+func (mc *MasterCluster) WaitForLeader(timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if idx, _ := mc.FindLeader(); idx >= 0 {
+			return nil
+		}
+		time.Sleep(waitTick)
+	}
+	return fmt.Errorf("no leader elected within %v", timeout)
+}
+
+// WaitForNoLeader waits until no running master claims leadership. goraft only
+// checks whether it still has a quorum on an election-timeout ticker, and needs
+// its peers to go quiet for a full timeout first, so a master that has lost its
+// quorum can keep claiming leadership for tens of seconds. It cannot commit
+// anything in that window.
+func (mc *MasterCluster) WaitForNoLeader(timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		idx, _ := mc.FindLeader()
+		if idx < 0 {
+			return nil
+		}
+		time.Sleep(waitTick)
+	}
+	idx, addr := mc.FindLeader()
+	return fmt.Errorf("master %d at %s still claims leadership after %v", idx, addr, timeout)
+}
+
+// WaitForNewLeader waits for a leader that is different from the given address.
+func (mc *MasterCluster) WaitForNewLeader(oldLeaderAddr string, timeout time.Duration) (int, string, error) {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		idx, addr := mc.FindLeader()
+		if idx >= 0 && addr != oldLeaderAddr {
+			return idx, addr, nil
+		}
+		time.Sleep(waitTick)
+	}
+	return -1, "", fmt.Errorf("no new leader (different from %s) within %v", oldLeaderAddr, timeout)
+}
+
+// WaitForTopologyId waits until the leader reports a non-empty TopologyId, and
+// returns it. It is only readable once the leader has applied the raft entry
+// carrying it, which lands after the election it won.
+func (mc *MasterCluster) WaitForTopologyId(timeout time.Duration) (string, error) {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if idx, _ := mc.FindLeader(); idx >= 0 {
+			if id, err := mc.GetTopologyId(idx); err == nil && id != "" {
+				return id, nil
+			}
+		}
+		time.Sleep(waitTick)
+	}
+	return "", fmt.Errorf("TopologyId not available within %v", timeout)
+}
+
+// WaitForNodeReady waits for node i to respond to HTTP.
+func (mc *MasterCluster) WaitForNodeReady(i int, timeout time.Duration) error {
+	client := &http.Client{Timeout: 1 * time.Second}
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		resp, err := client.Get(mc.NodeURL(i) + "/cluster/status")
+		if err == nil {
+			resp.Body.Close()
+			return nil
+		}
+		time.Sleep(waitTick)
+	}
+	return fmt.Errorf("node %d not ready within %v", i, timeout)
+}
+
+// LogContains reports whether node i's log holds the given text.
+func (mc *MasterCluster) LogContains(i int, text string) bool {
+	b, err := os.ReadFile(mc.nodes[i].logFile)
+	if err != nil {
+		return false
+	}
+	return strings.Contains(string(b), text)
+}
+
+// topologyIdLine matches every line a master logs when it learns a TopologyId,
+// whichever raft implementation applied it.
+var topologyIdLine = regexp.MustCompile(`TopologyId[^:]*: ([0-9a-f-]{36})`)
+
+// NodeTopologyIds returns the TopologyIds node i has logged. /dir/status is
+// proxied to the leader, so a master's own view of the cluster identity is only
+// visible in its log, and that is where a fork shows up.
+func (mc *MasterCluster) NodeTopologyIds(i int) []string {
+	b, err := os.ReadFile(mc.nodes[i].logFile)
+	if err != nil {
+		return nil
+	}
+	var ids []string
+	for _, m := range topologyIdLine.FindAllStringSubmatch(string(b), -1) {
+		ids = append(ids, m[1])
+	}
+	return ids
+}
+
+// WaitForNodeTopologyIds waits until every master has logged a TopologyId and
+// returns what each one saw.
+func (mc *MasterCluster) WaitForNodeTopologyIds(timeout time.Duration) ([3][]string, error) {
+	var ids [3][]string
+	deadline := time.Now().Add(timeout)
+	for {
+		missing := -1
+		for i := range 3 {
+			ids[i] = mc.NodeTopologyIds(i)
+			if len(ids[i]) == 0 {
+				missing = i
+			}
+		}
+		if missing < 0 {
+			return ids, nil
+		}
+		if !time.Now().Before(deadline) {
+			return ids, fmt.Errorf("master %d logged no TopologyId within %v", missing, timeout)
+		}
+		time.Sleep(waitTick)
+	}
+}
+
+// DumpLogs prints the tail of all master logs.
+func (mc *MasterCluster) DumpLogs() {
+	for i := range 3 {
+		mc.t.Logf("=== master%d log tail ===\n%s", i, mc.tailLog(i))
+	}
+}
+
+func (mc *MasterCluster) tailLog(i int) string {
+	f, err := os.Open(mc.nodes[i].logFile)
+	if err != nil {
+		return "(no log)"
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	lines := make([]string, 0, 50)
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+		if len(lines) > 50 {
+			lines = lines[1:]
+		}
+	}
+	return strings.Join(lines, "\n")
+}
+
+func findOrBuildWeedBinary() (string, error) {
+	if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" {
+		if isExecutableFile(fromEnv) {
+			return fromEnv, nil
+		}
+		return "", fmt.Errorf("WEED_BINARY not executable: %s", fromEnv)
+	}
+
+	repoRoot := ""
+	if _, file, _, ok := runtime.Caller(0); ok {
+		repoRoot = filepath.Clean(filepath.Join(filepath.Dir(file), "..", ".."))
+	}
+	if repoRoot == "" {
+		return "", fmt.Errorf("unable to detect repository root")
+	}
+
+	// Check if already built
+	binDir := filepath.Join(os.TempDir(), "seaweedfs_multi_master_it_bin")
+	os.MkdirAll(binDir, 0o755)
+	binPath := filepath.Join(binDir, "weed")
+	if isExecutableFile(binPath) {
+		return binPath, nil
+	}
+
+	cmd := exec.Command("go", "build", "-o", binPath, ".")
+	cmd.Dir = filepath.Join(repoRoot, "weed")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &out
+	if err := cmd.Run(); err != nil {
+		return "", fmt.Errorf("build weed binary: %w\n%s", err, out.String())
+	}
+	return binPath, nil
+}
+
+func isExecutableFile(path string) bool {
+	info, err := os.Stat(path)
+	if err != nil || info.IsDir() {
+		return false
+	}
+	return info.Mode().Perm()&0o111 != 0
+}

test/multi_master/cluster_membership_test.go

@@ -0,0 +1,103 @@
+package multi_master
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+// A filer only learns about its peers from the cluster node updates on its
+// KeepConnected stream, and those are broadcast to whoever is connected at that
+// moment. A filer that reconnects has to be told the membership again, or it
+// never subscribes to the peers that registered while it was away.
+func TestKeepConnectedSendsExistingFilers(t *testing.T) {
+	mc := StartMasterCluster(t)
+
+	leaderIdx, leaderAddr := mc.FindLeader()
+	if leaderIdx < 0 {
+		t.Fatal("no leader")
+	}
+	master := pb.ServerAddress(leaderAddr)
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+
+	const existingFiler = "127.0.0.1:18888"
+	const joiningFiler = "127.0.0.1:18889"
+
+	ctx, cancel := context.WithTimeout(context.Background(), waitTimeout)
+	defer cancel()
+
+	err := pb.WithMasterClient(ctx, true, master, dialOption, false, func(client master_pb.SeaweedClient) error {
+		stream, err := client.KeepConnected(ctx)
+		if err != nil {
+			return err
+		}
+		if err := stream.Send(&master_pb.KeepConnectedRequest{
+			ClientType:    cluster.FilerType,
+			ClientAddress: existingFiler,
+		}); err != nil {
+			return err
+		}
+		if err := waitForClusterNode(ctx, client, existingFiler); err != nil {
+			return err
+		}
+
+		return pb.WithMasterClient(ctx, true, master, dialOption, false, func(joining master_pb.SeaweedClient) error {
+			joiningCtx, cancelJoining := context.WithTimeout(ctx, waitTimeout)
+			defer cancelJoining()
+
+			joiningStream, err := joining.KeepConnected(joiningCtx)
+			if err != nil {
+				return err
+			}
+			if err := joiningStream.Send(&master_pb.KeepConnectedRequest{
+				ClientType:    cluster.FilerType,
+				ClientAddress: joiningFiler,
+			}); err != nil {
+				return err
+			}
+			for i := 0; ; i++ {
+				resp, err := joiningStream.Recv()
+				if err != nil {
+					return err
+				}
+				// a client only reads the volume locations out of the first
+				// message, an update sent ahead of them would be dropped
+				if i == 0 && resp.VolumeLocation == nil {
+					return fmt.Errorf("first message is not a volume location: %+v", resp)
+				}
+				if update := resp.ClusterNodeUpdate; update != nil && update.IsAdd && update.Address == existingFiler {
+					return nil
+				}
+			}
+		})
+	})
+	if err != nil {
+		mc.DumpLogs()
+		t.Fatalf("a joining filer was not told about %s: %v", existingFiler, err)
+	}
+}
+
+func waitForClusterNode(ctx context.Context, client master_pb.SeaweedClient, address string) error {
+	deadline := time.Now().Add(waitTimeout)
+	for time.Now().Before(deadline) {
+		resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{ClientType: cluster.FilerType})
+		if err != nil {
+			return err
+		}
+		for _, node := range resp.ClusterNodes {
+			if node.Address == address {
+				return nil
+			}
+		}
+		time.Sleep(waitTick)
+	}
+	return context.DeadlineExceeded
+}

test/multi_master/failover_test.go

@@ -0,0 +1,309 @@
+package multi_master
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+const (
+	// Election timeout is 3s in our cluster config; allow generous margin.
+	leaderElectionTimeout = 30 * time.Second
+
+	// Losing a quorum is much slower to show than winning one. goraft only
+	// notices on an election-timeout ticker, after its peers have been quiet
+	// for a full timeout, and has been measured taking over 30s to step down.
+	leaderStepDownTimeout = 60 * time.Second
+)
+
+// TestLeaderDownAndRecoverQuickly verifies that when the leader is stopped and
+// restarted quickly, the cluster re-elects a leader and the restarted node
+// rejoins as a follower. TopologyId must be consistent across all nodes.
+func TestLeaderDownAndRecoverQuickly(t *testing.T) {
+	mc := StartMasterCluster(t)
+
+	// Record initial state.
+	leaderIdx, leaderAddr := mc.FindLeader()
+	if leaderIdx < 0 {
+		t.Fatal("no leader found after cluster start")
+	}
+	t.Logf("initial leader: node %d at %s", leaderIdx, leaderAddr)
+
+	topologyId, err := mc.GetTopologyId(leaderIdx)
+	if err != nil || topologyId == "" {
+		t.Fatalf("failed to get initial TopologyId: %v", err)
+	}
+	t.Logf("initial TopologyId: %s", topologyId)
+
+	// Stop the leader.
+	mc.StopNode(leaderIdx)
+	t.Logf("stopped leader node %d", leaderIdx)
+
+	// Wait for a new leader from the remaining 2 nodes.
+	newLeaderIdx, newLeaderAddr, err := mc.WaitForNewLeader(leaderAddr, leaderElectionTimeout)
+	if err != nil {
+		mc.DumpLogs()
+		t.Fatalf("new leader not elected after stopping old leader: %v", err)
+	}
+	t.Logf("new leader: node %d at %s", newLeaderIdx, newLeaderAddr)
+
+	// Restart the old leader quickly.
+	mc.StartNode(leaderIdx)
+	if err := mc.WaitForNodeReady(leaderIdx, waitTimeout); err != nil {
+		mc.DumpLogs()
+		t.Fatalf("restarted node %d not ready: %v", leaderIdx, err)
+	}
+	t.Logf("restarted node %d", leaderIdx)
+
+	// Give raft time to settle.
+	time.Sleep(3 * time.Second)
+
+	// Verify leader is stable.
+	finalLeaderIdx, _ := mc.FindLeader()
+	if finalLeaderIdx < 0 {
+		mc.DumpLogs()
+		t.Fatal("no leader after restarting old leader node")
+	}
+
+	// Verify TopologyId is consistent across all nodes.
+	assertTopologyIdConsistent(t, mc, topologyId)
+}
+
+// TestLeaderDownSlowRecover verifies that when the leader goes down and takes
+// a long time to come back, the remaining 2 nodes elect a new leader and the
+// cluster continues to function. When the slow node returns, it rejoins.
+func TestLeaderDownSlowRecover(t *testing.T) {
+	mc := StartMasterCluster(t)
+
+	leaderIdx, leaderAddr := mc.FindLeader()
+	if leaderIdx < 0 {
+		t.Fatal("no leader found")
+	}
+	topologyId, err := mc.GetTopologyId(leaderIdx)
+	if err != nil || topologyId == "" {
+		t.Fatalf("failed to get initial TopologyId: %v", err)
+	}
+	t.Logf("initial leader: node %d, TopologyId: %s", leaderIdx, topologyId)
+
+	// Stop the leader.
+	mc.StopNode(leaderIdx)
+
+	// Wait for a new leader.
+	newLeaderIdx, _, err := mc.WaitForNewLeader(leaderAddr, leaderElectionTimeout)
+	if err != nil {
+		mc.DumpLogs()
+		t.Fatalf("new leader not elected: %v", err)
+	}
+	t.Logf("new leader: node %d", newLeaderIdx)
+
+	// Verify cluster functions with only 2 nodes (quorum is 2/3).
+	cs, err := mc.GetClusterStatus(newLeaderIdx)
+	if err != nil {
+		mc.DumpLogs()
+		t.Fatalf("cannot get cluster status from new leader: %v", err)
+	}
+	if !cs.IsLeader {
+		t.Fatalf("node %d claims not to be leader", newLeaderIdx)
+	}
+
+	// Simulate slow recovery: wait significantly longer than election timeout.
+	t.Log("simulating slow recovery (10 seconds)...")
+	time.Sleep(10 * time.Second)
+
+	// Verify leader is still stable during the outage.
+	stableLeaderIdx, _ := mc.FindLeader()
+	if stableLeaderIdx < 0 {
+		mc.DumpLogs()
+		t.Fatal("leader lost during extended outage of one node")
+	}
+
+	// Restart the downed node.
+	mc.StartNode(leaderIdx)
+	if err := mc.WaitForNodeReady(leaderIdx, waitTimeout); err != nil {
+		mc.DumpLogs()
+		t.Fatalf("slow-recovered node %d not ready: %v", leaderIdx, err)
+	}
+
+	time.Sleep(3 * time.Second)
+	assertTopologyIdConsistent(t, mc, topologyId)
+}
+
+// TestTwoMastersDownAndRestart verifies that when 2 of 3 masters go down
+// (losing quorum), the cluster cannot elect a leader. When both restart,
+// a leader is elected and TopologyId is preserved.
+func TestTwoMastersDownAndRestart(t *testing.T) {
+	mc := StartMasterCluster(t)
+
+	leaderIdx, _ := mc.FindLeader()
+	if leaderIdx < 0 {
+		t.Fatal("no leader found")
+	}
+	topologyId, err := mc.GetTopologyId(leaderIdx)
+	if err != nil || topologyId == "" {
+		t.Fatalf("failed to get initial TopologyId: %v", err)
+	}
+	t.Logf("initial TopologyId: %s", topologyId)
+
+	// Determine which 2 nodes to stop (stop the leader + one follower).
+	down1 := leaderIdx
+	down2 := (leaderIdx + 1) % 3
+	survivor := (leaderIdx + 2) % 3
+	t.Logf("stopping nodes %d and %d, keeping node %d", down1, down2, survivor)
+
+	mc.StopNode(down1)
+	mc.StopNode(down2)
+
+	// The surviving node alone cannot form a quorum, so it has to give up the
+	// leadership it is still claiming.
+	if err := mc.WaitForNoLeader(leaderStepDownTimeout); err != nil {
+		mc.DumpLogs()
+		t.Fatalf("expected no leader with only 1 of 3 nodes: %v", err)
+	}
+
+	// Restart both downed nodes.
+	mc.StartNode(down1)
+	mc.StartNode(down2)
+	for _, i := range []int{down1, down2} {
+		if err := mc.WaitForNodeReady(i, waitTimeout); err != nil {
+			mc.DumpLogs()
+			t.Fatalf("restarted node %d not ready: %v", i, err)
+		}
+	}
+
+	// Wait for leader election.
+	if err := mc.WaitForLeader(leaderElectionTimeout); err != nil {
+		mc.DumpLogs()
+		t.Fatalf("no leader after restarting 2 downed nodes: %v", err)
+	}
+
+	time.Sleep(3 * time.Second)
+	assertTopologyIdConsistent(t, mc, topologyId)
+}
+
+// TestAllMastersDownAndRestart verifies that when all 3 masters are stopped
+// and restarted, the cluster elects a leader and all nodes agree on a
+// TopologyId. With RaftResumeState=false (default), raft state is cleared on
+// restart. The TopologyId is recovered from snapshots when available; on a
+// short-lived cluster that hasn't taken snapshots on all nodes, a new
+// TopologyId may be generated — but all nodes must still agree.
+func TestAllMastersDownAndRestart(t *testing.T) {
+	mc := StartMasterCluster(t)
+
+	leaderIdx, _ := mc.FindLeader()
+	if leaderIdx < 0 {
+		t.Fatal("no leader found")
+	}
+	topologyId, _ := mc.GetTopologyId(leaderIdx)
+	if topologyId == "" {
+		t.Fatal("no TopologyId on initial leader")
+	}
+	t.Logf("initial TopologyId: %s", topologyId)
+
+	// Stop all nodes.
+	for i := range 3 {
+		mc.StopNode(i)
+	}
+	t.Log("all nodes stopped")
+
+	time.Sleep(2 * time.Second)
+
+	// Restart all nodes.
+	for i := range 3 {
+		mc.StartNode(i)
+	}
+	for i := range 3 {
+		if err := mc.WaitForNodeReady(i, waitTimeout); err != nil {
+			mc.DumpLogs()
+			t.Fatalf("node %d not ready after full restart: %v", i, err)
+		}
+	}
+
+	// Wait for leader.
+	if err := mc.WaitForLeader(leaderElectionTimeout); err != nil {
+		mc.DumpLogs()
+		t.Fatalf("no leader after full cluster restart: %v", err)
+	}
+
+	newLeaderIdx, _ := mc.FindLeader()
+	t.Logf("leader after full restart: node %d", newLeaderIdx)
+
+	time.Sleep(3 * time.Second)
+
+	// All nodes must agree on a TopologyId (may differ from original if
+	// snapshots were not yet taken on all nodes before shutdown).
+	newTopologyId, err := mc.GetTopologyId(newLeaderIdx)
+	if err != nil || newTopologyId == "" {
+		mc.DumpLogs()
+		t.Fatal("no TopologyId after full restart")
+	}
+	if newTopologyId == topologyId {
+		t.Logf("TopologyId preserved across full restart: %s", topologyId)
+	} else {
+		t.Logf("TopologyId changed (expected for short-lived cluster without snapshots): %s -> %s", topologyId, newTopologyId)
+	}
+	assertTopologyIdConsistent(t, mc, newTopologyId)
+}
+
+// TestLeaderConsistencyAcrossNodes verifies that all nodes agree on who the
+// leader is and report the same TopologyId.
+func TestLeaderConsistencyAcrossNodes(t *testing.T) {
+	mc := StartMasterCluster(t)
+
+	// Allow cluster to stabilize.
+	time.Sleep(3 * time.Second)
+
+	leaderIdx, leaderAddr := mc.FindLeader()
+	if leaderIdx < 0 {
+		t.Fatal("no leader found")
+	}
+	t.Logf("leader: node %d at %s", leaderIdx, leaderAddr)
+
+	// Every node should agree on the leader.
+	for i := range 3 {
+		cs, err := mc.GetClusterStatus(i)
+		if err != nil {
+			t.Fatalf("node %d cluster/status error: %v", i, err)
+		}
+		if i == leaderIdx {
+			if !cs.IsLeader {
+				t.Errorf("node %d should be leader but IsLeader=false", i)
+			}
+		} else {
+			if cs.IsLeader {
+				t.Errorf("node %d should not be leader but IsLeader=true", i)
+			}
+			// cs.Leader is a ServerAddress like "127.0.0.1:10000.20000";
+			// convert to HTTP address for comparison with leaderAddr.
+			leaderHttp := pb.ServerAddress(cs.Leader).ToHttpAddress()
+			if leaderHttp != leaderAddr {
+				t.Errorf("node %d reports leader %q (http: %s), expected %q", i, cs.Leader, leaderHttp, leaderAddr)
+			}
+		}
+	}
+
+	// All nodes should have the same TopologyId.
+	topologyId, _ := mc.GetTopologyId(leaderIdx)
+	if topologyId == "" {
+		t.Fatal("leader has no TopologyId")
+	}
+	assertTopologyIdConsistent(t, mc, topologyId)
+}
+
+// assertTopologyIdConsistent verifies that all running nodes report the expected TopologyId.
+func assertTopologyIdConsistent(t *testing.T, mc *MasterCluster, expectedId string) {
+	t.Helper()
+	for i := range 3 {
+		if !mc.IsNodeRunning(i) {
+			continue
+		}
+		id, err := mc.GetTopologyId(i)
+		if err != nil {
+			t.Errorf("node %d: failed to get TopologyId: %v", i, err)
+			continue
+		}
+		if id != expectedId {
+			t.Errorf("node %d: TopologyId=%q, expected %q", i, id, expectedId)
+		}
+	}
+}

test/multi_master/join_test.go

@@ -0,0 +1,212 @@
+package multi_master
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+var raftImplementations = []struct {
+	name          string
+	raftHashicorp bool
+}{
+	{"goraft", false},
+	{"hashicorp", true},
+}
+
+// TestFreshClusterFormsWithoutALeader covers the other half of the bootstrap
+// decision: with no leader anywhere, three masters starting together still have
+// to mint a cluster between them.
+func TestFreshClusterFormsWithoutALeader(t *testing.T) {
+	for _, impl := range raftImplementations {
+		t.Run(impl.name, func(t *testing.T) {
+			mc := NewMasterCluster(t, impl.raftHashicorp)
+			for i := range 3 {
+				mc.StartNode(i)
+			}
+
+			if _, err := waitForCommonLeader(mc, waitTimeout); err != nil {
+				mc.DumpLogs()
+				t.Fatalf("fresh cluster did not converge: %v", err)
+			}
+			for i := range 3 {
+				if err := waitForPeerCount(mc, i, 2, waitTimeout); err != nil {
+					mc.DumpLogs()
+					t.Fatalf("master %d does not see the full cluster: %v", i, err)
+				}
+			}
+		})
+	}
+}
+
+// TestScaleUpOntoExistingLeader mirrors a Kubernetes master StatefulSet whose
+// replica count goes back from one to three: master 0 keeps running as the
+// leader of a single-peer cluster while two fresh masters come up pointing at
+// all three. The newcomers start with an empty raft log, so neither raft
+// implementation lets them campaign — the sitting leader has to admit them.
+func TestScaleUpOntoExistingLeader(t *testing.T) {
+	for _, impl := range raftImplementations {
+		t.Run(impl.name, func(t *testing.T) {
+			mc := NewMasterCluster(t, impl.raftHashicorp)
+
+			// Master 0 is alone in its peer list, the way the operator renders
+			// -peers when spec.master.replicas is 1.
+			mc.SetNodePeers(0, mc.NodeAddress(0))
+			mc.StartNode(0)
+			if err := mc.WaitForLeader(waitTimeout); err != nil {
+				mc.DumpLogs()
+				t.Fatalf("single master did not become leader: %v", err)
+			}
+
+			// Scale up. These two carry the full peer list; master 0 still
+			// runs with the old one and has never heard of them.
+			mc.StartNode(1)
+			mc.StartNode(2)
+
+			leader, err := waitForCommonLeader(mc, waitTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatalf("masters did not converge after scaling up: %v", err)
+			}
+			if leader != mc.NodeAddress(0) {
+				t.Fatalf("leader moved to %s, want the sitting leader %s", leader, mc.NodeAddress(0))
+			}
+
+			if err := waitForPeerCount(mc, 0, 2, waitTimeout); err != nil {
+				mc.DumpLogs()
+				t.Fatalf("leader did not admit both new masters: %v", err)
+			}
+		})
+	}
+}
+
+// waitForCommonLeader waits until every running master names the same leader,
+// and returns it.
+func waitForCommonLeader(mc *MasterCluster, timeout time.Duration) (string, error) {
+	var lastErr error
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		leader, err := commonLeader(mc)
+		if err == nil {
+			return leader, nil
+		}
+		lastErr = err
+		time.Sleep(waitTick)
+	}
+	return "", lastErr
+}
+
+func commonLeader(mc *MasterCluster) (string, error) {
+	agreed := ""
+	for i := range 3 {
+		if !mc.IsNodeRunning(i) {
+			continue
+		}
+		cs, err := mc.GetClusterStatus(i)
+		if err != nil {
+			return "", err
+		}
+		leader := pb.ServerAddress(cs.Leader).ToHttpAddress()
+		if leader == "" {
+			return "", fmt.Errorf("master %d has no leader", i)
+		}
+		if agreed == "" {
+			agreed = leader
+		} else if agreed != leader {
+			return "", fmt.Errorf("masters disagree on the leader: %s and %s", agreed, leader)
+		}
+	}
+	if agreed == "" {
+		return "", fmt.Errorf("no master is running")
+	}
+	return agreed, nil
+}
+
+// waitForPeerCount waits until node i reports the given number of raft peers.
+// The count excludes the node itself.
+func waitForPeerCount(mc *MasterCluster, i, want int, timeout time.Duration) error {
+	got := -1
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		cs, err := mc.GetClusterStatus(i)
+		if err == nil {
+			got = peerCountExcludingSelf(cs.Peers, mc.NodeAddress(i))
+			if got == want {
+				return nil
+			}
+		}
+		time.Sleep(waitTick)
+	}
+	return fmt.Errorf("master %d reports %d peers, want %d", i, got, want)
+}
+
+func peerCountExcludingSelf(peers []string, self string) int {
+	count := 0
+	for _, peer := range peers {
+		if pb.ServerAddress(peer).ToHttpAddress() != self {
+			count++
+		}
+	}
+	return count
+}
+
+// TestRaftBootstrapKeepsExistingCluster covers a master restarting under
+// -raftBootstrap, the way the helm chart renders it on every master on every
+// roll. Bootstrapping is genesis: seeding a second cluster over committed raft
+// state mints a rival TopologyId, and the split-brain guard then Fatals every
+// master that still holds the first one.
+func TestRaftBootstrapKeepsExistingCluster(t *testing.T) {
+	for _, impl := range raftImplementations {
+		t.Run(impl.name, func(t *testing.T) {
+			mc := NewMasterCluster(t, impl.raftHashicorp)
+			for i := range 3 {
+				mc.SetRaftBootstrap(i)
+				mc.StartNode(i)
+			}
+			before, err := mc.WaitForTopologyId(waitTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatalf("cluster did not mint a TopologyId: %v", err)
+			}
+
+			for i := range 3 {
+				mc.StopNode(i)
+			}
+			for i := range 3 {
+				mc.StartNode(i)
+			}
+
+			after, err := mc.WaitForTopologyId(waitTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatalf("cluster did not come back after a restart: %v", err)
+			}
+			if after != before {
+				mc.DumpLogs()
+				t.Fatalf("-raftBootstrap re-seeded the cluster: TopologyId %s became %s", before, after)
+			}
+
+			// The leader answers for the whole cluster, so a follower that
+			// forked is only visible in its own log.
+			seen, err := mc.WaitForNodeTopologyIds(waitTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatal(err)
+			}
+			for i, ids := range seen {
+				for _, id := range ids {
+					if id != before {
+						mc.DumpLogs()
+						t.Fatalf("master %d saw TopologyId %s, want %s", i, id, before)
+					}
+				}
+				if mc.LogContains(i, "Split-brain detected") {
+					mc.DumpLogs()
+					t.Fatalf("master %d hit the split-brain guard", i)
+				}
+			}
+		})
+	}
+}