snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/metadata_subscribe/Makefile

@@ -0,0 +1,28 @@
+.PHONY: test test-short build-weed clean
+
+# Build weed binary first if needed
+build-weed:
+	cd ../../weed && go build -o weed .
+
+# Run all integration tests
+test: build-weed
+	go test -v -timeout 5m .
+
+# Run tests in short mode (skip integration tests)
+test-short:
+	go test -v -short .
+
+# Run specific test
+test-basic: build-weed
+	go test -v -timeout 3m -run TestMetadataSubscribeBasic .
+
+test-stall: build-weed
+	go test -v -timeout 5m -run TestMetadataSubscribeSingleFilerNoStall .
+
+test-resume: build-weed
+	go test -v -timeout 3m -run TestMetadataSubscribeResumeFromDisk .
+
+clean:
+	rm -f ../../weed/weed
+
+

test/metadata_subscribe/README.md

@@ -0,0 +1,50 @@
+# Metadata Subscribe Integration Tests
+
+This directory contains integration tests for the SeaweedFS metadata subscription functionality.
+
+## Tests
+
+### TestMetadataSubscribeBasic
+Tests basic metadata subscription functionality:
+- Start a SeaweedFS cluster (master, volume, filer)
+- Subscribe to metadata changes
+- Upload files and verify events are received
+
+### TestMetadataSubscribeSingleFilerNoStall
+Regression test for [issue #4977](https://github.com/seaweedfs/seaweedfs/issues/4977):
+- Tests that metadata subscription doesn't stall in single-filer setups
+- Simulates high-load file uploads while a subscriber tries to keep up
+- Verifies that events are received without significant stalling
+
+The bug was that in single-filer setups, `SubscribeMetadata` would block indefinitely
+on `MetaAggregator.MetaLogBuffer` which remains empty (no peers to aggregate from).
+The fix ensures that when the buffer is empty, the subscription returns to read from
+persisted logs on disk.
+
+### TestMetadataSubscribeResumeFromDisk
+Tests that subscription can resume from disk:
+- Upload files before starting subscription
+- Wait for logs to be flushed to disk
+- Start subscription from the beginning
+- Verify pre-uploaded files are received from disk
+
+## Running Tests
+
+```bash
+# Run all tests (requires weed binary in PATH or built)
+go test -v ./test/metadata_subscribe/...
+
+# Skip integration tests
+go test -short ./test/metadata_subscribe/...
+
+# Run with increased timeout for slow systems
+go test -v -timeout 5m ./test/metadata_subscribe/...
+```
+
+## Requirements
+
+- `weed` binary must be available in PATH or in the parent directories
+- Tests create temporary directories that are cleaned up after completion
+- Tests use ports 9333 (master), 8080 (volume), 8888 (filer)
+
+

test/metadata_subscribe/peer_resubscribe_test.go

@@ -0,0 +1,248 @@
+//go:build !windows
+
+package metadata_subscribe
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+// A filer drops the metadata subscription to a peer that leaves, and only an
+// add from the master brings it back. Those updates are broadcast to the
+// clients connected at that moment, so a filer whose master stream broke while
+// the peer came back used to stay unsubscribed for good, and metadata written
+// on the peer never reached it again.
+func TestFilerResubscribesToPeerAfterMasterReconnect(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	weedBinary := findWeedBinary()
+	require.NotEmpty(t, weedBinary, "weed binary not found")
+
+	testDir, err := os.MkdirTemp("", "seaweedfs_peer_resubscribe_")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		if t.Failed() {
+			t.Logf("logs kept at %s", testDir)
+			return
+		}
+		os.RemoveAll(testDir)
+	})
+
+	ports, err := testutil.AllocateMiniPorts(3)
+	require.NoError(t, err)
+	masterPort, filer1Port, filer2Port := ports[0], ports[1], ports[2]
+
+	master := pb.ServerAddress(fmt.Sprintf("127.0.0.1:%d", masterPort))
+	filer1Address := fmt.Sprintf("127.0.0.1:%d", filer1Port)
+	filer2Address := fmt.Sprintf("127.0.0.1:%d", filer2Port)
+	filer1Log := filepath.Join(testDir, "filer1.log")
+
+	masterArgs := []string{"master",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(masterPort),
+		"-mdir=" + mkdir(t, testDir, "master"),
+		"-peers=none"}
+	masterProcess := startProcess(t, weedBinary, filepath.Join(testDir, "master.log"), masterArgs...)
+	require.NoError(t, waitForLeader(masterPort, 60*time.Second))
+
+	// one at a time: a filer bootstraps from the peers the master already knows,
+	// and gives up if one of them is registered but not yet listening
+	filer1 := startFiler(t, weedBinary, testDir, "filer1", filer1Port, masterPort)
+	require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer1Port), 30*time.Second))
+	filer2 := startFiler(t, weedBinary, testDir, "filer2", filer2Port, masterPort)
+	require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer2Port), 30*time.Second))
+
+	// the peer subscription works to begin with
+	createPeerEntry(t, filer2Address, "baseline")
+	require.NoError(t, waitForPeerEntry(filer1Address, "baseline", 60*time.Second),
+		"filer1 never replicated the baseline entry from filer2")
+
+	// filer2 leaves, and filer1 drops the subscription
+	stopProcess(filer2)
+	require.NoError(t, waitForLog(filer1Log, "stop subscribing peer "+filer2Address, 60*time.Second),
+		"filer1 never dropped the subscription to filer2")
+
+	// filer1 stops reading its master stream, and restarting the master breaks
+	// it, so filer1 hears nothing until it reconnects
+	require.NoError(t, filer1.Process.Signal(syscall.SIGSTOP))
+	stopProcess(masterProcess)
+	startProcess(t, weedBinary, filepath.Join(testDir, "master.log"), masterArgs...)
+	require.NoError(t, waitForLeader(masterPort, 60*time.Second))
+
+	// filer2 comes back and registers while filer1 cannot hear about it
+	filer2 = startFiler(t, weedBinary, testDir, "filer2", filer2Port, masterPort)
+	require.NoError(t, waitForHTTPServer(fmt.Sprintf("http://127.0.0.1:%d/", filer2Port), 30*time.Second))
+	require.NoError(t, waitForClusterNode(master, filer2Address, 60*time.Second),
+		"the master never registered filer2 again")
+
+	require.NoError(t, filer1.Process.Signal(syscall.SIGCONT))
+	require.NoError(t, waitForClusterNode(master, filer1Address, 60*time.Second),
+		"filer1 never reconnected to the master")
+
+	createPeerEntry(t, filer2Address, "after-reconnect")
+	require.NoError(t, waitForPeerEntry(filer1Address, "after-reconnect", 90*time.Second),
+		"filer1 reconnected to the master but never resubscribed to filer2")
+}
+
+const peerEntryDir = "/peer-resubscribe"
+
+func createPeerEntry(t *testing.T, filerAddress, name string) {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	err := pb.WithFilerClient(false, 0, pb.ServerAddress(filerAddress), grpc.WithTransportCredentials(insecure.NewCredentials()), func(client filer_pb.SeaweedFilerClient) error {
+		_, err := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{
+			Directory: peerEntryDir,
+			Entry: &filer_pb.Entry{
+				Name: name,
+				Attributes: &filer_pb.FuseAttributes{
+					Mtime:    time.Now().Unix(),
+					FileMode: 0644,
+				},
+			},
+		})
+		return err
+	})
+	require.NoError(t, err, "create %s/%s on %s", peerEntryDir, name, filerAddress)
+}
+
+func waitForPeerEntry(filerAddress, name string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	var lastErr error
+	for time.Now().Before(deadline) {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		lastErr = pb.WithFilerClient(false, 0, pb.ServerAddress(filerAddress), grpc.WithTransportCredentials(insecure.NewCredentials()), func(client filer_pb.SeaweedFilerClient) error {
+			_, err := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
+				Directory: peerEntryDir,
+				Name:      name,
+			})
+			return err
+		})
+		cancel()
+		if lastErr == nil {
+			return nil
+		}
+		time.Sleep(time.Second)
+	}
+	return fmt.Errorf("%s/%s not on %s within %v: %w", peerEntryDir, name, filerAddress, timeout, lastErr)
+}
+
+func waitForClusterNode(master pb.ServerAddress, address string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		found := false
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		err := pb.WithMasterClient(ctx, false, master, grpc.WithTransportCredentials(insecure.NewCredentials()), false, func(client master_pb.SeaweedClient) error {
+			resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{ClientType: cluster.FilerType})
+			if err != nil {
+				return err
+			}
+			for _, node := range resp.ClusterNodes {
+				// the master reports the grpc port too, as "host:port.grpcPort"
+				if pb.ServerAddress(node.Address).Equals(pb.ServerAddress(address)) {
+					found = true
+				}
+			}
+			return nil
+		})
+		cancel()
+		if err == nil && found {
+			return nil
+		}
+		time.Sleep(time.Second)
+	}
+	return fmt.Errorf("%s not registered within %v", address, timeout)
+}
+
+func waitForLeader(masterPort int, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	url := fmt.Sprintf("http://127.0.0.1:%d/cluster/status", masterPort)
+	client := &http.Client{Timeout: 2 * time.Second}
+	for time.Now().Before(deadline) {
+		if resp, err := client.Get(url); err == nil {
+			body, _ := io.ReadAll(resp.Body)
+			resp.Body.Close()
+			if strings.Contains(string(body), `"IsLeader":true`) {
+				return nil
+			}
+		}
+		time.Sleep(time.Second)
+	}
+	return fmt.Errorf("master on %d has no leader within %v", masterPort, timeout)
+}
+
+func waitForLog(logFile, message string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		content, err := os.ReadFile(logFile)
+		if err == nil && strings.Contains(string(content), message) {
+			return nil
+		}
+		time.Sleep(time.Second)
+	}
+	return fmt.Errorf("%q not in %s within %v", message, logFile, timeout)
+}
+
+func mkdir(t *testing.T, dir, name string) string {
+	t.Helper()
+	path := filepath.Join(dir, name)
+	require.NoError(t, os.MkdirAll(path, 0755))
+	return path
+}
+
+func startFiler(t *testing.T, weedBinary, testDir, name string, port, masterPort int) *exec.Cmd {
+	t.Helper()
+	return startProcess(t, weedBinary, filepath.Join(testDir, name+".log"), "filer",
+		"-ip=127.0.0.1",
+		"-port="+strconv.Itoa(port),
+		"-master=127.0.0.1:"+strconv.Itoa(masterPort),
+		"-defaultStoreDir="+mkdir(t, testDir, name))
+}
+
+func startProcess(t *testing.T, weedBinary, logFile string, args ...string) *exec.Cmd {
+	t.Helper()
+	log, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
+	require.NoError(t, err)
+
+	cmd := exec.Command(weedBinary, args...)
+	cmd.Stdout = log
+	cmd.Stderr = log
+	require.NoError(t, cmd.Start())
+	t.Cleanup(func() {
+		cmd.Process.Signal(syscall.SIGCONT)
+		stopProcess(cmd)
+	})
+	return cmd
+}
+
+// stopProcess kills the process outright: a filer takes seconds to shut down
+// gracefully, long enough to register with the master again on the way out.
+func stopProcess(cmd *exec.Cmd) {
+	if cmd == nil || cmd.Process == nil {
+		return
+	}
+	cmd.Process.Kill()
+	cmd.Wait()
+}