snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/fuse_p2p/framework_test.go

@@ -0,0 +1,436 @@
+//go:build linux || darwin
+
+package fuse_p2p
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// p2pTestCluster manages a minimal SeaweedFS cluster exercising the peer
+// chunk-sharing (p2p) read path: 1 master, 1 volume, 1 filer, and N FUSE
+// mounts that all have -peer.enable set. Three mounts is the sweet spot
+// for integration testing — with 2 mounts the HRW owner of a chunk is
+// ~50% the reader itself (which short-circuits the peer path); with 3+
+// mounts it's ≤ 1/3, so a multi-chunk file almost certainly exercises
+// the remote-owner fan-out.
+type p2pTestCluster struct {
+	t          testing.TB
+	baseDir    string
+	weedBinary string
+
+	masterPort     int
+	masterGrpcPort int
+	volumePort     int
+	volumeGrpcPort int
+	filerPort      int
+	filerGrpcPort  int
+	mountPeerPorts []int
+	mountPoints    []string
+
+	masterCmd *exec.Cmd
+	volumeCmd *exec.Cmd
+	filerCmd  *exec.Cmd
+	mountCmds []*exec.Cmd
+	logFiles  []*os.File
+
+	cleanupOnce sync.Once
+}
+
+// startP2PTestCluster brings up a cluster with numMounts FUSE mounts,
+// every one advertising on its own -peer.listen port. All mounts register
+// with the same filer. Verbose logging (-v=4) is enabled on each mount so
+// peer-read success/failure messages land in the log file and the test
+// can grep them to verify the p2p path fired.
+func startP2PTestCluster(t testing.TB, numMounts int) *p2pTestCluster {
+	require.GreaterOrEqual(t, numMounts, 2, "need at least 2 mounts to exercise p2p")
+	binary := findWeedBinary()
+	if binary == "" {
+		t.Skip("weed binary not found; set WEED_BINARY or ensure it is on PATH")
+	}
+	baseDir, err := os.MkdirTemp("", "seaweedfs_fuse_p2p_test_")
+	require.NoError(t, err)
+
+	c := &p2pTestCluster{
+		t:              t,
+		baseDir:        baseDir,
+		weedBinary:     binary,
+		mountPeerPorts: make([]int, numMounts),
+		mountPoints:    make([]string, numMounts),
+		mountCmds:      make([]*exec.Cmd, numMounts),
+	}
+	t.Cleanup(c.Stop)
+
+	// master(2) + volume(2) + filer(2) + one peer port per mount.
+	// testutil.AllocatePorts holds all listeners open until every port
+	// is reserved, avoiding the brief close→bind race that would
+	// happen with a per-listener close loop.
+	ports, err := testutil.AllocatePorts(6 + numMounts)
+	require.NoError(t, err)
+	c.masterPort = ports[0]
+	c.masterGrpcPort = ports[1]
+	c.volumePort = ports[2]
+	c.volumeGrpcPort = ports[3]
+	c.filerPort = ports[4]
+	c.filerGrpcPort = ports[5]
+	for i := 0; i < numMounts; i++ {
+		c.mountPeerPorts[i] = ports[6+i]
+	}
+
+	configDir := filepath.Join(baseDir, "config")
+	require.NoError(t, os.MkdirAll(configDir, 0755))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(""), 0644))
+
+	require.NoError(t, c.startMaster(configDir))
+	require.NoError(t, c.waitForTCP(c.masterCmd, "master",
+		fmt.Sprintf("127.0.0.1:%d", c.masterPort), 30*time.Second))
+
+	require.NoError(t, c.startVolume(configDir))
+	require.NoError(t, c.waitForTCP(c.volumeCmd, "volume",
+		fmt.Sprintf("127.0.0.1:%d", c.volumePort), 30*time.Second))
+	require.NoError(t, c.waitForVolumeRegistered(30*time.Second),
+		"volume server registration\n%s", c.tailLog("master"))
+
+	require.NoError(t, c.startFiler(configDir))
+	require.NoError(t, c.waitForTCP(c.filerCmd, "filer",
+		fmt.Sprintf("127.0.0.1:%d", c.filerGrpcPort), 30*time.Second))
+
+	for i := 0; i < numMounts; i++ {
+		mp := filepath.Join(baseDir, fmt.Sprintf("mount%d", i))
+		require.NoError(t, os.MkdirAll(mp, 0755))
+		c.mountPoints[i] = mp
+		require.NoError(t, c.startMount(i, configDir))
+		require.NoError(t, c.waitForMount(mp, 30*time.Second),
+			"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
+	}
+	return c
+}
+
+func (c *p2pTestCluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		for i := len(c.mountCmds) - 1; i >= 0; i-- {
+			stopCmd(c.mountCmds[i])
+			// Backup unmount in case the FUSE teardown didn't clean up.
+			exec.Command("fusermount3", "-u", c.mountPoints[i]).Run()
+			exec.Command("fusermount", "-u", c.mountPoints[i]).Run()
+		}
+		stopCmd(c.filerCmd)
+		stopCmd(c.volumeCmd)
+		stopCmd(c.masterCmd)
+
+		for _, f := range c.logFiles {
+			f.Close()
+		}
+		c.copyLogsForCI()
+		if !c.t.Failed() {
+			os.RemoveAll(c.baseDir)
+		}
+		time.Sleep(2 * time.Second) // let ports drain
+	})
+}
+
+// MountDir returns the filesystem path of the i-th mount.
+func (c *p2pTestCluster) MountDir(i int) string { return c.mountPoints[i] }
+
+// masterAddress / filerAddress return SeaweedFS-style addresses encoding
+// both ports as "host:httpPort.grpcPort". Without this, downstream
+// components fall back to the grpcPort = httpPort + 10000 default,
+// which doesn't match the random port we allocate.
+func (c *p2pTestCluster) masterAddress() string {
+	return string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
+}
+
+func (c *p2pTestCluster) filerAddress() string {
+	return string(pb.NewServerAddress("127.0.0.1", c.filerPort, c.filerGrpcPort))
+}
+
+// MountLog returns the contents of mount i's log file.
+func (c *p2pTestCluster) MountLog(i int) string {
+	return c.tailLogFull(fmt.Sprintf("mount%d", i))
+}
+
+func (c *p2pTestCluster) startMaster(configDir string) error {
+	c.masterCmd = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"master",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.masterPort),
+		"-port.grpc="+strconv.Itoa(c.masterGrpcPort),
+		"-mdir="+filepath.Join(c.baseDir, "master"),
+	)
+	return c.startCmd(c.masterCmd, "master")
+}
+
+func (c *p2pTestCluster) startVolume(configDir string) error {
+	volDir := filepath.Join(c.baseDir, "volume")
+	if err := os.MkdirAll(volDir, 0755); err != nil {
+		return fmt.Errorf("create volume dir: %w", err)
+	}
+	c.volumeCmd = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"volume",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.volumePort),
+		"-port.grpc="+strconv.Itoa(c.volumeGrpcPort),
+		"-master="+c.masterAddress(),
+		"-dir="+volDir,
+		"-max=10",
+	)
+	return c.startCmd(c.volumeCmd, "volume")
+}
+
+func (c *p2pTestCluster) startFiler(configDir string) error {
+	filerDir := filepath.Join(c.baseDir, "filer")
+	if err := os.MkdirAll(filerDir, 0755); err != nil {
+		return fmt.Errorf("create filer dir: %w", err)
+	}
+	c.filerCmd = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"filer",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port="+strconv.Itoa(c.filerPort),
+		"-port.grpc="+strconv.Itoa(c.filerGrpcPort),
+		"-master="+c.masterAddress(),
+		"-defaultStoreDir="+filerDir,
+	)
+	return c.startCmd(c.filerCmd, "filer")
+}
+
+func (c *p2pTestCluster) startMount(idx int, configDir string) error {
+	cacheDir := filepath.Join(c.baseDir, fmt.Sprintf("cache%d", idx))
+	if err := os.MkdirAll(cacheDir, 0755); err != nil {
+		return fmt.Errorf("create cache dir: %w", err)
+	}
+	c.mountCmds[idx] = exec.Command(c.weedBinary,
+		"-logdir="+filepath.Join(c.baseDir, "logs"),
+		"-v=4",
+		"mount",
+		"-filer="+c.filerAddress(),
+		"-dir="+c.mountPoints[idx],
+		"-filer.path=/",
+		"-dirAutoCreate",
+		"-allowOthers=false",
+		"-cacheDir="+cacheDir,
+		"-peer.enable=true",
+		fmt.Sprintf("-peer.listen=127.0.0.1:%d", c.mountPeerPorts[idx]),
+		fmt.Sprintf("-peer.advertise=127.0.0.1:%d", c.mountPeerPorts[idx]),
+		"-peer.dataCenter=dc1",
+		fmt.Sprintf("-peer.rack=rack%d", idx),
+	)
+	return c.startCmd(c.mountCmds[idx], fmt.Sprintf("mount%d", idx))
+}
+
+func (c *p2pTestCluster) startCmd(cmd *exec.Cmd, name string) error {
+	logPath := filepath.Join(c.baseDir, "logs")
+	if err := os.MkdirAll(logPath, 0755); err != nil {
+		return fmt.Errorf("create log dir: %w", err)
+	}
+	logFile, err := os.Create(filepath.Join(logPath, name+".log"))
+	if err != nil {
+		return err
+	}
+	c.logFiles = append(c.logFiles, logFile)
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	return cmd.Start()
+}
+
+func (c *p2pTestCluster) tailLog(name string) string {
+	data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
+	if err != nil {
+		return fmt.Sprintf("(log %s not available: %v)", name, err)
+	}
+	const maxTail = 8192
+	if len(data) > maxTail {
+		data = data[len(data)-maxTail:]
+	}
+	return string(data)
+}
+
+func (c *p2pTestCluster) tailLogFull(name string) string {
+	data, err := os.ReadFile(filepath.Join(c.baseDir, "logs", name+".log"))
+	if err != nil {
+		return ""
+	}
+	return string(data)
+}
+
+func (c *p2pTestCluster) copyLogsForCI() {
+	// One subdirectory per test: a flat layout lets every teardown overwrite
+	// the previous test's logs, so the CI artifact only ever shows the last
+	// cluster, never the failing one.
+	ciLogDir := filepath.Join("/tmp/seaweedfs-fuse-p2p-logs", strings.ReplaceAll(c.t.Name(), "/", "_"))
+	os.MkdirAll(ciLogDir, 0755)
+	logsDir := filepath.Join(c.baseDir, "logs")
+	entries, err := os.ReadDir(logsDir)
+	if err != nil {
+		return
+	}
+	for _, e := range entries {
+		data, err := os.ReadFile(filepath.Join(logsDir, e.Name()))
+		if err != nil {
+			continue
+		}
+		os.WriteFile(filepath.Join(ciLogDir, e.Name()), data, 0644)
+	}
+}
+
+// waitForVolumeRegistered waits until the volume server shows up in the master
+// topology with its slots reported. An open volume port only means the process
+// is listening: until the master has elected itself and accepted a heartbeat,
+// an assign fails and the first write on a fresh mount surfaces it as ENOSPC.
+func (c *p2pTestCluster) waitForVolumeRegistered(timeout time.Duration) error {
+	addr := fmt.Sprintf("127.0.0.1:%d", c.masterGrpcPort)
+	conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	client := master_pb.NewSeaweedClient(conn)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		resp, err := client.VolumeList(ctx, &master_pb.VolumeListRequest{})
+		cancel()
+		if err == nil && resp.TopologyInfo != nil {
+			var slots int64
+			for _, dc := range resp.TopologyInfo.DataCenterInfos {
+				for _, rack := range dc.RackInfos {
+					for _, dn := range rack.DataNodeInfos {
+						for _, disk := range dn.DiskInfos {
+							slots += disk.MaxVolumeCount
+						}
+					}
+				}
+			}
+			if slots > 0 {
+				return nil
+			}
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("volume server not registered with the master within %v", timeout)
+}
+
+// waitForTCP polls addr until it accepts a connection, OR the supplied
+// subprocess exits — whichever comes first. Short-circuiting on child
+// exit turns a 30 s-spin-on-dead-process into an immediate failure with
+// the tail of its log. cmd may be nil for callers that don't track a
+// process; in that case we fall back to pure polling.
+//
+// Liveness is checked with signal 0 (POSIX "is this process alive").
+// We deliberately do NOT call cmd.Wait() in a goroutine here because
+// stopCmd() later calls Wait() at shutdown, and only one Wait per
+// process is allowed.
+func (c *p2pTestCluster) waitForTCP(cmd *exec.Cmd, name, addr string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", addr, time.Second)
+		if err == nil {
+			conn.Close()
+			return nil
+		}
+		if cmd != nil && cmd.Process != nil {
+			// signal 0 doesn't actually send anything; it just asks
+			// the kernel whether the process exists. An error here
+			// (typically "process already finished" or ESRCH) means
+			// the child died before coming up.
+			if sigErr := cmd.Process.Signal(syscall.Signal(0)); sigErr != nil {
+				return fmt.Errorf("%s exited before listening on %s: %v\n%s",
+					name, addr, sigErr, c.tailLog(name))
+			}
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("service at %s not ready within timeout\n%s", addr, c.tailLog(name))
+}
+
+// waitForMount waits for a FUSE filesystem to actually be mounted by
+// watching the device id flip (FUSE mounts have a different Dev than
+// their parent dir).
+func (c *p2pTestCluster) waitForMount(mountPoint string, timeout time.Duration) error {
+	parentDir := filepath.Dir(mountPoint)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		parentStat, err := os.Stat(parentDir)
+		if err != nil {
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+		mountStat, err := os.Stat(mountPoint)
+		if err != nil {
+			time.Sleep(200 * time.Millisecond)
+			continue
+		}
+		parentSys := parentStat.Sys().(*syscall.Stat_t)
+		mountSys := mountStat.Sys().(*syscall.Stat_t)
+		if parentSys.Dev != mountSys.Dev {
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("mount point %s not ready within timeout (FUSE not detected)", mountPoint)
+}
+
+// --- utilities (binary discovery, process shutdown) ---
+//
+// Port allocation is delegated to testutil.MustAllocatePorts, which
+// holds all listeners open until every port is reserved before
+// returning them in a batch — safer than the per-listener
+// close-then-reserve pattern fuse_dlm originally used.
+
+func findWeedBinary() string {
+	if env := os.Getenv("WEED_BINARY"); env != "" {
+		if _, err := os.Stat(env); err == nil {
+			return env
+		}
+	}
+	if p, err := exec.LookPath("weed"); err == nil {
+		return p
+	}
+	return ""
+}
+
+// stopCmd kills a child process with SIGTERM and waits up to 5 s, then
+// escalates to SIGKILL. Tolerates nil and already-exited processes.
+func stopCmd(cmd *exec.Cmd) {
+	if cmd == nil || cmd.Process == nil {
+		return
+	}
+	cmd.Process.Signal(syscall.SIGTERM)
+	done := make(chan struct{})
+	go func() {
+		cmd.Wait()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		cmd.Process.Signal(syscall.SIGKILL)
+		<-done
+	}
+}

test/fuse_p2p/peer_chunk_sharing_test.go

@@ -0,0 +1,147 @@
+package fuse_p2p
+
+import (
+	"bytes"
+	"crypto/md5"
+	"fmt"
+	"math/rand/v2"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// peerReadSuccessMarker is the log line tryPeerRead emits when a peer
+// fetch succeeded. The test greps non-writer mount logs for it to
+// prove the p2p path fired. At glog verbosity 4 (framework sets -v=4).
+const peerReadSuccessMarker = "peer read successful"
+
+// seedConvergenceTimeout bounds how long the test waits for:
+//   - the filer registry to list all mounts,
+//   - every mount's seed view (via MountList poll) to include all peers,
+//   - the first announcer flush cycle to publish chunk holders.
+//
+// With defaults the mount polls MountList every 30 s and flushes
+// ChunkAnnounce every 15 s. Allowing 45 s absorbs one MountList refresh
+// plus one announce cycle plus some slop for CI variance.
+const seedConvergenceTimeout = 90 * time.Second
+
+// TestPeerChunkSharing_ReadersPullFromPeerCache is the headline p2p
+// integration test. It proves at least one non-writer mount can satisfy
+// a read from the writer's chunk cache instead of the volume tier.
+//
+//  1. Bring up 1 master/volume/filer + 3 mounts, all with -peer.enable.
+//  2. Mount 0 writes a ~8 MiB file and reads it back so chunks land in
+//     its local cache and the announcer publishes them.
+//  3. Wait for seed convergence + at least one announcer flush cycle.
+//  4. BOTH mount 1 and mount 2 read the file.
+//
+// Why both readers: with 3 mounts, HRW picks one owner for the chunk.
+// If that owner is mount 1, only mount 2's read will hit the peer
+// path (mount 1's tryPeerRead bails on owner==self). If that owner is
+// mount 2, only mount 1's read will. If that owner is mount 0 (the
+// writer), both can. So by reading from both, we deterministically
+// guarantee at least one non-writer mount exercises the peer path.
+//
+// Once the peer fetch populates its local cache, subsequent reads
+// short-circuit on IsInCache — so we only get one real shot per mount
+// per chunk. That's fine: one success is all the test needs.
+func TestPeerChunkSharing_ReadersPullFromPeerCache(t *testing.T) {
+	c := startP2PTestCluster(t, 3)
+
+	// ~8 MiB, pseudo-random so compression doesn't collapse it to one block.
+	payload := make([]byte, 8*1024*1024)
+	rng := rand.New(rand.NewPCG(1, 2))
+	for i := range payload {
+		payload[i] = byte(rng.Uint32())
+	}
+
+	const relPath = "p2p-test.bin"
+	writer := c.MountDir(0)
+	require.NoError(t, os.WriteFile(filepath.Join(writer, relPath), payload, 0644))
+
+	// Warm mount 0's chunk cache by reading back through its own FUSE.
+	// Without this the chunks are on the volume server but not yet
+	// in anyone's peer-servable cache.
+	readBack, err := os.ReadFile(filepath.Join(writer, relPath))
+	require.NoError(t, err)
+	require.True(t, bytes.Equal(readBack, payload), "write-then-read on writer mount should match")
+
+	waitForSeedConvergence(t, c, seedConvergenceTimeout)
+
+	// Give the announcer several flush windows to push chunk-holder
+	// entries to the HRW owners. First flush may see the writer's seed
+	// view incomplete (only self) and defer all fids; subsequent
+	// flushes re-check against a refreshed seed view. announce interval
+	// is 15 s, so 45 s covers three attempts.
+	time.Sleep(45 * time.Second)
+
+	// Read from both non-writer mounts. For any HRW outcome on any
+	// chunk, at least one of these reads will NOT have the reader as
+	// the HRW owner, so its tryPeerRead will proceed to ChunkLookup +
+	// FetchChunk.
+	for _, idx := range []int{1, 2} {
+		got, err := os.ReadFile(filepath.Join(c.MountDir(idx), relPath))
+		require.NoError(t, err, "read from mount %d must succeed\n--- mount%d ---\n%s",
+			idx, idx, tailLines(c.MountLog(idx), 80))
+		require.Equal(t, md5.Sum(payload), md5.Sum(got),
+			"mount %d returned mismatched bytes (len got=%d want=%d)", idx, len(got), len(payload))
+	}
+
+	// Content matches alone doesn't prove p2p — the volume fallback
+	// would also satisfy the reads. Require at least one non-writer
+	// mount's log to contain the peer-read success marker.
+	var sawPeerRead bool
+	for _, idx := range []int{1, 2} {
+		if strings.Contains(c.MountLog(idx), peerReadSuccessMarker) {
+			sawPeerRead = true
+			break
+		}
+	}
+	if !sawPeerRead {
+		t.Fatalf("no non-writer mount logged %q — peer read path never fired.\n"+
+			"--- mount0 (writer) tail ---\n%s\n--- mount1 tail ---\n%s\n--- mount2 tail ---\n%s",
+			peerReadSuccessMarker,
+			tailLines(c.MountLog(0), 60), tailLines(c.MountLog(1), 60), tailLines(c.MountLog(2), 60))
+	}
+}
+
+// waitForSeedConvergence polls each mount's log looking for any sign
+// that MountList returned a peer list containing the other mounts.
+func waitForSeedConvergence(t *testing.T, c *p2pTestCluster, timeout time.Duration) {
+	t.Helper()
+	// The first MountRegister happens synchronously during mount startup,
+	// and the first MountList is pulled right after. 30 s is the refresh
+	// interval; waiting one full cycle here guarantees every mount has
+	// at minimum observed the others in its seed view.
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		ready := true
+		for i := range c.mountCmds {
+			if !strings.Contains(c.MountLog(i), "peer-grpc listening on") {
+				ready = false
+				break
+			}
+		}
+		if ready {
+			time.Sleep(30 * time.Second)
+			return
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	t.Fatalf("peer-grpc servers did not come up within %s", timeout)
+}
+
+// tailLines returns the last n newline-delimited lines of s, or the
+// whole thing if shorter. Keeps test failures readable.
+func tailLines(s string, n int) string {
+	lines := strings.Split(s, "\n")
+	if len(lines) <= n {
+		return s
+	}
+	return fmt.Sprintf("... (%d earlier lines omitted) ...\n%s",
+		len(lines)-n, strings.Join(lines[len(lines)-n:], "\n"))
+}