snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/volume_server/framework/cluster.go

@@ -0,0 +1,503 @@
+package framework
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// goBuildTags names the build tags a server the harness compiles must carry to
+// hold the same offsets as the test binary asking for it. A 4-byte server and a
+// 5-byte one disagree about every .idx row and reject each other's .vif, and
+// the mixed Go/Rust suites run both at once.
+func goBuildTags() []string {
+	if types.OffsetSize == 5 {
+		return []string{"-tags", "5BytesOffset"}
+	}
+	return nil
+}
+
+const (
+	defaultWaitTimeout    = 30 * time.Second
+	defaultWaitTick       = 200 * time.Millisecond
+	testVolumeSizeLimitMB = 32
+)
+
+var (
+	weedBinaryOnce sync.Once
+	weedBinaryPath string
+	weedBinaryErr  error
+)
+
+// Cluster is a lightweight SeaweedFS master + one volume server test harness.
+type Cluster struct {
+	testingTB testing.TB
+	profile   matrix.Profile
+
+	weedBinary string
+	baseDir    string
+	configDir  string
+	logsDir    string
+	keepLogs   bool
+
+	masterPort     int
+	masterGrpcPort int
+	volumePort     int
+	volumeGrpcPort int
+	volumePubPort  int
+
+	masterCmd *exec.Cmd
+	volumeCmd *exec.Cmd
+
+	volumeDataDirs []string
+
+	cleanupOnce sync.Once
+}
+
+// StartSingleVolumeCluster boots one master and one volume server.
+func StartSingleVolumeCluster(t testing.TB, profile matrix.Profile) *Cluster {
+	return StartSingleVolumeClusterWithDataDirs(t, profile, 1)
+}
+
+// StartSingleVolumeClusterWithDataDirs boots one master and one volume server
+// with dataDirCount separate data directories (passed to -dir as a comma list).
+// Each directory becomes its own DiskLocation on the volume server, letting
+// tests exercise multi-disk EC placement paths.
+func StartSingleVolumeClusterWithDataDirs(t testing.TB, profile matrix.Profile, dataDirCount int) *Cluster {
+	t.Helper()
+	if dataDirCount < 1 {
+		t.Fatalf("dataDirCount must be >= 1, got %d", dataDirCount)
+	}
+
+	weedBinary, err := FindOrBuildWeedBinary()
+	if err != nil {
+		t.Fatalf("resolve weed binary: %v", err)
+	}
+
+	baseDir, keepLogs, err := newWorkDir()
+	if err != nil {
+		t.Fatalf("create temp test directory: %v", err)
+	}
+
+	configDir := filepath.Join(baseDir, "config")
+	logsDir := filepath.Join(baseDir, "logs")
+	masterDataDir := filepath.Join(baseDir, "master")
+	volumeDataDirs := make([]string, dataDirCount)
+	// Single-dir layout stays at baseDir/volume so existing fixtures
+	// (CorruptDatFile etc.) that hardcode that path keep working. Only
+	// multi-dir clusters get the "volumeN" layout.
+	for i := 0; i < dataDirCount; i++ {
+		if dataDirCount == 1 {
+			volumeDataDirs[i] = filepath.Join(baseDir, "volume")
+		} else {
+			volumeDataDirs[i] = filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
+		}
+	}
+	setupDirs := append([]string{configDir, logsDir, masterDataDir}, volumeDataDirs...)
+	for _, dir := range setupDirs {
+		if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
+			t.Fatalf("create %s: %v", dir, mkErr)
+		}
+	}
+
+	if err = writeSecurityConfig(configDir, profile); err != nil {
+		t.Fatalf("write security config: %v", err)
+	}
+
+	miniPorts, ports, err := testutil.AllocatePortSet(1, 3)
+	if err != nil {
+		t.Fatalf("allocate ports: %v", err)
+	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset
+
+	c := &Cluster{
+		testingTB:      t,
+		profile:        profile,
+		weedBinary:     weedBinary,
+		baseDir:        baseDir,
+		configDir:      configDir,
+		logsDir:        logsDir,
+		keepLogs:       keepLogs,
+		masterPort:     masterPort,
+		masterGrpcPort: masterGrpcPort,
+		volumePort:     ports[0],
+		volumeGrpcPort: ports[1],
+		volumePubPort:  ports[0],
+	}
+	if profile.SplitPublicPort {
+		c.volumePubPort = ports[2]
+	}
+
+	if err = c.startMaster(masterDataDir); err != nil {
+		c.Stop()
+		t.Fatalf("start master: %v", err)
+	}
+	if err = c.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
+		masterLog := c.tailLog("master.log")
+		c.Stop()
+		t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
+	}
+
+	if err = c.startVolume(volumeDataDirs); err != nil {
+		masterLog := c.tailLog("master.log")
+		c.Stop()
+		t.Fatalf("start volume: %v\nmaster log tail:\n%s", err, masterLog)
+	}
+	c.volumeDataDirs = volumeDataDirs
+	if err = c.waitForHTTP(c.VolumeAdminURL() + "/status"); err != nil {
+		volumeLog := c.tailLog("volume.log")
+		c.Stop()
+		t.Fatalf("wait for volume readiness: %v\nvolume log tail:\n%s", err, volumeLog)
+	}
+	if err = c.waitForTCP(c.VolumeGRPCAddress()); err != nil {
+		volumeLog := c.tailLog("volume.log")
+		c.Stop()
+		t.Fatalf("wait for volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog)
+	}
+
+	t.Cleanup(func() {
+		c.Stop()
+	})
+
+	return c
+}
+
+// Stop terminates all processes and cleans temporary files.
+func (c *Cluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		stopProcess(c.volumeCmd)
+		stopProcess(c.masterCmd)
+		if !c.keepLogs && !c.testingTB.Failed() {
+			_ = os.RemoveAll(c.baseDir)
+		} else if c.baseDir != "" {
+			c.testingTB.Logf("volume server integration logs kept at %s", c.baseDir)
+		}
+	})
+}
+
+// RestartVolumeServer kills the volume server and starts a new one against
+// the same data dirs and ports. The master keeps running. The previous run's
+// volume.log is moved to volume.log.previous so the first run's logs survive
+// a second-run startup failure.
+func (c *Cluster) RestartVolumeServer() {
+	c.testingTB.Helper()
+	stopProcess(c.volumeCmd)
+	c.volumeCmd = nil
+	// Rotate the log; absent on the first call after a clean start, which
+	// is fine — startVolume will create it. Any real filesystem failure
+	// surfaces immediately on the next os.Create in startVolume.
+	oldLog := filepath.Join(c.logsDir, "volume.log")
+	_ = os.Rename(oldLog, filepath.Join(c.logsDir, "volume.log.previous"))
+	if err := c.startVolume(c.volumeDataDirs); err != nil {
+		c.testingTB.Fatalf("restart volume server: %v", err)
+	}
+	if err := c.waitForHTTP(c.VolumeAdminURL() + "/status"); err != nil {
+		c.testingTB.Fatalf("wait for volume admin readiness after restart: %v\nvolume log tail:\n%s", err, c.tailLog("volume.log"))
+	}
+	if err := c.waitForTCP(c.VolumeGRPCAddress()); err != nil {
+		c.testingTB.Fatalf("wait for volume grpc readiness after restart: %v\nvolume log tail:\n%s", err, c.tailLog("volume.log"))
+	}
+}
+
+// StopVolumeServer kills the volume server but leaves the master and data
+// dirs alone. Pair with RestartVolumeServer or Stop.
+func (c *Cluster) StopVolumeServer() {
+	c.testingTB.Helper()
+	stopProcess(c.volumeCmd)
+	c.volumeCmd = nil
+}
+
+func (c *Cluster) startMaster(dataDir string) error {
+	logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"master",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(c.masterPort),
+		"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
+		"-mdir=" + dataDir,
+		"-peers=none",
+		"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
+		"-defaultReplication=000",
+	}
+
+	c.masterCmd = exec.Command(c.weedBinary, args...)
+	c.masterCmd.Dir = c.baseDir
+	c.masterCmd.Stdout = logFile
+	c.masterCmd.Stderr = logFile
+	return c.masterCmd.Start()
+}
+
+func (c *Cluster) startVolume(dataDirs []string) error {
+	logFile, err := os.Create(filepath.Join(c.logsDir, "volume.log"))
+	if err != nil {
+		return err
+	}
+
+	maxPerDir := make([]string, len(dataDirs))
+	for i := range dataDirs {
+		maxPerDir[i] = "16"
+	}
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"volume",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(c.volumePort),
+		"-port.grpc=" + strconv.Itoa(c.volumeGrpcPort),
+		"-port.public=" + strconv.Itoa(c.volumePubPort),
+		"-dir=" + strings.Join(dataDirs, ","),
+		"-max=" + strings.Join(maxPerDir, ","),
+		"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
+		"-readMode=" + c.profile.ReadMode,
+		"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
+		"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
+		// Integration tests deliberately exercise loopback S3 endpoints
+		// (the test rig boots weed-mini next to the volume server); allow
+		// the SSRF guard to be bypassed for them.
+		"-volume.allowUntrustedRemoteEndpoints",
+	}
+	if c.profile.InflightUploadTimeout > 0 {
+		args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
+	}
+	if c.profile.InflightDownloadTimeout > 0 {
+		args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
+	}
+
+	c.volumeCmd = exec.Command(c.weedBinary, args...)
+	c.volumeCmd.Dir = c.baseDir
+	c.volumeCmd.Stdout = logFile
+	c.volumeCmd.Stderr = logFile
+	return c.volumeCmd.Start()
+}
+
+func (c *Cluster) waitForHTTP(url string) error {
+	client := &http.Client{Timeout: 1 * time.Second}
+	deadline := time.Now().Add(defaultWaitTimeout)
+	for time.Now().Before(deadline) {
+		resp, err := client.Get(url)
+		if err == nil {
+			_, _ = io.Copy(io.Discard, resp.Body)
+			resp.Body.Close()
+			if resp.StatusCode < 500 {
+				return nil
+			}
+		}
+		time.Sleep(defaultWaitTick)
+	}
+	return fmt.Errorf("timed out waiting for %s", url)
+}
+
+func (c *Cluster) waitForTCP(addr string) error {
+	deadline := time.Now().Add(defaultWaitTimeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", addr, time.Second)
+		if err == nil {
+			_ = conn.Close()
+			return nil
+		}
+		time.Sleep(defaultWaitTick)
+	}
+	return fmt.Errorf("timed out waiting for tcp %s", addr)
+}
+
+func stopProcess(cmd *exec.Cmd) {
+	if cmd == nil || cmd.Process == nil {
+		return
+	}
+
+	_ = cmd.Process.Signal(os.Interrupt)
+	done := make(chan error, 1)
+	go func() {
+		done <- cmd.Wait()
+	}()
+
+	select {
+	case <-time.After(10 * time.Second):
+		_ = cmd.Process.Kill()
+		<-done
+	case <-done:
+	}
+}
+
+func newWorkDir() (dir string, keepLogs bool, err error) {
+	keepLogs = os.Getenv("VOLUME_SERVER_IT_KEEP_LOGS") == "1"
+	dir, err = os.MkdirTemp("", "seaweedfs_volume_server_it_")
+	return dir, keepLogs, err
+}
+
+func writeSecurityConfig(configDir string, profile matrix.Profile) error {
+	var b strings.Builder
+	if profile.EnableJWT {
+		if profile.JWTSigningKey == "" || profile.JWTReadKey == "" {
+			return errors.New("jwt profile requires both write and read keys")
+		}
+		b.WriteString("[jwt.signing]\n")
+		b.WriteString("key = \"")
+		b.WriteString(profile.JWTSigningKey)
+		b.WriteString("\"\n")
+		b.WriteString("expires_after_seconds = 60\n\n")
+
+		b.WriteString("[jwt.signing.read]\n")
+		b.WriteString("key = \"")
+		b.WriteString(profile.JWTReadKey)
+		b.WriteString("\"\n")
+		b.WriteString("expires_after_seconds = 60\n")
+	}
+	if profile.EnableUIAccess {
+		if b.Len() > 0 {
+			b.WriteString("\n")
+		}
+		b.WriteString("[access]\n")
+		b.WriteString("ui = true\n")
+	}
+	if b.Len() == 0 {
+		b.WriteString("# optional security config generated for integration tests\n")
+	}
+	return os.WriteFile(filepath.Join(configDir, "security.toml"), []byte(b.String()), 0o644)
+}
+
+// FindOrBuildWeedBinary returns an executable weed binary, building one when needed.
+func FindOrBuildWeedBinary() (string, error) {
+	if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" {
+		if isExecutableFile(fromEnv) {
+			return fromEnv, nil
+		}
+		return "", fmt.Errorf("WEED_BINARY is set but not executable: %s", fromEnv)
+	}
+
+	weedBinaryOnce.Do(func() {
+		repoRoot := ""
+		if _, file, _, ok := runtime.Caller(0); ok {
+			repoRoot = filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
+		}
+		if repoRoot == "" {
+			weedBinaryErr = errors.New("unable to detect repository root")
+			return
+		}
+
+		binDir := filepath.Join(os.TempDir(), "seaweedfs_volume_server_it_bin")
+		if err := os.MkdirAll(binDir, 0o755); err != nil {
+			weedBinaryErr = fmt.Errorf("create binary directory %s: %w", binDir, err)
+			return
+		}
+		binPath := filepath.Join(binDir, "weed")
+
+		args := append([]string{"build"}, goBuildTags()...)
+		args = append(args, "-o", binPath, ".")
+		cmd := exec.Command("go", args...)
+		cmd.Dir = filepath.Join(repoRoot, "weed")
+		var out bytes.Buffer
+		cmd.Stdout = &out
+		cmd.Stderr = &out
+		if err := cmd.Run(); err != nil {
+			weedBinaryErr = fmt.Errorf("build weed binary: %w\n%s", err, out.String())
+			return
+		}
+		if !isExecutableFile(binPath) {
+			weedBinaryErr = fmt.Errorf("built weed binary is not executable: %s", binPath)
+			return
+		}
+		weedBinaryPath = binPath
+	})
+
+	if weedBinaryErr != nil {
+		return "", weedBinaryErr
+	}
+	return weedBinaryPath, nil
+}
+
+func isExecutableFile(path string) bool {
+	info, err := os.Stat(path)
+	if err != nil || info.IsDir() {
+		return false
+	}
+	mode := info.Mode().Perm()
+	return mode&0o111 != 0
+}
+
+func (c *Cluster) tailLog(logName string) string {
+	f, err := os.Open(filepath.Join(c.logsDir, logName))
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	lines := make([]string, 0, 40)
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+		if len(lines) > 40 {
+			lines = lines[1:]
+		}
+	}
+	return strings.Join(lines, "\n")
+}
+
+func (c *Cluster) MasterAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
+}
+
+func (c *Cluster) VolumeAdminAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort))
+}
+
+func (c *Cluster) VolumePublicAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPort))
+}
+
+func (c *Cluster) VolumeGRPCAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort))
+}
+
+// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort
+func (c *Cluster) VolumeServerAddress() string {
+	return fmt.Sprintf("%s.%d", c.VolumeAdminAddress(), c.volumeGrpcPort)
+}
+
+func (c *Cluster) MasterURL() string {
+	return "http://" + c.MasterAddress()
+}
+
+func (c *Cluster) VolumeAdminURL() string {
+	return "http://" + c.VolumeAdminAddress()
+}
+
+func (c *Cluster) VolumePublicURL() string {
+	return "http://" + c.VolumePublicAddress()
+}
+
+func (c *Cluster) BaseDir() string {
+	return c.baseDir
+}
+
+// VolumeDataDirs returns the data directories the volume server was started with.
+// Index 0 corresponds to DiskLocation 0, index 1 to DiskLocation 1, and so on.
+// Tests can scan these directories to verify where files physically landed.
+func (c *Cluster) VolumeDataDirs() []string {
+	return append([]string(nil), c.volumeDataDirs...)
+}

test/volume_server/framework/cluster_dual.go

@@ -0,0 +1,17 @@
+package framework
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+// DualVolumeCluster is deprecated. Use MultiVolumeCluster instead.
+// For backward compatibility, it's a type alias for MultiVolumeCluster.
+type DualVolumeCluster = MultiVolumeCluster
+
+// StartDualVolumeCluster starts a cluster with 2 volume servers.
+// Deprecated: Use StartMultiVolumeClusterAuto(t, profile, 2) directly.
+func StartDualVolumeCluster(t testing.TB, profile matrix.Profile) MultiCluster {
+	return StartMultiVolumeClusterAuto(t, profile, 2)
+}

test/volume_server/framework/cluster_interface.go

@@ -0,0 +1,63 @@
+package framework
+
+import (
+	"os"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+// TestCluster is the common interface for single-volume cluster harnesses.
+// Both *Cluster (Go volume) and *RustCluster (Rust volume) satisfy it.
+type TestCluster interface {
+	MasterAddress() string
+	VolumeAdminAddress() string
+	VolumePublicAddress() string
+	VolumeGRPCAddress() string
+	VolumeServerAddress() string
+	MasterURL() string
+	VolumeAdminURL() string
+	VolumePublicURL() string
+	BaseDir() string
+	Stop()
+}
+
+func useRustVolumeServer() bool {
+	return os.Getenv("VOLUME_SERVER_IMPL") == "rust"
+}
+
+// StartVolumeCluster starts a single-volume cluster using either the Go or
+// Rust volume server, depending on the VOLUME_SERVER_IMPL environment variable.
+// Set VOLUME_SERVER_IMPL=rust to use the Rust volume server.
+func StartVolumeCluster(t testing.TB, profile matrix.Profile) TestCluster {
+	t.Helper()
+	if useRustVolumeServer() {
+		return StartRustVolumeCluster(t, profile)
+	}
+	return StartSingleVolumeCluster(t, profile)
+}
+
+// MultiCluster is the common interface for multi-volume cluster harnesses.
+// Both *MultiVolumeCluster (Go) and *RustMultiVolumeCluster (Rust) satisfy it.
+type MultiCluster interface {
+	MasterAddress() string
+	MasterURL() string
+	BaseDir() string
+	VolumeAdminAddress(index int) string
+	VolumeAdminURL(index int) string
+	VolumePublicAddress(index int) string
+	VolumePublicURL(index int) string
+	VolumeGRPCAddress(index int) string
+	Stop()
+}
+
+// StartMultiVolumeClusterAuto starts a multi-volume cluster using either Go or
+// Rust volume servers, depending on the VOLUME_SERVER_IMPL environment variable.
+// Set VOLUME_SERVER_IMPL=rust to use Rust volume servers.
+func StartMultiVolumeClusterAuto(t testing.TB, profile matrix.Profile, count int) MultiCluster {
+	t.Helper()
+	if useRustVolumeServer() {
+		return StartRustMultiVolumeCluster(t, profile, count)
+	}
+	return StartMultiVolumeCluster(t, profile, count)
+}

test/volume_server/framework/cluster_interface_test.go

@@ -0,0 +1,20 @@
+package framework
+
+import "testing"
+
+func TestUseRustVolumeServer(t *testing.T) {
+	t.Setenv("VOLUME_SERVER_IMPL", "rust")
+	if !useRustVolumeServer() {
+		t.Fatalf("expected rust selection when VOLUME_SERVER_IMPL=rust")
+	}
+
+	t.Setenv("VOLUME_SERVER_IMPL", "go")
+	if useRustVolumeServer() {
+		t.Fatalf("expected go selection when VOLUME_SERVER_IMPL=go")
+	}
+
+	t.Setenv("VOLUME_SERVER_IMPL", "")
+	if useRustVolumeServer() {
+		t.Fatalf("expected go selection when VOLUME_SERVER_IMPL is unset")
+	}
+}

test/volume_server/framework/cluster_mixed.go

@@ -0,0 +1,368 @@
+package framework
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+// MixedVolumeCluster wraps a Go master + a mix of Go and Rust volume servers
+// for integration testing. Go servers occupy indices [0, goCount) and Rust
+// servers occupy indices [goCount, goCount+rustCount).
+type MixedVolumeCluster struct {
+	testingTB testing.TB
+	profile   matrix.Profile
+
+	weedBinary       string // Go weed binary (master + Go volume servers)
+	rustVolumeBinary string // Rust volume binary
+
+	baseDir   string
+	configDir string
+	logsDir   string
+	keepLogs  bool
+
+	masterPort     int
+	masterGrpcPort int
+
+	volumePorts     []int
+	volumeGrpcPorts []int
+	volumePubPorts  []int
+	isRust          []bool // which servers are Rust
+
+	masterCmd  *exec.Cmd
+	volumeCmds []*exec.Cmd
+
+	cleanupOnce sync.Once
+}
+
+// StartMixedVolumeCluster starts a cluster with 1 Go master, goCount Go volume
+// servers, and rustCount Rust volume servers. Go servers come first in the index.
+func StartMixedVolumeCluster(t testing.TB, profile matrix.Profile, goCount, rustCount int) *MixedVolumeCluster {
+	t.Helper()
+
+	if goCount < 0 || rustCount < 0 {
+		t.Fatalf("goCount and rustCount must be non-negative, got go=%d rust=%d", goCount, rustCount)
+	}
+	total := goCount + rustCount
+	if total < 2 {
+		t.Fatalf("need at least 2 volume servers, got %d", total)
+	}
+
+	weedBinary, err := FindOrBuildWeedBinary()
+	if err != nil {
+		t.Fatalf("resolve weed binary: %v", err)
+	}
+
+	// Only build the Rust binary when Rust servers are requested.
+	var rustBinary string
+	if rustCount > 0 {
+		rustBinary, err = FindOrBuildRustBinary()
+		if err != nil {
+			t.Skipf("skipping mixed cluster test: rust binary unavailable: %v", err)
+		}
+	}
+
+	baseDir, keepLogs, err := newWorkDir()
+	if err != nil {
+		t.Fatalf("create temp test directory: %v", err)
+	}
+
+	configDir := filepath.Join(baseDir, "config")
+	logsDir := filepath.Join(baseDir, "logs")
+	masterDataDir := filepath.Join(baseDir, "master")
+
+	dirs := []string{configDir, logsDir, masterDataDir}
+	for i := 0; i < total; i++ {
+		dirs = append(dirs, filepath.Join(baseDir, fmt.Sprintf("volume%d", i)))
+	}
+	for _, dir := range dirs {
+		if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
+			t.Fatalf("create %s: %v", dir, mkErr)
+		}
+	}
+
+	if err = writeSecurityConfig(configDir, profile); err != nil {
+		t.Fatalf("write security config: %v", err)
+	}
+
+	// 2 ports per server (admin, grpc); add 1 more when public port is split out.
+	portsPerServer := 2
+	if profile.SplitPublicPort {
+		portsPerServer = 3
+	}
+	miniPorts, ports, err := testutil.AllocatePortSet(1, total*portsPerServer)
+	if err != nil {
+		t.Fatalf("allocate ports: %v", err)
+	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset
+
+	isRust := make([]bool, total)
+	for i := goCount; i < total; i++ {
+		isRust[i] = true
+	}
+
+	c := &MixedVolumeCluster{
+		testingTB:        t,
+		profile:          profile,
+		weedBinary:       weedBinary,
+		rustVolumeBinary: rustBinary,
+		baseDir:          baseDir,
+		configDir:        configDir,
+		logsDir:          logsDir,
+		keepLogs:         keepLogs,
+		masterPort:       masterPort,
+		masterGrpcPort:   masterGrpcPort,
+		volumePorts:      make([]int, total),
+		volumeGrpcPorts:  make([]int, total),
+		volumePubPorts:   make([]int, total),
+		isRust:           isRust,
+		volumeCmds:       make([]*exec.Cmd, total),
+	}
+
+	for i := 0; i < total; i++ {
+		baseIdx := i * portsPerServer
+		c.volumePorts[i] = ports[baseIdx]
+		c.volumeGrpcPorts[i] = ports[baseIdx+1]
+		if profile.SplitPublicPort {
+			c.volumePubPorts[i] = ports[baseIdx+2]
+		} else {
+			c.volumePubPorts[i] = c.volumePorts[i] // reuse admin port
+		}
+	}
+
+	// Start master
+	if err = c.startMaster(masterDataDir); err != nil {
+		c.Stop()
+		t.Fatalf("start master: %v", err)
+	}
+	helper := &Cluster{logsDir: logsDir}
+	if err = helper.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
+		masterLog := helper.tailLog("master.log")
+		c.Stop()
+		t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
+	}
+
+	// Start volume servers
+	for i := 0; i < total; i++ {
+		volumeDataDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
+		if isRust[i] {
+			err = c.startRustVolume(i, volumeDataDir)
+		} else {
+			err = c.startGoVolume(i, volumeDataDir)
+		}
+		if err != nil {
+			logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
+			c.Stop()
+			t.Fatalf("start volume server %d (rust=%v): %v\nlog tail:\n%s", i, isRust[i], err, logTail)
+		}
+
+		// Rust uses /healthz, Go uses /status
+		healthURL := c.VolumeAdminURL(i) + "/status"
+		if isRust[i] {
+			healthURL = c.VolumeAdminURL(i) + "/healthz"
+		}
+		if err = helper.waitForHTTP(healthURL); err != nil {
+			logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
+			c.Stop()
+			t.Fatalf("wait for volume server %d readiness: %v\nlog tail:\n%s", i, err, logTail)
+		}
+		if err = helper.waitForTCP(c.VolumeGRPCAddress(i)); err != nil {
+			logTail := helper.tailLog(fmt.Sprintf("volume%d.log", i))
+			c.Stop()
+			t.Fatalf("wait for volume server %d grpc readiness: %v\nlog tail:\n%s", i, err, logTail)
+		}
+	}
+
+	t.Cleanup(func() {
+		c.Stop()
+	})
+
+	return c
+}
+
+func (c *MixedVolumeCluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		for i := len(c.volumeCmds) - 1; i >= 0; i-- {
+			stopProcess(c.volumeCmds[i])
+		}
+		stopProcess(c.masterCmd)
+		if !c.keepLogs && !c.testingTB.Failed() {
+			_ = os.RemoveAll(c.baseDir)
+		} else if c.baseDir != "" {
+			c.testingTB.Logf("mixed volume server integration logs kept at %s", c.baseDir)
+		}
+	})
+}
+
+func (c *MixedVolumeCluster) startMaster(dataDir string) error {
+	logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"master",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(c.masterPort),
+		"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
+		"-mdir=" + dataDir,
+		"-peers=none",
+		"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
+		"-defaultReplication=000",
+	}
+
+	c.masterCmd = exec.Command(c.weedBinary, args...)
+	c.masterCmd.Dir = c.baseDir
+	c.masterCmd.Stdout = logFile
+	c.masterCmd.Stderr = logFile
+	if err = c.masterCmd.Start(); err != nil {
+		logFile.Close()
+		return err
+	}
+	logFile.Close() // child inherited the fd
+	return nil
+}
+
+func (c *MixedVolumeCluster) startGoVolume(index int, dataDir string) error {
+	logName := fmt.Sprintf("volume%d.log", index)
+	logFile, err := os.Create(filepath.Join(c.logsDir, logName))
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"volume",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(c.volumePorts[index]),
+		"-port.grpc=" + strconv.Itoa(c.volumeGrpcPorts[index]),
+		"-port.public=" + strconv.Itoa(c.volumePubPorts[index]),
+		"-dir=" + dataDir,
+		"-max=16",
+		"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
+		"-readMode=" + c.profile.ReadMode,
+		"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
+		"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
+		// Integration tests deliberately exercise loopback S3 endpoints; allow the SSRF guard to be bypassed for them.
+		"-volume.allowUntrustedRemoteEndpoints",
+	}
+	if c.profile.InflightUploadTimeout > 0 {
+		args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
+	}
+	if c.profile.InflightDownloadTimeout > 0 {
+		args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
+	}
+
+	cmd := exec.Command(c.weedBinary, args...)
+	cmd.Dir = c.baseDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	if err = cmd.Start(); err != nil {
+		logFile.Close()
+		return err
+	}
+	logFile.Close() // child inherited the fd
+	c.volumeCmds[index] = cmd
+	return nil
+}
+
+func (c *MixedVolumeCluster) startRustVolume(index int, dataDir string) error {
+	logName := fmt.Sprintf("volume%d.log", index)
+	logFile, err := os.Create(filepath.Join(c.logsDir, logName))
+	if err != nil {
+		return err
+	}
+
+	args := rustVolumeArgs(
+		c.profile,
+		c.configDir,
+		c.masterPort,
+		c.volumePorts[index],
+		c.volumeGrpcPorts[index],
+		c.volumePubPorts[index],
+		dataDir,
+	)
+
+	cmd := exec.Command(c.rustVolumeBinary, args...)
+	cmd.Dir = c.baseDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	if err = cmd.Start(); err != nil {
+		logFile.Close()
+		return err
+	}
+	logFile.Close() // child inherited the fd
+	c.volumeCmds[index] = cmd
+	return nil
+}
+
+// --- accessor methods (mirror MultiVolumeCluster) ---
+
+func (c *MixedVolumeCluster) MasterAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
+}
+
+func (c *MixedVolumeCluster) MasterURL() string {
+	return "http://" + c.MasterAddress()
+}
+
+func (c *MixedVolumeCluster) VolumeAdminAddress(index int) string {
+	if index < 0 || index >= len(c.volumePorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePorts[index]))
+}
+
+func (c *MixedVolumeCluster) VolumePublicAddress(index int) string {
+	if index < 0 || index >= len(c.volumePubPorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPorts[index]))
+}
+
+func (c *MixedVolumeCluster) VolumeGRPCAddress(index int) string {
+	if index < 0 || index >= len(c.volumeGrpcPorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPorts[index]))
+}
+
+func (c *MixedVolumeCluster) VolumeAdminURL(index int) string {
+	return "http://" + c.VolumeAdminAddress(index)
+}
+
+func (c *MixedVolumeCluster) VolumePublicURL(index int) string {
+	return "http://" + c.VolumePublicAddress(index)
+}
+
+func (c *MixedVolumeCluster) BaseDir() string {
+	return c.baseDir
+}
+
+// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort
+func (c *MixedVolumeCluster) VolumeServerAddress(index int) string {
+	if index < 0 || index >= len(c.volumePorts) {
+		return ""
+	}
+	return fmt.Sprintf("%s.%d", c.VolumeAdminAddress(index), c.volumeGrpcPorts[index])
+}
+
+func (c *MixedVolumeCluster) IsRust(index int) bool {
+	if index < 0 || index >= len(c.isRust) {
+		return false
+	}
+	return c.isRust[index]
+}

test/volume_server/framework/cluster_multi.go

@@ -0,0 +1,348 @@
+package framework
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+type MultiVolumeCluster struct {
+	testingTB testing.TB
+	profile   matrix.Profile
+
+	weedBinary        string
+	baseDir           string
+	configDir         string
+	logsDir           string
+	keepLogs          bool
+	volumeServerCount int
+	disksPerServer    int
+
+	masterPort     int
+	masterGrpcPort int
+
+	volumePorts     []int
+	volumeGrpcPorts []int
+	volumePubPorts  []int
+	volumeDiskDirs  [][]string // per server: the data directories (one DiskLocation each)
+
+	masterCmd  *exec.Cmd
+	volumeCmds []*exec.Cmd
+
+	cleanupOnce sync.Once
+}
+
+// StartMultiVolumeCluster starts a cluster with serverCount volume servers, one
+// data directory (DiskLocation) each.
+func StartMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount int) *MultiVolumeCluster {
+	return StartMultiVolumeClusterWithDisks(t, profile, serverCount, 1)
+}
+
+// StartMultiVolumeClusterWithDisks starts serverCount volume servers, each with
+// disksPerServer data directories passed to -dir as a comma list so every
+// directory becomes its own DiskLocation. Lets tests exercise per-disk EC
+// layouts (e.g. shards on one disk, a stub/index on a sibling disk).
+func StartMultiVolumeClusterWithDisks(t testing.TB, profile matrix.Profile, serverCount, disksPerServer int) *MultiVolumeCluster {
+	t.Helper()
+
+	if serverCount < 1 {
+		t.Fatalf("serverCount must be at least 1, got %d", serverCount)
+	}
+	if disksPerServer < 1 {
+		t.Fatalf("disksPerServer must be at least 1, got %d", disksPerServer)
+	}
+
+	weedBinary, err := FindOrBuildWeedBinary()
+	if err != nil {
+		t.Fatalf("resolve weed binary: %v", err)
+	}
+
+	baseDir, keepLogs, err := newWorkDir()
+	if err != nil {
+		t.Fatalf("create temp test directory: %v", err)
+	}
+
+	configDir := filepath.Join(baseDir, "config")
+	logsDir := filepath.Join(baseDir, "logs")
+	masterDataDir := filepath.Join(baseDir, "master")
+
+	// Create directories for master and all volume servers. With one disk the
+	// layout stays at baseDir/volumeN; multi-disk servers get baseDir/volumeN/diskD.
+	dirs := []string{configDir, logsDir, masterDataDir}
+	volumeDiskDirs := make([][]string, serverCount)
+	for i := 0; i < serverCount; i++ {
+		serverDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
+		volumeDiskDirs[i] = make([]string, disksPerServer)
+		for d := 0; d < disksPerServer; d++ {
+			if disksPerServer == 1 {
+				volumeDiskDirs[i][d] = serverDir
+			} else {
+				volumeDiskDirs[i][d] = filepath.Join(serverDir, fmt.Sprintf("disk%d", d))
+			}
+			dirs = append(dirs, volumeDiskDirs[i][d])
+		}
+	}
+	for _, dir := range dirs {
+		if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
+			t.Fatalf("create %s: %v", dir, mkErr)
+		}
+	}
+
+	if err = writeSecurityConfig(configDir, profile); err != nil {
+		t.Fatalf("write security config: %v", err)
+	}
+
+	// Allocate ports for all volume servers (3 ports per server: admin, grpc, public)
+	// If SplitPublicPort is true, we need an additional port per server
+	portsPerServer := 3
+	if profile.SplitPublicPort {
+		portsPerServer = 4
+	}
+	totalPorts := serverCount * portsPerServer
+	miniPorts, ports, err := testutil.AllocatePortSet(1, totalPorts)
+	if err != nil {
+		t.Fatalf("allocate ports: %v", err)
+	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset
+
+	c := &MultiVolumeCluster{
+		testingTB:         t,
+		profile:           profile,
+		weedBinary:        weedBinary,
+		baseDir:           baseDir,
+		configDir:         configDir,
+		logsDir:           logsDir,
+		keepLogs:          keepLogs,
+		volumeServerCount: serverCount,
+		disksPerServer:    disksPerServer,
+		masterPort:        masterPort,
+		masterGrpcPort:    masterGrpcPort,
+		volumePorts:       make([]int, serverCount),
+		volumeGrpcPorts:   make([]int, serverCount),
+		volumePubPorts:    make([]int, serverCount),
+		volumeDiskDirs:    volumeDiskDirs,
+		volumeCmds:        make([]*exec.Cmd, serverCount),
+	}
+
+	// Assign ports to each volume server
+	for i := 0; i < serverCount; i++ {
+		baseIdx := i * portsPerServer
+		c.volumePorts[i] = ports[baseIdx]
+		c.volumeGrpcPorts[i] = ports[baseIdx+1]
+
+		// Assign public port, using baseIdx+3 if SplitPublicPort, else baseIdx+2
+		pubPortIdx := baseIdx + 2
+		if profile.SplitPublicPort {
+			pubPortIdx = baseIdx + 3
+		}
+		c.volumePubPorts[i] = ports[pubPortIdx]
+	}
+
+	// Start master
+	if err = c.startMaster(masterDataDir); err != nil {
+		c.Stop()
+		t.Fatalf("start master: %v", err)
+	}
+	if err = c.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
+		masterLog := c.tailLog("master.log")
+		c.Stop()
+		t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
+	}
+
+	// Start all volume servers
+	for i := 0; i < serverCount; i++ {
+		if err = c.startVolume(i, c.volumeDiskDirs[i]); err != nil {
+			// Log current server's log for debugging startup failures
+			volumeLog := fmt.Sprintf("volume%d.log", i)
+			c.Stop()
+			t.Fatalf("start volume server %d: %v\nvolume log tail:\n%s", i, err, c.tailLog(volumeLog))
+		}
+		if err = c.waitForHTTP(c.VolumeAdminURL(i) + "/status"); err != nil {
+			volumeLog := fmt.Sprintf("volume%d.log", i)
+			c.Stop()
+			t.Fatalf("wait for volume server %d readiness: %v\nvolume log tail:\n%s", i, err, c.tailLog(volumeLog))
+		}
+		if err = c.waitForTCP(c.VolumeGRPCAddress(i)); err != nil {
+			volumeLog := fmt.Sprintf("volume%d.log", i)
+			c.Stop()
+			t.Fatalf("wait for volume server %d grpc readiness: %v\nvolume log tail:\n%s", i, err, c.tailLog(volumeLog))
+		}
+	}
+
+	t.Cleanup(func() {
+		c.Stop()
+	})
+
+	return c
+}
+
+// StartTripleVolumeCluster is a convenience wrapper that starts a cluster with 3 volume servers
+func StartTripleVolumeCluster(t testing.TB, profile matrix.Profile) *MultiVolumeCluster {
+	return StartMultiVolumeCluster(t, profile, 3)
+}
+
+func (c *MultiVolumeCluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		// Stop volume servers in reverse order
+		for i := len(c.volumeCmds) - 1; i >= 0; i-- {
+			stopProcess(c.volumeCmds[i])
+		}
+		stopProcess(c.masterCmd)
+		if !c.keepLogs && !c.testingTB.Failed() {
+			_ = os.RemoveAll(c.baseDir)
+		} else if c.baseDir != "" {
+			c.testingTB.Logf("volume server integration logs kept at %s", c.baseDir)
+		}
+	})
+}
+
+func (c *MultiVolumeCluster) startMaster(dataDir string) error {
+	logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"master",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(c.masterPort),
+		"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
+		"-mdir=" + dataDir,
+		"-peers=none",
+		"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
+		"-defaultReplication=000",
+	}
+
+	c.masterCmd = exec.Command(c.weedBinary, args...)
+	c.masterCmd.Dir = c.baseDir
+	c.masterCmd.Stdout = logFile
+	c.masterCmd.Stderr = logFile
+	return c.masterCmd.Start()
+}
+
+func (c *MultiVolumeCluster) startVolume(index int, dataDirs []string) error {
+	logName := fmt.Sprintf("volume%d.log", index)
+	logFile, err := os.Create(filepath.Join(c.logsDir, logName))
+	if err != nil {
+		return err
+	}
+
+	maxPerDir := make([]string, len(dataDirs))
+	for i := range dataDirs {
+		maxPerDir[i] = "16"
+	}
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"volume",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(c.volumePorts[index]),
+		"-port.grpc=" + strconv.Itoa(c.volumeGrpcPorts[index]),
+		"-port.public=" + strconv.Itoa(c.volumePubPorts[index]),
+		"-dir=" + strings.Join(dataDirs, ","),
+		"-max=" + strings.Join(maxPerDir, ","),
+		"-master=127.0.0.1:" + strconv.Itoa(c.masterPort),
+		"-readMode=" + c.profile.ReadMode,
+		"-concurrentUploadLimitMB=" + strconv.Itoa(c.profile.ConcurrentUploadLimitMB),
+		"-concurrentDownloadLimitMB=" + strconv.Itoa(c.profile.ConcurrentDownloadLimitMB),
+		// Integration tests deliberately exercise loopback S3 endpoints; allow the SSRF guard to be bypassed for them.
+		"-volume.allowUntrustedRemoteEndpoints",
+	}
+	if c.profile.InflightUploadTimeout > 0 {
+		args = append(args, "-inflightUploadDataTimeout="+c.profile.InflightUploadTimeout.String())
+	}
+	if c.profile.InflightDownloadTimeout > 0 {
+		args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
+	}
+
+	cmd := exec.Command(c.weedBinary, args...)
+	cmd.Dir = c.baseDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+
+	if err = cmd.Start(); err != nil {
+		return err
+	}
+	c.volumeCmds[index] = cmd
+	return nil
+}
+
+func (c *MultiVolumeCluster) waitForHTTP(url string) error {
+	return (&Cluster{}).waitForHTTP(url)
+}
+
+func (c *MultiVolumeCluster) waitForTCP(addr string) error {
+	return (&Cluster{}).waitForTCP(addr)
+}
+
+func (c *MultiVolumeCluster) tailLog(logName string) string {
+	return (&Cluster{logsDir: c.logsDir}).tailLog(logName)
+}
+
+func (c *MultiVolumeCluster) MasterAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
+}
+
+func (c *MultiVolumeCluster) MasterURL() string {
+	return "http://" + c.MasterAddress()
+}
+
+func (c *MultiVolumeCluster) VolumeAdminAddress(index int) string {
+	if index < 0 || index >= len(c.volumePorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePorts[index]))
+}
+
+func (c *MultiVolumeCluster) VolumePublicAddress(index int) string {
+	if index < 0 || index >= len(c.volumePubPorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPorts[index]))
+}
+
+func (c *MultiVolumeCluster) VolumeGRPCAddress(index int) string {
+	if index < 0 || index >= len(c.volumeGrpcPorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPorts[index]))
+}
+
+func (c *MultiVolumeCluster) VolumeAdminURL(index int) string {
+	return "http://" + c.VolumeAdminAddress(index)
+}
+
+func (c *MultiVolumeCluster) VolumePublicURL(index int) string {
+	return "http://" + c.VolumePublicAddress(index)
+}
+
+func (c *MultiVolumeCluster) BaseDir() string {
+	return c.baseDir
+}
+
+// VolumeDiskDir returns the data directory backing disk diskIndex on volume
+// server serverIndex — i.e. the directory for that DiskLocation, for file
+// inspection in tests.
+func (c *MultiVolumeCluster) VolumeDiskDir(serverIndex, diskIndex int) string {
+	if serverIndex < 0 || serverIndex >= len(c.volumeDiskDirs) {
+		return ""
+	}
+	disks := c.volumeDiskDirs[serverIndex]
+	if diskIndex < 0 || diskIndex >= len(disks) {
+		return ""
+	}
+	return disks[diskIndex]
+}

test/volume_server/framework/cluster_multi_rust.go

@@ -0,0 +1,287 @@
+package framework
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+// RustMultiVolumeCluster wraps a Go master + multiple Rust volume servers
+// for integration testing. It mirrors MultiVolumeCluster but uses the Rust
+// volume binary instead of the Go weed binary for volume servers.
+type RustMultiVolumeCluster struct {
+	testingTB testing.TB
+	profile   matrix.Profile
+
+	weedBinary       string // Go weed binary (for the master)
+	rustVolumeBinary string // Rust volume binary
+
+	baseDir           string
+	configDir         string
+	logsDir           string
+	keepLogs          bool
+	volumeServerCount int
+
+	masterPort     int
+	masterGrpcPort int
+
+	volumePorts     []int
+	volumeGrpcPorts []int
+	volumePubPorts  []int
+
+	masterCmd  *exec.Cmd
+	volumeCmds []*exec.Cmd
+
+	cleanupOnce sync.Once
+}
+
+// StartRustMultiVolumeCluster starts a cluster with a Go master and the
+// specified number of Rust volume servers.
+func StartRustMultiVolumeCluster(t testing.TB, profile matrix.Profile, serverCount int) *RustMultiVolumeCluster {
+	t.Helper()
+
+	if serverCount < 1 {
+		t.Fatalf("serverCount must be at least 1, got %d", serverCount)
+	}
+
+	weedBinary, err := FindOrBuildWeedBinary()
+	if err != nil {
+		t.Fatalf("resolve weed binary: %v", err)
+	}
+
+	rustBinary, err := FindOrBuildRustBinary()
+	if err != nil {
+		t.Fatalf("resolve rust volume binary: %v", err)
+	}
+
+	baseDir, keepLogs, err := newWorkDir()
+	if err != nil {
+		t.Fatalf("create temp test directory: %v", err)
+	}
+
+	configDir := filepath.Join(baseDir, "config")
+	logsDir := filepath.Join(baseDir, "logs")
+	masterDataDir := filepath.Join(baseDir, "master")
+
+	// Create directories for master and all volume servers
+	dirs := []string{configDir, logsDir, masterDataDir}
+	for i := 0; i < serverCount; i++ {
+		dirs = append(dirs, filepath.Join(baseDir, fmt.Sprintf("volume%d", i)))
+	}
+	for _, dir := range dirs {
+		if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
+			t.Fatalf("create %s: %v", dir, mkErr)
+		}
+	}
+
+	if err = writeSecurityConfig(configDir, profile); err != nil {
+		t.Fatalf("write security config: %v", err)
+	}
+
+	// Allocate ports for all volume servers (3 ports per server: admin, grpc, public)
+	// If SplitPublicPort is true, we need an additional port per server
+	portsPerServer := 3
+	if profile.SplitPublicPort {
+		portsPerServer = 4
+	}
+	totalPorts := serverCount * portsPerServer
+	miniPorts, ports, err := testutil.AllocatePortSet(1, totalPorts)
+	if err != nil {
+		t.Fatalf("allocate ports: %v", err)
+	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset
+
+	c := &RustMultiVolumeCluster{
+		testingTB:         t,
+		profile:           profile,
+		weedBinary:        weedBinary,
+		rustVolumeBinary:  rustBinary,
+		baseDir:           baseDir,
+		configDir:         configDir,
+		logsDir:           logsDir,
+		keepLogs:          keepLogs,
+		volumeServerCount: serverCount,
+		masterPort:        masterPort,
+		masterGrpcPort:    masterGrpcPort,
+		volumePorts:       make([]int, serverCount),
+		volumeGrpcPorts:   make([]int, serverCount),
+		volumePubPorts:    make([]int, serverCount),
+		volumeCmds:        make([]*exec.Cmd, serverCount),
+	}
+
+	// Assign ports to each volume server
+	for i := 0; i < serverCount; i++ {
+		baseIdx := i * portsPerServer
+		c.volumePorts[i] = ports[baseIdx]
+		c.volumeGrpcPorts[i] = ports[baseIdx+1]
+
+		// Assign public port, using baseIdx+3 if SplitPublicPort, else baseIdx+2
+		pubPortIdx := baseIdx + 2
+		if profile.SplitPublicPort {
+			pubPortIdx = baseIdx + 3
+		}
+		c.volumePubPorts[i] = ports[pubPortIdx]
+	}
+
+	// Start master (Go)
+	if err = c.startMaster(masterDataDir); err != nil {
+		c.Stop()
+		t.Fatalf("start master: %v", err)
+	}
+	helper := &Cluster{logsDir: logsDir}
+	if err = helper.waitForHTTP(c.MasterURL() + "/dir/status"); err != nil {
+		masterLog := helper.tailLog("master.log")
+		c.Stop()
+		t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
+	}
+
+	// Start all Rust volume servers
+	for i := 0; i < serverCount; i++ {
+		volumeDataDir := filepath.Join(baseDir, fmt.Sprintf("volume%d", i))
+		if err = c.startRustVolume(i, volumeDataDir); err != nil {
+			volumeLog := fmt.Sprintf("volume%d.log", i)
+			c.Stop()
+			t.Fatalf("start rust volume server %d: %v\nvolume log tail:\n%s", i, err, helper.tailLog(volumeLog))
+		}
+		if err = helper.waitForHTTP(c.VolumeAdminURL(i) + "/healthz"); err != nil {
+			volumeLog := fmt.Sprintf("volume%d.log", i)
+			c.Stop()
+			t.Fatalf("wait for rust volume server %d readiness: %v\nvolume log tail:\n%s", i, err, helper.tailLog(volumeLog))
+		}
+		if err = helper.waitForTCP(c.VolumeGRPCAddress(i)); err != nil {
+			volumeLog := fmt.Sprintf("volume%d.log", i)
+			c.Stop()
+			t.Fatalf("wait for rust volume server %d grpc readiness: %v\nvolume log tail:\n%s", i, err, helper.tailLog(volumeLog))
+		}
+	}
+
+	t.Cleanup(func() {
+		c.Stop()
+	})
+
+	return c
+}
+
+func (c *RustMultiVolumeCluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		// Stop volume servers in reverse order
+		for i := len(c.volumeCmds) - 1; i >= 0; i-- {
+			stopProcess(c.volumeCmds[i])
+		}
+		stopProcess(c.masterCmd)
+		if !c.keepLogs && !c.testingTB.Failed() {
+			_ = os.RemoveAll(c.baseDir)
+		} else if c.baseDir != "" {
+			c.testingTB.Logf("rust multi volume server integration logs kept at %s", c.baseDir)
+		}
+	})
+}
+
+func (c *RustMultiVolumeCluster) startMaster(dataDir string) error {
+	logFile, err := os.Create(filepath.Join(c.logsDir, "master.log"))
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"master",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(c.masterPort),
+		"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
+		"-mdir=" + dataDir,
+		"-peers=none",
+		"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
+		"-defaultReplication=000",
+	}
+
+	c.masterCmd = exec.Command(c.weedBinary, args...)
+	c.masterCmd.Dir = c.baseDir
+	c.masterCmd.Stdout = logFile
+	c.masterCmd.Stderr = logFile
+	return c.masterCmd.Start()
+}
+
+func (c *RustMultiVolumeCluster) startRustVolume(index int, dataDir string) error {
+	logName := fmt.Sprintf("volume%d.log", index)
+	logFile, err := os.Create(filepath.Join(c.logsDir, logName))
+	if err != nil {
+		return err
+	}
+
+	args := rustVolumeArgs(
+		c.profile,
+		c.configDir,
+		c.masterPort,
+		c.volumePorts[index],
+		c.volumeGrpcPorts[index],
+		c.volumePubPorts[index],
+		dataDir,
+	)
+
+	cmd := exec.Command(c.rustVolumeBinary, args...)
+	cmd.Dir = c.baseDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+
+	if err = cmd.Start(); err != nil {
+		return err
+	}
+	c.volumeCmds[index] = cmd
+	return nil
+}
+
+// --- accessor methods (mirror MultiVolumeCluster) ---
+
+func (c *RustMultiVolumeCluster) MasterAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
+}
+
+func (c *RustMultiVolumeCluster) MasterURL() string {
+	return "http://" + c.MasterAddress()
+}
+
+func (c *RustMultiVolumeCluster) VolumeAdminAddress(index int) string {
+	if index < 0 || index >= len(c.volumePorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePorts[index]))
+}
+
+func (c *RustMultiVolumeCluster) VolumePublicAddress(index int) string {
+	if index < 0 || index >= len(c.volumePubPorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePubPorts[index]))
+}
+
+func (c *RustMultiVolumeCluster) VolumeGRPCAddress(index int) string {
+	if index < 0 || index >= len(c.volumeGrpcPorts) {
+		return ""
+	}
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPorts[index]))
+}
+
+func (c *RustMultiVolumeCluster) VolumeAdminURL(index int) string {
+	return "http://" + c.VolumeAdminAddress(index)
+}
+
+func (c *RustMultiVolumeCluster) VolumePublicURL(index int) string {
+	return "http://" + c.VolumePublicAddress(index)
+}
+
+func (c *RustMultiVolumeCluster) BaseDir() string {
+	return c.baseDir
+}

test/volume_server/framework/cluster_rust.go

@@ -0,0 +1,348 @@
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// RustCluster wraps a Go master + Rust volume server for integration testing.
+type RustCluster struct {
+	testingTB testing.TB
+	profile   matrix.Profile
+
+	weedBinary       string // Go weed binary (for the master)
+	rustVolumeBinary string // Rust volume binary
+
+	baseDir   string
+	configDir string
+	logsDir   string
+	keepLogs  bool
+
+	masterPort     int
+	masterGrpcPort int
+	volumePort     int
+	volumeGrpcPort int
+	volumePubPort  int
+
+	masterCmd *exec.Cmd
+	volumeCmd *exec.Cmd
+
+	cleanupOnce sync.Once
+}
+
+var (
+	rustBinaryOnce sync.Once
+	rustBinaryPath string
+	rustBinaryErr  error
+)
+
+// StartRustVolumeCluster starts a Go master + Rust volume server.
+func StartRustVolumeCluster(t testing.TB, profile matrix.Profile) *RustCluster {
+	t.Helper()
+
+	weedBinary, err := FindOrBuildWeedBinary()
+	if err != nil {
+		t.Fatalf("resolve weed binary: %v", err)
+	}
+
+	rustBinary, err := FindOrBuildRustBinary()
+	if err != nil {
+		t.Fatalf("resolve rust volume binary: %v", err)
+	}
+
+	baseDir, keepLogs, err := newWorkDir()
+	if err != nil {
+		t.Fatalf("create temp test directory: %v", err)
+	}
+
+	configDir := filepath.Join(baseDir, "config")
+	logsDir := filepath.Join(baseDir, "logs")
+	masterDataDir := filepath.Join(baseDir, "master")
+	volumeDataDir := filepath.Join(baseDir, "volume")
+	for _, dir := range []string{configDir, logsDir, masterDataDir, volumeDataDir} {
+		if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
+			t.Fatalf("create %s: %v", dir, mkErr)
+		}
+	}
+
+	if err = writeSecurityConfig(configDir, profile); err != nil {
+		t.Fatalf("write security config: %v", err)
+	}
+
+	miniPorts, ports, err := testutil.AllocatePortSet(1, 3)
+	if err != nil {
+		t.Fatalf("allocate ports: %v", err)
+	}
+	masterPort := miniPorts[0]
+	masterGrpcPort := masterPort + testutil.GrpcPortOffset
+
+	rc := &RustCluster{
+		testingTB:        t,
+		profile:          profile,
+		weedBinary:       weedBinary,
+		rustVolumeBinary: rustBinary,
+		baseDir:          baseDir,
+		configDir:        configDir,
+		logsDir:          logsDir,
+		keepLogs:         keepLogs,
+		masterPort:       masterPort,
+		masterGrpcPort:   masterGrpcPort,
+		volumePort:       ports[0],
+		volumeGrpcPort:   ports[1],
+		volumePubPort:    ports[0],
+	}
+	if profile.SplitPublicPort {
+		rc.volumePubPort = ports[2]
+	}
+
+	if err = rc.startMaster(masterDataDir); err != nil {
+		rc.Stop()
+		t.Fatalf("start master: %v", err)
+	}
+	// Reuse the same HTTP readiness helper via an unexported Cluster shim.
+	helper := &Cluster{logsDir: logsDir}
+	if err = helper.waitForHTTP(rc.MasterURL() + "/dir/status"); err != nil {
+		masterLog := helper.tailLog("master.log")
+		rc.Stop()
+		t.Fatalf("wait for master readiness: %v\nmaster log tail:\n%s", err, masterLog)
+	}
+
+	if err = rc.startRustVolume(volumeDataDir); err != nil {
+		masterLog := helper.tailLog("master.log")
+		rc.Stop()
+		t.Fatalf("start rust volume: %v\nmaster log tail:\n%s", err, masterLog)
+	}
+	if err = helper.waitForHTTP(rc.VolumeAdminURL() + "/healthz"); err != nil {
+		volumeLog := helper.tailLog("volume.log")
+		rc.Stop()
+		t.Fatalf("wait for rust volume readiness: %v\nvolume log tail:\n%s", err, volumeLog)
+	}
+	if err = helper.waitForTCP(rc.VolumeGRPCAddress()); err != nil {
+		volumeLog := helper.tailLog("volume.log")
+		rc.Stop()
+		t.Fatalf("wait for rust volume grpc readiness: %v\nvolume log tail:\n%s", err, volumeLog)
+	}
+
+	t.Cleanup(func() {
+		rc.Stop()
+	})
+
+	return rc
+}
+
+// Stop terminates all processes and cleans temporary files.
+func (rc *RustCluster) Stop() {
+	if rc == nil {
+		return
+	}
+	rc.cleanupOnce.Do(func() {
+		stopProcess(rc.volumeCmd)
+		stopProcess(rc.masterCmd)
+		if !rc.keepLogs && !rc.testingTB.Failed() {
+			_ = os.RemoveAll(rc.baseDir)
+		} else if rc.baseDir != "" {
+			rc.testingTB.Logf("rust volume server integration logs kept at %s", rc.baseDir)
+		}
+	})
+}
+
+func (rc *RustCluster) startMaster(dataDir string) error {
+	logFile, err := os.Create(filepath.Join(rc.logsDir, "master.log"))
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-config_dir=" + rc.configDir,
+		"master",
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(rc.masterPort),
+		"-port.grpc=" + strconv.Itoa(rc.masterGrpcPort),
+		"-mdir=" + dataDir,
+		"-peers=none",
+		"-volumeSizeLimitMB=" + strconv.Itoa(testVolumeSizeLimitMB),
+		"-defaultReplication=000",
+	}
+
+	rc.masterCmd = exec.Command(rc.weedBinary, args...)
+	rc.masterCmd.Dir = rc.baseDir
+	rc.masterCmd.Stdout = logFile
+	rc.masterCmd.Stderr = logFile
+	return rc.masterCmd.Start()
+}
+
+func rustVolumeArgs(
+	profile matrix.Profile,
+	configDir string,
+	masterPort int,
+	volumePort int,
+	volumeGrpcPort int,
+	volumePubPort int,
+	dataDir string,
+) []string {
+	args := []string{
+		"--port", strconv.Itoa(volumePort),
+		"--port.grpc", strconv.Itoa(volumeGrpcPort),
+		"--port.public", strconv.Itoa(volumePubPort),
+		"--ip", "127.0.0.1",
+		"--ip.bind", "127.0.0.1",
+		"--dir", dataDir,
+		"--max", "16",
+		"--master", "127.0.0.1:" + strconv.Itoa(masterPort),
+		"--securityFile", filepath.Join(configDir, "security.toml"),
+		"--readMode", profile.ReadMode,
+		"--concurrentUploadLimitMB", strconv.Itoa(profile.ConcurrentUploadLimitMB),
+		"--concurrentDownloadLimitMB", strconv.Itoa(profile.ConcurrentDownloadLimitMB),
+		"--preStopSeconds", "0",
+	}
+	if profile.InflightUploadTimeout > 0 {
+		args = append(args, "--inflightUploadDataTimeout", profile.InflightUploadTimeout.String())
+	}
+	if profile.InflightDownloadTimeout > 0 {
+		args = append(args, "--inflightDownloadDataTimeout", profile.InflightDownloadTimeout.String())
+	}
+	return args
+}
+
+func (rc *RustCluster) startRustVolume(dataDir string) error {
+	logFile, err := os.Create(filepath.Join(rc.logsDir, "volume.log"))
+	if err != nil {
+		return err
+	}
+
+	args := rustVolumeArgs(
+		rc.profile,
+		rc.configDir,
+		rc.masterPort,
+		rc.volumePort,
+		rc.volumeGrpcPort,
+		rc.volumePubPort,
+		dataDir,
+	)
+
+	rc.volumeCmd = exec.Command(rc.rustVolumeBinary, args...)
+	rc.volumeCmd.Dir = rc.baseDir
+	rc.volumeCmd.Stdout = logFile
+	rc.volumeCmd.Stderr = logFile
+	return rc.volumeCmd.Start()
+}
+
+// FindOrBuildRustBinary returns an executable Rust volume binary, building one when needed.
+func FindOrBuildRustBinary() (string, error) {
+	if fromEnv := os.Getenv("RUST_VOLUME_BINARY"); fromEnv != "" {
+		if isExecutableFile(fromEnv) {
+			return fromEnv, nil
+		}
+		return "", fmt.Errorf("RUST_VOLUME_BINARY is set but not executable: %s", fromEnv)
+	}
+
+	rustBinaryOnce.Do(func() {
+		// Derive the Rust volume crate directory from this source file's location.
+		rustCrateDir := ""
+		if _, file, _, ok := runtime.Caller(0); ok {
+			repoRoot := filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
+			for _, candidate := range []string{"seaweed-volume", "weed-volume"} {
+				dir := filepath.Join(repoRoot, candidate)
+				if isDir(dir) && isFile(filepath.Join(dir, "Cargo.toml")) {
+					rustCrateDir = dir
+					break
+				}
+			}
+		}
+		if rustCrateDir == "" {
+			rustBinaryErr = fmt.Errorf("unable to detect Rust volume crate directory")
+			return
+		}
+
+		releaseBin := filepath.Join(rustCrateDir, "target", "release", "weed-volume")
+
+		// Always rebuild once per test process so the harness uses current source
+		// and features. The crate defaults to 5bytes, so a test binary built
+		// without 5BytesOffset has to turn it off or the Rust server refuses the
+		// .vif the Go server just wrote.
+		args := []string{"build", "--release"}
+		if types.OffsetSize != 5 {
+			args = append(args, "--no-default-features")
+		}
+		cmd := exec.Command("cargo", args...)
+		cmd.Dir = rustCrateDir
+		var out bytes.Buffer
+		cmd.Stdout = &out
+		cmd.Stderr = &out
+		if err := cmd.Run(); err != nil {
+			rustBinaryErr = fmt.Errorf("build rust volume binary: %w\n%s", err, out.String())
+			return
+		}
+		if !isExecutableFile(releaseBin) {
+			rustBinaryErr = fmt.Errorf("built rust volume binary is not executable: %s", releaseBin)
+			return
+		}
+		rustBinaryPath = releaseBin
+	})
+
+	if rustBinaryErr != nil {
+		return "", rustBinaryErr
+	}
+	return rustBinaryPath, nil
+}
+
+func isDir(path string) bool {
+	info, err := os.Stat(path)
+	return err == nil && info.IsDir()
+}
+
+func isFile(path string) bool {
+	info, err := os.Stat(path)
+	return err == nil && info.Mode().IsRegular()
+}
+
+// --- accessor methods (mirror Cluster) ---
+
+func (rc *RustCluster) MasterAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(rc.masterPort))
+}
+
+func (rc *RustCluster) VolumeAdminAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(rc.volumePort))
+}
+
+func (rc *RustCluster) VolumePublicAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(rc.volumePubPort))
+}
+
+func (rc *RustCluster) VolumeGRPCAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(rc.volumeGrpcPort))
+}
+
+// VolumeServerAddress returns SeaweedFS server address format: ip:httpPort.grpcPort
+func (rc *RustCluster) VolumeServerAddress() string {
+	return fmt.Sprintf("%s.%d", rc.VolumeAdminAddress(), rc.volumeGrpcPort)
+}
+
+func (rc *RustCluster) MasterURL() string {
+	return "http://" + rc.MasterAddress()
+}
+
+func (rc *RustCluster) VolumeAdminURL() string {
+	return "http://" + rc.VolumeAdminAddress()
+}
+
+func (rc *RustCluster) VolumePublicURL() string {
+	return "http://" + rc.VolumePublicAddress()
+}
+
+func (rc *RustCluster) BaseDir() string {
+	return rc.baseDir
+}

test/volume_server/framework/cluster_rust_test.go

@@ -0,0 +1,38 @@
+package framework
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+func TestRustVolumeArgsIncludeReadMode(t *testing.T) {
+	profile := matrix.P1()
+	profile.ReadMode = "redirect"
+	profile.ConcurrentUploadLimitMB = 7
+	profile.ConcurrentDownloadLimitMB = 9
+	profile.InflightUploadTimeout = 3 * time.Second
+	profile.InflightDownloadTimeout = 4 * time.Second
+
+	args := rustVolumeArgs(profile, "/tmp/config", 9333, 18080, 28080, 38080, "/tmp/data")
+
+	assertArgPair(t, args, "--readMode", "redirect")
+	assertArgPair(t, args, "--concurrentUploadLimitMB", "7")
+	assertArgPair(t, args, "--concurrentDownloadLimitMB", "9")
+	assertArgPair(t, args, "--inflightUploadDataTimeout", "3s")
+	assertArgPair(t, args, "--inflightDownloadDataTimeout", "4s")
+}
+
+func assertArgPair(t *testing.T, args []string, flag string, want string) {
+	t.Helper()
+	for i := 0; i+1 < len(args); i += 2 {
+		if args[i] == flag {
+			if args[i+1] != want {
+				t.Fatalf("%s value mismatch: got %q want %q", flag, args[i+1], want)
+			}
+			return
+		}
+	}
+	t.Fatalf("missing %s in args: %v", flag, args)
+}

test/volume_server/framework/cluster_with_filer.go

@@ -0,0 +1,92 @@
+package framework
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+)
+
+type ClusterWithFiler struct {
+	*Cluster
+
+	filerCmd      *exec.Cmd
+	filerPort     int
+	filerGrpcPort int
+}
+
+func StartSingleVolumeClusterWithFiler(t testing.TB, profile matrix.Profile) *ClusterWithFiler {
+	t.Helper()
+
+	baseCluster := StartSingleVolumeCluster(t, profile)
+
+	ports, err := testutil.AllocatePorts(2)
+	if err != nil {
+		t.Fatalf("allocate filer ports: %v", err)
+	}
+
+	filerDataDir := filepath.Join(baseCluster.baseDir, "filer")
+	if mkErr := os.MkdirAll(filerDataDir, 0o755); mkErr != nil {
+		t.Fatalf("create filer data dir: %v", mkErr)
+	}
+
+	logFile, err := os.Create(filepath.Join(baseCluster.logsDir, "filer.log"))
+	if err != nil {
+		t.Fatalf("create filer log file: %v", err)
+	}
+
+	filerPort := ports[0]
+	filerGrpcPort := ports[1]
+	args := []string{
+		"-config_dir=" + baseCluster.configDir,
+		"filer",
+		"-master=127.0.0.1:" + strconv.Itoa(baseCluster.masterPort),
+		"-ip=127.0.0.1",
+		"-port=" + strconv.Itoa(filerPort),
+		"-port.grpc=" + strconv.Itoa(filerGrpcPort),
+		"-defaultStoreDir=" + filerDataDir,
+	}
+
+	filerCmd := exec.Command(baseCluster.weedBinary, args...)
+	filerCmd.Dir = baseCluster.baseDir
+	filerCmd.Stdout = logFile
+	filerCmd.Stderr = logFile
+	if err = filerCmd.Start(); err != nil {
+		t.Fatalf("start filer: %v", err)
+	}
+
+	if err = baseCluster.waitForTCP(net.JoinHostPort("127.0.0.1", strconv.Itoa(filerGrpcPort))); err != nil {
+		filerLogTail := baseCluster.tailLog("filer.log")
+		stopProcess(filerCmd)
+		t.Fatalf("wait for filer grpc readiness: %v\nfiler log tail:\n%s", err, filerLogTail)
+	}
+
+	t.Cleanup(func() {
+		stopProcess(filerCmd)
+	})
+
+	return &ClusterWithFiler{
+		Cluster:       baseCluster,
+		filerCmd:      filerCmd,
+		filerPort:     filerPort,
+		filerGrpcPort: filerGrpcPort,
+	}
+}
+
+func (c *ClusterWithFiler) FilerAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.filerPort))
+}
+
+func (c *ClusterWithFiler) FilerGRPCAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.filerGrpcPort))
+}
+
+func (c *ClusterWithFiler) FilerServerAddress() string {
+	return fmt.Sprintf("%s.%d", c.FilerAddress(), c.filerGrpcPort)
+}

test/volume_server/framework/fault_injection.go

@@ -0,0 +1,8 @@
+package framework
+
+// Phase 0 placeholder for future fault injection utilities.
+//
+// Planned extensions:
+// - restart/kill selected processes
+// - temporary network isolation hooks
+// - master or peer outage helpers for proxy/replication branch coverage

test/volume_server/framework/grpc_client.go

@@ -0,0 +1,28 @@
+package framework
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func DialVolumeServer(t testing.TB, address string) (*grpc.ClientConn, volume_server_pb.VolumeServerClient) {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	conn, err := grpc.DialContext(ctx, address,
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithBlock(),
+	)
+	if err != nil {
+		t.Fatalf("dial volume grpc %s: %v", address, err)
+	}
+
+	return conn, volume_server_pb.NewVolumeServerClient(conn)
+}

test/volume_server/framework/http_client.go

@@ -0,0 +1,34 @@
+package framework
+
+import (
+	"io"
+	"net/http"
+	"testing"
+	"time"
+)
+
+func NewHTTPClient() *http.Client {
+	return &http.Client{Timeout: 10 * time.Second}
+}
+
+func DoRequest(t testing.TB, client *http.Client, req *http.Request) *http.Response {
+	t.Helper()
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("http request %s %s: %v", req.Method, req.URL.String(), err)
+	}
+	return resp
+}
+
+func ReadAllAndClose(t testing.TB, resp *http.Response) []byte {
+	t.Helper()
+	if resp == nil {
+		return nil
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("read response body: %v", err)
+	}
+	return body
+}

test/volume_server/framework/volume_fixture.go

@@ -0,0 +1,127 @@
+package framework
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+func AllocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string) {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{
+		VolumeId:    volumeID,
+		Collection:  collection,
+		Replication: "000",
+		Version:     uint32(needle.GetCurrentVersion()),
+	})
+	if err != nil {
+		t.Fatalf("allocate volume %d: %v", volumeID, err)
+	}
+}
+
+func NewFileID(volumeID uint32, key uint64, cookie uint32) string {
+	return needle.NewFileId(needle.VolumeId(volumeID), key, cookie).String()
+}
+
+func UploadBytes(t testing.TB, client *http.Client, volumeURL, fid string, data []byte) *http.Response {
+	t.Helper()
+
+	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/%s", volumeURL, fid), bytes.NewReader(data))
+	if err != nil {
+		t.Fatalf("build upload request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/octet-stream")
+	req.Header.Set("Content-Length", fmt.Sprintf("%d", len(data)))
+	return DoRequest(t, client, req)
+}
+
+// CorruptIndexFile appends garbage bytes to a volume's .idx file on disk so
+// that CheckIndexFile detects a size mismatch during scrub.
+func CorruptIndexFile(t testing.TB, baseDir string, volumeID uint32) {
+	t.Helper()
+	idxPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.idx", volumeID))
+	f, err := os.OpenFile(idxPath, os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		t.Fatalf("open idx file for corruption: %v", err)
+	}
+	defer f.Close()
+	if _, err := f.Write([]byte{0xDE, 0xAD}); err != nil {
+		t.Fatalf("corrupt idx file: %v", err)
+	}
+}
+
+// EnableMaintenanceMode puts the volume server into maintenance mode.
+func EnableMaintenanceMode(t testing.TB, ctx context.Context, client volume_server_pb.VolumeServerClient) {
+	t.Helper()
+	stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{
+			Maintenance: true,
+			Version:     stateResp.GetState().GetVersion(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+}
+
+// CorruptDatFile truncates a volume's .dat file to just the superblock (8 bytes)
+// so that needle reads fail during a full scrub due to data file size mismatch.
+func CorruptDatFile(t testing.TB, baseDir string, volumeID uint32) {
+	t.Helper()
+	datPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.dat", volumeID))
+	// Truncate to superblock size only, removing all needle data.
+	if err := os.Truncate(datPath, 8); err != nil {
+		t.Fatalf("truncate dat file for corruption: %v", err)
+	}
+}
+
+// CorruptEcxFile appends garbage bytes to a volume's .ecx file on disk so
+// that CheckIndexFile detects a size mismatch during EC index scrub.
+func CorruptEcxFile(t testing.TB, baseDir string, volumeID uint32) {
+	t.Helper()
+	ecxPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.ecx", volumeID))
+	f, err := os.OpenFile(ecxPath, os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		t.Fatalf("open ecx file for corruption: %v", err)
+	}
+	defer f.Close()
+	if _, err := f.Write([]byte{0xDE, 0xAD}); err != nil {
+		t.Fatalf("corrupt ecx file: %v", err)
+	}
+}
+
+// CorruptEcShardFile truncates an EC shard file to 1 byte so that local shard
+// reads fail during an EC scrub.
+func CorruptEcShardFile(t testing.TB, baseDir string, volumeID uint32, shardID int) {
+	t.Helper()
+	shardPath := filepath.Join(baseDir, "volume", fmt.Sprintf("%d.ec%02d", volumeID, shardID))
+	if err := os.Truncate(shardPath, 1); err != nil {
+		t.Fatalf("truncate EC shard file %s: %v", shardPath, err)
+	}
+}
+
+func ReadBytes(t testing.TB, client *http.Client, volumeURL, fid string) *http.Response {
+	t.Helper()
+
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", volumeURL, fid), nil)
+	if err != nil {
+		t.Fatalf("build read request: %v", err)
+	}
+	return DoRequest(t, client, req)
+}