snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/volume_server/grpc/admin_extra_test.go

@@ -0,0 +1,517 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestVolumeNeedleStatusForUploadedFile(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(21)
+	const needleID = uint64(778899)
+	const cookie = uint32(0xA1B2C3D4)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	client := framework.NewHTTPClient()
+	payload := []byte("needle-status-payload")
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
+		VolumeId: volumeID,
+		NeedleId: needleID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeNeedleStatus failed: %v", err)
+	}
+	if statusResp.GetNeedleId() != needleID {
+		t.Fatalf("needle id mismatch: got %d want %d", statusResp.GetNeedleId(), needleID)
+	}
+	if statusResp.GetCookie() != cookie {
+		t.Fatalf("cookie mismatch: got %d want %d", statusResp.GetCookie(), cookie)
+	}
+	if statusResp.GetSize() == 0 {
+		t.Fatalf("expected non-zero needle size")
+	}
+}
+
+func TestVolumeNeedleStatusIncludesTtlAndLastModified(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(27)
+	const needleID = uint64(778901)
+	const cookie = uint32(0xA1B2C3D6)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	client := framework.NewHTTPClient()
+	uploadReq := mustNewRequest(t, http.MethodPost, clusterHarness.VolumeAdminURL()+"/"+fid+"?ttl=7d&ts=1700000000")
+	uploadReq.Body = io.NopCloser(strings.NewReader("needle-status-ttl-payload"))
+	uploadReq.ContentLength = int64(len("needle-status-ttl-payload"))
+	uploadReq.Header.Set("Content-Type", "application/octet-stream")
+	uploadResp := framework.DoRequest(t, client, uploadReq)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
+		VolumeId: volumeID,
+		NeedleId: needleID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeNeedleStatus with ttl failed: %v", err)
+	}
+	// Go's ReadTTL normalizes via fitTtlCount: 7d → 1w (7 days = 1 week)
+	if statusResp.GetTtl() != "1w" {
+		t.Fatalf("ttl mismatch: got %q want %q", statusResp.GetTtl(), "1w")
+	}
+	if statusResp.GetLastModified() != 1700000000 {
+		t.Fatalf("last modified mismatch: got %d want %d", statusResp.GetLastModified(), 1700000000)
+	}
+}
+
+func TestVolumeNeedleStatusViaEcShardsWhenNormalVolumeUnmounted(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(26)
+	const needleID = uint64(778900)
+	const cookie = uint32(0xA1B2C3D5)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	payload := []byte("needle-status-ec-path-payload")
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload status: expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount data shards failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeUnmount failed: %v", err)
+	}
+
+	statusResp, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
+		VolumeId: volumeID,
+		NeedleId: needleID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeNeedleStatus via EC shards failed: %v", err)
+	}
+	if statusResp.GetNeedleId() != needleID {
+		t.Fatalf("needle id mismatch: got %d want %d", statusResp.GetNeedleId(), needleID)
+	}
+	if statusResp.GetCookie() != cookie {
+		t.Fatalf("cookie mismatch: got %d want %d", statusResp.GetCookie(), cookie)
+	}
+	if statusResp.GetSize() == 0 {
+		t.Fatalf("expected non-zero needle size from EC-backed needle status")
+	}
+
+	_, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
+		VolumeId: volumeID,
+		NeedleId: needleID + 999999,
+	})
+	if err == nil || !strings.Contains(strings.ToLower(err.Error()), "not found") {
+		t.Fatalf("VolumeNeedleStatus via EC shards missing-needle error mismatch: %v", err)
+	}
+}
+
+func TestVolumeNeedleStatusMissingVolumeAndNeedle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(25)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
+		VolumeId: 99925,
+		NeedleId: 1,
+	})
+	if err == nil {
+		t.Fatalf("VolumeNeedleStatus should fail for missing volume")
+	}
+	if !strings.Contains(strings.ToLower(err.Error()), "volume not found") {
+		t.Fatalf("VolumeNeedleStatus missing-volume error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
+		VolumeId: volumeID,
+		NeedleId: 123456789,
+	})
+	if err == nil {
+		t.Fatalf("VolumeNeedleStatus should fail for missing needle")
+	}
+	if !strings.Contains(strings.ToLower(err.Error()), "not found") {
+		t.Fatalf("VolumeNeedleStatus missing-needle error mismatch: %v", err)
+	}
+}
+
+func mustNewRequest(t testing.TB, method, url string) *http.Request {
+	t.Helper()
+	req, err := http.NewRequest(method, url, nil)
+	if err != nil {
+		t.Fatalf("create request %s %s: %v", method, url, err)
+	}
+	return req
+}
+
+func TestVolumeConfigureInvalidReplication(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(22)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{
+		VolumeId:    volumeID,
+		Replication: "bad-replication",
+	})
+	if err != nil {
+		t.Fatalf("VolumeConfigure returned grpc error: %v", err)
+	}
+	if resp.GetError() == "" {
+		t.Fatalf("VolumeConfigure expected response error for invalid replication")
+	}
+	if !strings.Contains(strings.ToLower(resp.GetError()), "replication") {
+		t.Fatalf("VolumeConfigure error should mention replication, got: %q", resp.GetError())
+	}
+}
+
+func TestVolumeConfigureSuccessAndMissingRollbackPath(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(24)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	successResp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{
+		VolumeId:    volumeID,
+		Replication: "000",
+	})
+	if err != nil {
+		t.Fatalf("VolumeConfigure success path returned grpc error: %v", err)
+	}
+	if successResp.GetError() != "" {
+		t.Fatalf("VolumeConfigure success path expected empty response error, got: %q", successResp.GetError())
+	}
+
+	statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus after successful configure failed: %v", err)
+	}
+	if statusResp.GetIsReadOnly() {
+		t.Fatalf("VolumeStatus after configure expected writable volume")
+	}
+
+	missingResp, err := grpcClient.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{
+		VolumeId:    99024,
+		Replication: "000",
+	})
+	if err != nil {
+		t.Fatalf("VolumeConfigure missing-volume branch should return response error, got grpc error: %v", err)
+	}
+	if missingResp.GetError() == "" {
+		t.Fatalf("VolumeConfigure missing-volume expected non-empty response error")
+	}
+	lower := strings.ToLower(missingResp.GetError())
+	if !strings.Contains(lower, "not found on disk") {
+		t.Fatalf("VolumeConfigure missing-volume error should mention not found on disk, got: %q", missingResp.GetError())
+	}
+	if !strings.Contains(lower, "failed to restore mount") {
+		t.Fatalf("VolumeConfigure missing-volume error should include remount rollback failure, got: %q", missingResp.GetError())
+	}
+}
+
+func TestPingVolumeTargetAndLeaveAffectsHealthz(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	// Volume servers only track their masters as ping peers, so drive the
+	// success path through the master target and rely on the master->volume
+	// admission for cross-type coverage.
+	pingResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
+		TargetType: cluster.MasterType,
+		Target:     clusterHarness.MasterAddress(),
+	})
+	if err != nil {
+		t.Fatalf("Ping master from volume server failed: %v", err)
+	}
+	if pingResp.GetRemoteTimeNs() == 0 {
+		t.Fatalf("expected remote timestamp from ping master")
+	}
+
+	if _, err = grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil {
+		t.Fatalf("VolumeServerLeave failed: %v", err)
+	}
+
+	client := framework.NewHTTPClient()
+	healthURL := clusterHarness.VolumeAdminURL() + "/healthz"
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, healthURL))
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode == http.StatusServiceUnavailable {
+			return
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("expected healthz to return 503 after leave, got %d", resp.StatusCode)
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+func TestVolumeServerLeaveIsIdempotent(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	if _, err := grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil {
+		t.Fatalf("first VolumeServerLeave failed: %v", err)
+	}
+	if _, err := grpcClient.VolumeServerLeave(ctx, &volume_server_pb.VolumeServerLeaveRequest{}); err != nil {
+		t.Fatalf("second VolumeServerLeave should be idempotent success, got: %v", err)
+	}
+
+	client := framework.NewHTTPClient()
+	healthURL := clusterHarness.VolumeAdminURL() + "/healthz"
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, healthURL))
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode == http.StatusServiceUnavailable {
+			return
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("expected healthz to stay 503 after repeated leave, got %d", resp.StatusCode)
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+func TestPingUnknownAndUnreachableTargetPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	// The volume server gates Ping on a known peer list. Unknown target types
+	// and addresses outside that list are rejected with InvalidArgument
+	// before any outbound dial is attempted.
+	_, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
+		TargetType: "unknown-type",
+		Target:     "127.0.0.1:12345",
+	})
+	if err == nil {
+		t.Fatalf("Ping unknown target type should be rejected by admission")
+	}
+	if got := status.Code(err); got != codes.InvalidArgument {
+		t.Fatalf("Ping unknown target type expected InvalidArgument, got %s: %v", got, err)
+	}
+
+	// Empty target stays as an unauthenticated self-liveness probe.
+	emptyTargetResp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{})
+	if err != nil {
+		t.Fatalf("Ping empty target should not return grpc error, got: %v", err)
+	}
+	if emptyTargetResp.GetRemoteTimeNs() != 0 {
+		t.Fatalf("Ping empty target expected remote_time_ns=0, got %d", emptyTargetResp.GetRemoteTimeNs())
+	}
+	if emptyTargetResp.GetStopTimeNs() < emptyTargetResp.GetStartTimeNs() {
+		t.Fatalf("Ping empty target expected stop_time_ns >= start_time_ns")
+	}
+
+	// 127.0.0.1:1 is not in the seed/current master set, so admission rejects
+	// the call before it can reach the network.
+	_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
+		TargetType: cluster.MasterType,
+		Target:     "127.0.0.1:1",
+	})
+	if err == nil {
+		t.Fatalf("Ping master target should be rejected when not in the known-peer set")
+	}
+	if got := status.Code(err); got != codes.InvalidArgument {
+		t.Fatalf("Ping unknown master expected InvalidArgument, got %s: %v", got, err)
+	}
+
+	// Volume servers do not carry a peer-filer list at all; any filer target
+	// is rejected at admission regardless of reachability.
+	_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
+		TargetType: cluster.FilerType,
+		Target:     "127.0.0.1:1",
+	})
+	if err == nil {
+		t.Fatalf("Ping filer target should be rejected by admission")
+	}
+	if got := status.Code(err); got != codes.InvalidArgument {
+		t.Fatalf("Ping filer expected InvalidArgument, got %s: %v", got, err)
+	}
+}
+
+func TestPingMasterTargetSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
+		TargetType: cluster.MasterType,
+		Target:     clusterHarness.MasterAddress(),
+	})
+	if err != nil {
+		t.Fatalf("Ping master target success path failed: %v", err)
+	}
+	if resp.GetRemoteTimeNs() == 0 {
+		t.Fatalf("Ping master target expected non-zero remote time")
+	}
+	if resp.GetStopTimeNs() < resp.GetStartTimeNs() {
+		t.Fatalf("Ping master target expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs())
+	}
+}
+
+func TestPingFilerTargetSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	// Volume servers do not maintain a peer-filer index, so drive the
+	// filer-presence success path through the volume server's master peer:
+	// reaching that master in a filer-bearing cluster is sufficient signal
+	// that filer joined the cluster without breaking volume-server ping.
+	clusterHarness := framework.StartSingleVolumeClusterWithFiler(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
+		TargetType: cluster.MasterType,
+		Target:     clusterHarness.MasterAddress(),
+	})
+	if err != nil {
+		t.Fatalf("Ping master from filer-bearing cluster failed: %v", err)
+	}
+	if resp.GetRemoteTimeNs() == 0 {
+		t.Fatalf("Ping master expected non-zero remote time")
+	}
+	if resp.GetStopTimeNs() < resp.GetStartTimeNs() {
+		t.Fatalf("Ping master expected stop >= start, got start=%d stop=%d", resp.GetStartTimeNs(), resp.GetStopTimeNs())
+	}
+}

test/volume_server/grpc/admin_lifecycle_test.go

@@ -0,0 +1,209 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func TestVolumeAdminLifecycleRPCs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	const volumeID = uint32(11)
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	statusResp, err := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus failed: %v", err)
+	}
+	if statusResp.GetFileCount() != 0 {
+		t.Fatalf("new volume should be empty, got file_count=%d", statusResp.GetFileCount())
+	}
+
+	if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil {
+		t.Fatalf("VolumeUnmount failed: %v", err)
+	}
+	if _, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID}); err != nil {
+		t.Fatalf("VolumeMount failed: %v", err)
+	}
+
+	if _, err = client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}); err != nil {
+		t.Fatalf("VolumeDelete failed: %v", err)
+	}
+
+	_, err = client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err == nil {
+		t.Fatalf("VolumeStatus should fail after delete")
+	}
+	if st, ok := status.FromError(err); !ok || st.Code() == codes.OK {
+		t.Fatalf("VolumeStatus error should be a non-OK grpc status, got: %v", err)
+	}
+}
+
+func TestVolumeDeleteOnlyEmptyVariants(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(13)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 66001, 0x11223344)
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("volume-delete-only-empty"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true})
+	if err == nil || !strings.Contains(err.Error(), "volume not empty") {
+		t.Fatalf("VolumeDelete only_empty=true expected volume-not-empty error, got: %v", err)
+	}
+
+	_, err = grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: false})
+	if err != nil {
+		t.Fatalf("VolumeDelete only_empty=false failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err == nil {
+		t.Fatalf("VolumeStatus should fail after non-empty delete with only_empty=false")
+	}
+}
+
+func TestMaintenanceModeRejectsAllocateVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	_, err = client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{VolumeId: 12, Replication: "000"})
+	if err == nil {
+		t.Fatalf("AllocateVolume should fail when maintenance mode is enabled")
+	}
+	if !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("expected maintenance mode error, got: %v", err)
+	}
+}
+
+func TestAllocateDuplicateAndMountUnmountMissingVariants(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	const missingVolumeID = uint32(99331)
+	const volumeID = uint32(14)
+
+	if _, err := client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: missingVolumeID}); err != nil {
+		t.Fatalf("VolumeUnmount missing volume should be idempotent success, got: %v", err)
+	}
+
+	_, err := client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: missingVolumeID})
+	if err == nil {
+		t.Fatalf("VolumeMount missing volume should fail")
+	}
+	if !strings.Contains(err.Error(), "not found on disk") {
+		t.Fatalf("VolumeMount missing volume error mismatch: %v", err)
+	}
+
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	_, err = client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{
+		VolumeId:    volumeID,
+		Replication: "000",
+	})
+	if err == nil {
+		t.Fatalf("AllocateVolume duplicate should fail")
+	}
+	if !strings.Contains(strings.ToLower(err.Error()), "already exists") {
+		t.Fatalf("AllocateVolume duplicate error mismatch: %v", err)
+	}
+
+	if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil {
+		t.Fatalf("VolumeUnmount existing volume failed: %v", err)
+	}
+	if _, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID}); err != nil {
+		t.Fatalf("VolumeUnmount already-unmounted volume should be idempotent success, got: %v", err)
+	}
+	if _, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID}); err != nil {
+		t.Fatalf("VolumeMount remount failed: %v", err)
+	}
+}
+
+// Evacuating a server in maintenance mode ends each move by deleting the source
+// volume, so VolumeDelete must stay available in that mode (issue #11066).
+func TestMaintenanceModeAllowsVolumeDelete(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(15)
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	framework.EnableMaintenanceMode(t, ctx, client)
+
+	if _, err := client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{VolumeId: volumeID, OnlyEmpty: true}); err != nil {
+		t.Fatalf("VolumeDelete should succeed in maintenance mode, got: %v", err)
+	}
+
+	_, err := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err == nil {
+		t.Fatalf("VolumeStatus should fail once the volume is deleted")
+	}
+}

test/volume_server/grpc/admin_readonly_collection_test.go

@@ -0,0 +1,191 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestVolumeMarkReadonlyAndWritableLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(72)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
+		VolumeId: volumeID,
+		Persist:  false,
+	})
+	if err != nil {
+		t.Fatalf("VolumeMarkReadonly failed: %v", err)
+	}
+
+	readOnlyStatus, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus after readonly failed: %v", err)
+	}
+	if !readOnlyStatus.GetIsReadOnly() {
+		t.Fatalf("VolumeStatus expected readonly=true after VolumeMarkReadonly")
+	}
+
+	_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeMarkWritable failed: %v", err)
+	}
+
+	writableStatus, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus after writable failed: %v", err)
+	}
+	if writableStatus.GetIsReadOnly() {
+		t.Fatalf("VolumeStatus expected readonly=false after VolumeMarkWritable")
+	}
+}
+
+func TestVolumeMarkReadonlyPersistTrue(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(74)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
+		VolumeId: volumeID,
+		Persist:  true,
+	})
+	if err != nil {
+		t.Fatalf("VolumeMarkReadonly persist=true failed: %v", err)
+	}
+
+	statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus after persist readonly failed: %v", err)
+	}
+	if !statusResp.GetIsReadOnly() {
+		t.Fatalf("VolumeStatus expected readonly=true after persist readonly")
+	}
+}
+
+func TestVolumeMarkReadonlyWritableErrorPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(75)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	// non-existent volumes should return "not found"
+	_, err := grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 98771, Persist: true})
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeMarkReadonly missing-volume error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 98772})
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeMarkWritable missing-volume error mismatch: %v", err)
+	}
+
+	// enter maintenance mode
+	framework.EnableMaintenanceMode(t, ctx, grpcClient)
+
+	// marking readonly only restricts the server further and is the first step
+	// of evacuating it, so maintenance mode lets it through (issue #11066)
+	_, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: volumeID, Persist: true})
+	if err != nil {
+		t.Fatalf("VolumeMarkReadonly should succeed in maintenance mode, got: %v", err)
+	}
+	statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus after readonly in maintenance failed: %v", err)
+	}
+	if !statusResp.GetIsReadOnly() {
+		t.Fatalf("VolumeStatus expected readonly=true after VolumeMarkReadonly in maintenance mode")
+	}
+
+	// reopening a volume for writes is still refused on a read-only server
+	_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: volumeID})
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeMarkWritable maintenance error mismatch: %v", err)
+	}
+
+	// non-existent volume in maintenance mode should still return "not found"
+	_, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: 98773, Persist: true})
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeMarkReadonly missing-volume in maintenance error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: 98774})
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeMarkWritable missing-volume in maintenance error mismatch: %v", err)
+	}
+}
+
+func TestDeleteCollectionRemovesVolumeAndIsIdempotent(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(73)
+	const collection = "it-delete-collection"
+
+	framework.AllocateVolume(t, grpcClient, volumeID, collection)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus before DeleteCollection failed: %v", err)
+	}
+
+	_, err = grpcClient.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{Collection: collection})
+	if err != nil {
+		t.Fatalf("DeleteCollection existing collection failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err == nil {
+		t.Fatalf("VolumeStatus should fail after collection delete")
+	}
+	if !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("VolumeStatus after DeleteCollection error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.DeleteCollection(ctx, &volume_server_pb.DeleteCollectionRequest{Collection: collection})
+	if err != nil {
+		t.Fatalf("DeleteCollection idempotent retry failed: %v", err)
+	}
+}

test/volume_server/grpc/batch_delete_test.go

@@ -0,0 +1,267 @@
+package volume_server_grpc_test
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestBatchDeleteInvalidFidAndMaintenanceMode(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{"bad-fid"}})
+	if err != nil {
+		t.Fatalf("BatchDelete invalid fid should return response, got error: %v", err)
+	}
+	if len(resp.GetResults()) != 1 {
+		t.Fatalf("expected one batch delete result, got %d", len(resp.GetResults()))
+	}
+	if got := resp.GetResults()[0].GetStatus(); got != 400 {
+		t.Fatalf("invalid fid expected status 400, got %d", got)
+	}
+
+	stateResp, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	_, err = client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{"1,1234567890ab"}})
+	if err == nil {
+		t.Fatalf("BatchDelete should fail when maintenance mode is enabled")
+	}
+	if !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("expected maintenance mode error, got: %v", err)
+	}
+}
+
+func TestBatchDeleteCookieMismatchAndSkipCheck(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(31)
+	const needleID = uint64(900001)
+	const correctCookie = uint32(0x1122AABB)
+	const wrongCookie = uint32(0x1122AABC)
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, correctCookie)
+	uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), fid, []byte("batch-delete-cookie-check"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	wrongCookieFid := framework.NewFileID(volumeID, needleID, wrongCookie)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	mismatchResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
+		FileIds:         []string{wrongCookieFid},
+		SkipCookieCheck: false,
+	})
+	if err != nil {
+		t.Fatalf("BatchDelete with cookie check failed: %v", err)
+	}
+	if len(mismatchResp.GetResults()) != 1 {
+		t.Fatalf("BatchDelete cookie mismatch expected 1 result, got %d", len(mismatchResp.GetResults()))
+	}
+	if mismatchResp.GetResults()[0].GetStatus() != http.StatusBadRequest {
+		t.Fatalf("BatchDelete cookie mismatch expected status 400, got %d", mismatchResp.GetResults()[0].GetStatus())
+	}
+
+	skipCheckResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
+		FileIds:         []string{wrongCookieFid},
+		SkipCookieCheck: true,
+	})
+	if err != nil {
+		t.Fatalf("BatchDelete skip cookie check failed: %v", err)
+	}
+	if len(skipCheckResp.GetResults()) != 1 {
+		t.Fatalf("BatchDelete skip check expected 1 result, got %d", len(skipCheckResp.GetResults()))
+	}
+	if skipCheckResp.GetResults()[0].GetStatus() != http.StatusAccepted {
+		t.Fatalf("BatchDelete skip check expected status 202, got %d", skipCheckResp.GetResults()[0].GetStatus())
+	}
+
+	readAfterDelete := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fid)
+	_ = framework.ReadAllAndClose(t, readAfterDelete)
+	if readAfterDelete.StatusCode != http.StatusNotFound {
+		t.Fatalf("read after skip-check batch delete expected 404, got %d", readAfterDelete.StatusCode)
+	}
+}
+
+func TestBatchDeleteMixedStatusesAndMismatchContinuesProcessing(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(32)
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	const needleA = uint64(910001)
+	const needleB = uint64(910002)
+	const needleC = uint64(910003)
+	const cookieA = uint32(0x11111111)
+	const cookieB = uint32(0x22222222)
+	const cookieC = uint32(0x33333333)
+
+	httpClient := framework.NewHTTPClient()
+	fidA := framework.NewFileID(volumeID, needleA, cookieA)
+	fidB := framework.NewFileID(volumeID, needleB, cookieB)
+	fidC := framework.NewFileID(volumeID, needleC, cookieC)
+
+	for _, tc := range []struct {
+		fid  string
+		body string
+	}{
+		{fid: fidA, body: "batch-delete-mixed-a"},
+		{fid: fidB, body: "batch-delete-mixed-b"},
+		{fid: fidC, body: "batch-delete-mixed-c"},
+	} {
+		uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), tc.fid, []byte(tc.body))
+		_ = framework.ReadAllAndClose(t, uploadResp)
+		if uploadResp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s expected 201, got %d", tc.fid, uploadResp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	missingFid := framework.NewFileID(volumeID, 919999, 0x44444444)
+	mixedResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
+		FileIds: []string{"bad-fid", fidA, missingFid},
+	})
+	if err != nil {
+		t.Fatalf("BatchDelete mixed status request failed: %v", err)
+	}
+	if len(mixedResp.GetResults()) != 3 {
+		t.Fatalf("BatchDelete mixed status expected 3 results, got %d", len(mixedResp.GetResults()))
+	}
+	if mixedResp.GetResults()[0].GetStatus() != http.StatusBadRequest {
+		t.Fatalf("BatchDelete mixed result[0] expected 400, got %d", mixedResp.GetResults()[0].GetStatus())
+	}
+	if mixedResp.GetResults()[1].GetStatus() != http.StatusAccepted {
+		t.Fatalf("BatchDelete mixed result[1] expected 202, got %d", mixedResp.GetResults()[1].GetStatus())
+	}
+	if mixedResp.GetResults()[2].GetStatus() != http.StatusNotFound {
+		t.Fatalf("BatchDelete mixed result[2] expected 404, got %d", mixedResp.GetResults()[2].GetStatus())
+	}
+
+	readDeletedA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidA)
+	_ = framework.ReadAllAndClose(t, readDeletedA)
+	if readDeletedA.StatusCode != http.StatusNotFound {
+		t.Fatalf("fidA should be deleted after batch delete, got status %d", readDeletedA.StatusCode)
+	}
+
+	wrongCookieB := framework.NewFileID(volumeID, needleB, cookieB+1)
+	mismatchContinueResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
+		FileIds: []string{wrongCookieB, fidC},
+	})
+	if err != nil {
+		t.Fatalf("BatchDelete mismatch-continue request failed: %v", err)
+	}
+	if len(mismatchContinueResp.GetResults()) != 2 {
+		t.Fatalf("BatchDelete mismatch-continue expected 2 results, got %d", len(mismatchContinueResp.GetResults()))
+	}
+	if mismatchContinueResp.GetResults()[0].GetStatus() != http.StatusBadRequest {
+		t.Fatalf("BatchDelete mismatch-continue result[0] expected 400, got %d", mismatchContinueResp.GetResults()[0].GetStatus())
+	}
+	if mismatchContinueResp.GetResults()[1].GetStatus() != http.StatusAccepted {
+		t.Fatalf("BatchDelete mismatch-continue result[1] expected 202, got %d", mismatchContinueResp.GetResults()[1].GetStatus())
+	}
+
+	readB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB)
+	_ = framework.ReadAllAndClose(t, readB)
+	if readB.StatusCode != http.StatusOK {
+		t.Fatalf("fidB should remain after cookie mismatch, got %d", readB.StatusCode)
+	}
+
+	readC := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidC)
+	_ = framework.ReadAllAndClose(t, readC)
+	if readC.StatusCode != http.StatusNotFound {
+		t.Fatalf("fidC should be deleted when processing continues past a mismatch, got %d", readC.StatusCode)
+	}
+}
+
+func TestBatchDeleteRejectsChunkManifestNeedles(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(33)
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 920001, 0x5555AAAA)
+	req, err := http.NewRequest(http.MethodPost, cluster.VolumeAdminURL()+"/"+fid+"?cm=true", bytes.NewReader([]byte("manifest-placeholder-payload")))
+	if err != nil {
+		t.Fatalf("create chunk manifest upload request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/octet-stream")
+	uploadResp := framework.DoRequest(t, httpClient, req)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("chunk manifest upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{fid}})
+	if err != nil {
+		t.Fatalf("BatchDelete chunk manifest should return response, got grpc error: %v", err)
+	}
+	if len(resp.GetResults()) != 1 {
+		t.Fatalf("BatchDelete chunk manifest expected one result, got %d", len(resp.GetResults()))
+	}
+	if resp.GetResults()[0].GetStatus() != http.StatusNotAcceptable {
+		t.Fatalf("BatchDelete chunk manifest expected status 406, got %d", resp.GetResults()[0].GetStatus())
+	}
+	if !strings.Contains(resp.GetResults()[0].GetError(), "ChunkManifest") {
+		t.Fatalf("BatchDelete chunk manifest expected error mentioning ChunkManifest, got %q", resp.GetResults()[0].GetError())
+	}
+
+	readResp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fid)
+	_ = framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusOK {
+		t.Fatalf("chunk manifest should not be deleted by BatchDelete reject path, got %d", readResp.StatusCode)
+	}
+}

test/volume_server/grpc/copy_receive_variants_test.go

@@ -0,0 +1,490 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"io"
+	"math"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestVolumeIncrementalCopyDataAndNoDataPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(91)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 770001, 0x1122AABB)
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("incremental-copy-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	dataStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{
+		VolumeId: volumeID,
+		SinceNs:  0,
+	})
+	if err != nil {
+		t.Fatalf("VolumeIncrementalCopy start failed: %v", err)
+	}
+
+	totalBytes := 0
+	for {
+		msg, recvErr := dataStream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("VolumeIncrementalCopy recv failed: %v", recvErr)
+		}
+		totalBytes += len(msg.GetFileContent())
+	}
+	if totalBytes == 0 {
+		t.Fatalf("VolumeIncrementalCopy expected streamed bytes for since_ns=0")
+	}
+
+	noDataStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{
+		VolumeId: volumeID,
+		SinceNs:  math.MaxUint64,
+	})
+	if err != nil {
+		t.Fatalf("VolumeIncrementalCopy no-data start failed: %v", err)
+	}
+	_, err = noDataStream.Recv()
+	if err != io.EOF {
+		t.Fatalf("VolumeIncrementalCopy no-data expected EOF, got: %v", err)
+	}
+}
+
+func TestCopyFileIgnoreNotFoundAndStopOffsetZeroPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(92)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	missingNoIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:                 volumeID,
+		Ext:                      ".missing",
+		CompactionRevision:       math.MaxUint32,
+		StopOffset:               1,
+		IgnoreSourceFileNotFound: false,
+	})
+	if err == nil {
+		_, err = missingNoIgnore.Recv()
+	}
+	if err == nil {
+		t.Fatalf("CopyFile should fail for missing source file when ignore_source_file_not_found=false")
+	}
+
+	missingIgnored, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:                 volumeID,
+		Ext:                      ".missing",
+		CompactionRevision:       math.MaxUint32,
+		StopOffset:               1,
+		IgnoreSourceFileNotFound: true,
+	})
+	if err != nil {
+		t.Fatalf("CopyFile ignore-not-found start failed: %v", err)
+	}
+	_, err = missingIgnored.Recv()
+	if err != io.EOF {
+		t.Fatalf("CopyFile ignore-not-found expected EOF, got: %v", err)
+	}
+
+	stopZeroStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:                 volumeID,
+		Ext:                      ".missing",
+		CompactionRevision:       math.MaxUint32,
+		StopOffset:               0,
+		IgnoreSourceFileNotFound: false,
+	})
+	if err != nil {
+		t.Fatalf("CopyFile stop_offset=0 start failed: %v", err)
+	}
+	_, err = stopZeroStream.Recv()
+	if err != io.EOF {
+		t.Fatalf("CopyFile stop_offset=0 expected EOF, got: %v", err)
+	}
+}
+
+func TestCopyFileStopOffsetZeroExistingFileSendsMetadata(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(93)
+	const needleID = uint64(770101)
+	const cookie = uint32(0x1234ABCD)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	uploadResp := framework.UploadBytes(
+		t,
+		client,
+		clusterHarness.VolumeAdminURL(),
+		framework.NewFileID(volumeID, needleID, cookie),
+		[]byte("copy-file-stop-zero"),
+	)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:                 volumeID,
+		Ext:                      ".dat",
+		CompactionRevision:       math.MaxUint32,
+		StopOffset:               0,
+		IgnoreSourceFileNotFound: false,
+	})
+	if err != nil {
+		t.Fatalf("CopyFile stop_offset=0 existing file start failed: %v", err)
+	}
+
+	msg, err := stream.Recv()
+	if err != nil {
+		t.Fatalf("CopyFile stop_offset=0 existing file recv failed: %v", err)
+	}
+	if len(msg.GetFileContent()) != 0 {
+		t.Fatalf("CopyFile stop_offset=0 existing file should not send content, got %d bytes", len(msg.GetFileContent()))
+	}
+	if msg.GetModifiedTsNs() == 0 {
+		t.Fatalf("CopyFile stop_offset=0 existing file expected non-zero ModifiedTsNs")
+	}
+
+	_, err = stream.Recv()
+	if err != io.EOF {
+		t.Fatalf("CopyFile stop_offset=0 existing file expected EOF after metadata frame, got: %v", err)
+	}
+}
+
+func TestCopyFileCompactionRevisionMismatch(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(94)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:           volumeID,
+		Ext:                ".idx",
+		CompactionRevision: 1, // fresh volume starts at revision 0
+		StopOffset:         1,
+	})
+	if err == nil {
+		_, err = stream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "is compacted") {
+		t.Fatalf("CopyFile compaction mismatch error mismatch: %v", err)
+	}
+}
+
+func TestReceiveFileProtocolViolationResponses(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	contentFirstStream, err := grpcClient.ReceiveFile(ctx)
+	if err != nil {
+		t.Fatalf("ReceiveFile stream create failed: %v", err)
+	}
+	if err = contentFirstStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_FileContent{
+			FileContent: []byte("content-before-info"),
+		},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send content-first failed: %v", err)
+	}
+	contentFirstResp, err := contentFirstStream.CloseAndRecv()
+	if err != nil {
+		t.Fatalf("ReceiveFile content-first close failed: %v", err)
+	}
+	if !strings.Contains(contentFirstResp.GetError(), "file info must be sent first") {
+		t.Fatalf("ReceiveFile content-first response mismatch: %+v", contentFirstResp)
+	}
+
+	unknownTypeStream, err := grpcClient.ReceiveFile(ctx)
+	if err != nil {
+		t.Fatalf("ReceiveFile stream create for unknown-type failed: %v", err)
+	}
+	if err = unknownTypeStream.Send(&volume_server_pb.ReceiveFileRequest{}); err != nil {
+		t.Fatalf("ReceiveFile send unknown-type request failed: %v", err)
+	}
+	unknownTypeResp, err := unknownTypeStream.CloseAndRecv()
+	if err != nil {
+		t.Fatalf("ReceiveFile unknown-type close failed: %v", err)
+	}
+	if !strings.Contains(unknownTypeResp.GetError(), "unknown message type") {
+		t.Fatalf("ReceiveFile unknown-type response mismatch: %+v", unknownTypeResp)
+	}
+}
+
+func TestReceiveFileSuccessForRegularVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(95)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	payloadA := []byte("receive-file-chunk-a:")
+	payloadB := []byte("receive-file-chunk-b")
+	expected := append(append([]byte{}, payloadA...), payloadB...)
+
+	receiveStream, err := grpcClient.ReceiveFile(ctx)
+	if err != nil {
+		t.Fatalf("ReceiveFile stream create failed: %v", err)
+	}
+
+	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_Info{
+			Info: &volume_server_pb.ReceiveFileInfo{
+				VolumeId:   volumeID,
+				Ext:        ".tmprecv",
+				Collection: "",
+				IsEcVolume: false,
+				FileSize:   uint64(len(expected)),
+			},
+		},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send info failed: %v", err)
+	}
+	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadA},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send payloadA failed: %v", err)
+	}
+	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadB},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send payloadB failed: %v", err)
+	}
+
+	resp, err := receiveStream.CloseAndRecv()
+	if err != nil {
+		t.Fatalf("ReceiveFile close failed: %v", err)
+	}
+	if resp.GetError() != "" {
+		t.Fatalf("ReceiveFile unexpected error response: %+v", resp)
+	}
+	if resp.GetBytesWritten() != uint64(len(expected)) {
+		t.Fatalf("ReceiveFile bytes_written mismatch: got %d want %d", resp.GetBytesWritten(), len(expected))
+	}
+
+	copyStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:           volumeID,
+		Ext:                ".tmprecv",
+		CompactionRevision: math.MaxUint32,
+		StopOffset:         uint64(len(expected)),
+	})
+	if err != nil {
+		t.Fatalf("CopyFile for received data start failed: %v", err)
+	}
+
+	var copied []byte
+	for {
+		msg, recvErr := copyStream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("CopyFile for received data recv failed: %v", recvErr)
+		}
+		copied = append(copied, msg.GetFileContent()...)
+	}
+
+	if string(copied) != string(expected) {
+		t.Fatalf("received file data mismatch: got %q want %q", string(copied), string(expected))
+	}
+}
+
+func TestReceiveFileSuccessForEcVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	const volumeID = uint32(96)
+	const collection = "ec-receive-success"
+	const ext = ".ec00"
+
+	payloadA := []byte("receive-ec-file-chunk-a:")
+	payloadB := []byte("receive-ec-file-chunk-b")
+	expected := append(append([]byte{}, payloadA...), payloadB...)
+
+	receiveStream, err := grpcClient.ReceiveFile(ctx)
+	if err != nil {
+		t.Fatalf("ReceiveFile stream create failed: %v", err)
+	}
+
+	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_Info{
+			Info: &volume_server_pb.ReceiveFileInfo{
+				VolumeId:   volumeID,
+				Ext:        ext,
+				Collection: collection,
+				IsEcVolume: true,
+				ShardId:    0,
+				FileSize:   uint64(len(expected)),
+			},
+		},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send EC info failed: %v", err)
+	}
+	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadA},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send EC payloadA failed: %v", err)
+	}
+	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payloadB},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send EC payloadB failed: %v", err)
+	}
+
+	resp, err := receiveStream.CloseAndRecv()
+	if err != nil {
+		t.Fatalf("ReceiveFile EC close failed: %v", err)
+	}
+	if resp.GetError() != "" {
+		t.Fatalf("ReceiveFile EC unexpected error response: %+v", resp)
+	}
+	if resp.GetBytesWritten() != uint64(len(expected)) {
+		t.Fatalf("ReceiveFile EC bytes_written mismatch: got %d want %d", resp.GetBytesWritten(), len(expected))
+	}
+
+	copyStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:           volumeID,
+		Collection:         collection,
+		IsEcVolume:         true,
+		Ext:                ext,
+		CompactionRevision: math.MaxUint32,
+		StopOffset:         uint64(len(expected)),
+	})
+	if err != nil {
+		t.Fatalf("CopyFile for received EC data start failed: %v", err)
+	}
+
+	var copied []byte
+	for {
+		msg, recvErr := copyStream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("CopyFile for received EC data recv failed: %v", recvErr)
+		}
+		copied = append(copied, msg.GetFileContent()...)
+	}
+
+	if string(copied) != string(expected) {
+		t.Fatalf("received EC file data mismatch: got %q want %q", string(copied), string(expected))
+	}
+}
+
+func TestCopyFileEcVolumeIgnoreMissingSourcePaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	streamNoIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:                 99601,
+		Collection:               "ec-copy-missing",
+		IsEcVolume:               true,
+		Ext:                      ".ec00",
+		CompactionRevision:       math.MaxUint32,
+		StopOffset:               1,
+		IgnoreSourceFileNotFound: false,
+	})
+	if err == nil {
+		_, err = streamNoIgnore.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found ec volume id") {
+		t.Fatalf("CopyFile EC missing source error mismatch: %v", err)
+	}
+
+	streamIgnore, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:                 99602,
+		Collection:               "ec-copy-missing",
+		IsEcVolume:               true,
+		Ext:                      ".ec00",
+		CompactionRevision:       math.MaxUint32,
+		StopOffset:               1,
+		IgnoreSourceFileNotFound: true,
+	})
+	if err != nil {
+		t.Fatalf("CopyFile EC ignore-missing start failed: %v", err)
+	}
+	_, err = streamIgnore.Recv()
+	if err != io.EOF {
+		t.Fatalf("CopyFile EC ignore-missing expected EOF, got: %v", err)
+	}
+}

test/volume_server/grpc/copy_sync_test.go

@@ -0,0 +1,395 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"io"
+	"math/rand"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestVolumeSyncStatusAndReadVolumeFileStatus(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	httpClient := framework.NewHTTPClient()
+	const volumeID = uint32(41)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+	fid := framework.NewFileID(volumeID, 1, 0x11112222)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("sync-status-payload"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	syncResp, err := grpcClient.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeSyncStatus failed: %v", err)
+	}
+	if syncResp.GetVolumeId() != volumeID {
+		t.Fatalf("VolumeSyncStatus volume id mismatch: got %d want %d", syncResp.GetVolumeId(), volumeID)
+	}
+
+	statusResp, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus failed: %v", err)
+	}
+	if statusResp.GetVolumeId() != volumeID {
+		t.Fatalf("ReadVolumeFileStatus volume id mismatch: got %d want %d", statusResp.GetVolumeId(), volumeID)
+	}
+	if statusResp.GetVersion() == 0 {
+		t.Fatalf("ReadVolumeFileStatus expected non-zero version")
+	}
+	if syncResp.GetTailOffset() == 0 {
+		t.Fatalf("VolumeSyncStatus expected non-zero tail offset after upload")
+	}
+	if syncResp.GetTailOffset() != statusResp.GetDatFileSize() {
+		t.Fatalf("VolumeSyncStatus tail offset mismatch: got %d want %d", syncResp.GetTailOffset(), statusResp.GetDatFileSize())
+	}
+}
+
+func TestCopyAndStreamMethodsMissingVolumePaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeSyncStatus(ctx, &volume_server_pb.VolumeSyncStatusRequest{VolumeId: 98761})
+	if err == nil {
+		t.Fatalf("VolumeSyncStatus should fail for missing volume")
+	}
+
+	incrementalStream, err := grpcClient.VolumeIncrementalCopy(ctx, &volume_server_pb.VolumeIncrementalCopyRequest{VolumeId: 98762, SinceNs: 0})
+	if err == nil {
+		_, err = incrementalStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("VolumeIncrementalCopy missing-volume error mismatch: %v", err)
+	}
+
+	readAllStream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{98763}})
+	if err == nil {
+		_, err = readAllStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("ReadAllNeedles missing-volume error mismatch: %v", err)
+	}
+
+	copyFileStream, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{VolumeId: 98764, Ext: ".dat", StopOffset: 1})
+	if err == nil {
+		_, err = copyFileStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("CopyFile missing-volume error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: 98765})
+	if err == nil || !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("ReadVolumeFileStatus missing-volume error mismatch: %v", err)
+	}
+}
+
+func TestVolumeCopyAndReceiveFileMaintenanceRejection(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	copyStream, err := grpcClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{VolumeId: 1, SourceDataNode: "127.0.0.1:1234"})
+	if err == nil {
+		_, err = copyStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeCopy maintenance error mismatch: %v", err)
+	}
+
+	receiveClient, err := grpcClient.ReceiveFile(ctx)
+	if err != nil {
+		t.Fatalf("ReceiveFile client creation failed: %v", err)
+	}
+	_ = receiveClient.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_Info{
+			Info: &volume_server_pb.ReceiveFileInfo{VolumeId: 1, Ext: ".dat"},
+		},
+	})
+	_, err = receiveClient.CloseAndRecv()
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("ReceiveFile maintenance error mismatch: %v", err)
+	}
+}
+
+func TestVolumeCopySuccessFromPeerAndMountsDestination(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
+	sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer sourceConn.Close()
+	destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
+	defer destConn.Close()
+
+	const volumeID = uint32(42)
+	framework.AllocateVolume(t, sourceClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 880001, 0x12345678)
+	payload := []byte("volume-copy-success-payload")
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload to source expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		Collection:     "",
+		SourceDataNode: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1],
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy start failed: %v", err)
+	}
+
+	sawFinalAppendTimestamp := false
+	for {
+		msg, recvErr := copyStream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("VolumeCopy recv failed: %v", recvErr)
+		}
+		if msg.GetLastAppendAtNs() > 0 {
+			sawFinalAppendTimestamp = true
+		}
+	}
+	if !sawFinalAppendTimestamp {
+		t.Fatalf("VolumeCopy expected final response with last_append_at_ns")
+	}
+
+	destReadResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid)
+	destReadBody := framework.ReadAllAndClose(t, destReadResp)
+	if destReadResp.StatusCode != http.StatusOK {
+		t.Fatalf("read from copied destination expected 200, got %d", destReadResp.StatusCode)
+	}
+	if string(destReadBody) != string(payload) {
+		t.Fatalf("destination copied payload mismatch: got %q want %q", string(destReadBody), string(payload))
+	}
+}
+
+func TestVolumeCopyOverwritesExistingDestinationVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
+	sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer sourceConn.Close()
+	destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
+	defer destConn.Close()
+
+	const volumeID = uint32(43)
+	framework.AllocateVolume(t, sourceClient, volumeID, "")
+	framework.AllocateVolume(t, destClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 880002, 0x23456789)
+	sourcePayload := []byte("volume-copy-overwrite-source")
+	destPayload := []byte("volume-copy-overwrite-destination-old")
+
+	sourceUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, sourcePayload)
+	_ = framework.ReadAllAndClose(t, sourceUploadResp)
+	if sourceUploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload to source expected 201, got %d", sourceUploadResp.StatusCode)
+	}
+
+	destUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid, destPayload)
+	_ = framework.ReadAllAndClose(t, destUploadResp)
+	if destUploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload to destination expected 201, got %d", destUploadResp.StatusCode)
+	}
+
+	destReadBeforeResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid)
+	destReadBeforeBody := framework.ReadAllAndClose(t, destReadBeforeResp)
+	if destReadBeforeResp.StatusCode != http.StatusOK {
+		t.Fatalf("destination pre-copy read expected 200, got %d", destReadBeforeResp.StatusCode)
+	}
+	if string(destReadBeforeBody) != string(destPayload) {
+		t.Fatalf("destination pre-copy payload mismatch: got %q want %q", string(destReadBeforeBody), string(destPayload))
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		Collection:     "",
+		SourceDataNode: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1],
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy overwrite start failed: %v", err)
+	}
+
+	sawFinalAppendTimestamp := false
+	for {
+		msg, recvErr := copyStream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("VolumeCopy overwrite recv failed: %v", recvErr)
+		}
+		if msg.GetLastAppendAtNs() > 0 {
+			sawFinalAppendTimestamp = true
+		}
+	}
+	if !sawFinalAppendTimestamp {
+		t.Fatalf("VolumeCopy overwrite expected final response with last_append_at_ns")
+	}
+
+	destReadAfterResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid)
+	destReadAfterBody := framework.ReadAllAndClose(t, destReadAfterResp)
+	if destReadAfterResp.StatusCode != http.StatusOK {
+		t.Fatalf("destination post-copy read expected 200, got %d", destReadAfterResp.StatusCode)
+	}
+	if string(destReadAfterBody) != string(sourcePayload) {
+		t.Fatalf("destination post-copy payload mismatch: got %q want %q", string(destReadAfterBody), string(sourcePayload))
+	}
+}
+
+// A VolumeCopy whose caller goes away must not leave a mounted volume behind on
+// the destination. weed-admin's batch balance routinely starts far more copies
+// than it finishes, and each abandoned copy that still mounts costs the
+// destination a volume it was never asked to hold: a per-volume index cache that
+// is never reclaimed, and — under replication=000 — a second writable copy of a
+// volume id that two writers can diverge.
+//
+// The copy is throttled so it is demonstrably still in flight when the caller
+// cancels; the assertion before the cancel keeps the test from passing
+// vacuously if the throttle ever stops biting.
+func TestVolumeCopyCancelledByCallerDoesNotMountDestination(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
+	sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer sourceConn.Close()
+	destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
+	defer destConn.Close()
+
+	const volumeID = uint32(44)
+	const payloadMiB = 192
+	framework.AllocateVolume(t, sourceClient, volumeID, "")
+
+	// The fixture size is load-bearing twice over; do not shrink it to save CI
+	// time without reading both reasons.
+	//
+	// First, the copy has to still be running when the caller cancels, and the
+	// only lever for that is IoBytePerSecond. It is a no-op below ~100ms of wall
+	// clock — the throttler never re-checks within its first window — and on a
+	// tmpfs loopback cluster 64 MiB copies in ~110ms.
+	//
+	// Second, 192 MiB is above the 128 MiB progress-report interval, which is
+	// what lets this test be green on Go: a failing stream.Send from that report
+	// is Go's only abort signal. Measured below the interval (120 MiB), a Go
+	// destination mounts the abandoned copy too, so a smaller fixture would fail
+	// the Go leg without any Rust change.
+	//
+	// The bytes are pseudo-random because the volume server gzips compressible
+	// uploads; a repeating payload lands on disk as a few KB and the throttle
+	// never bites.
+	httpClient := framework.NewHTTPClient()
+	chunk := make([]byte, 1024*1024)
+	if _, err := rand.New(rand.NewSource(11186)).Read(chunk); err != nil {
+		t.Fatalf("generate payload: %v", err)
+	}
+	for i := 0; i < payloadMiB; i++ {
+		fid := framework.NewFileID(volumeID, uint64(880100+i), 0x3456789a)
+		uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, chunk)
+		_ = framework.ReadAllAndClose(t, uploadResp)
+		if uploadResp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %d to source expected 201, got %d", i, uploadResp.StatusCode)
+		}
+	}
+
+	// 192 MiB at 16 MiB/s is about twelve seconds of copying.
+	copyCtx, cancelCopy := context.WithCancel(context.Background())
+	defer cancelCopy()
+	_, err := destClient.VolumeCopy(copyCtx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:        volumeID,
+		Collection:      "",
+		SourceDataNode:  clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1],
+		IoBytePerSecond: 16 * 1024 * 1024,
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy start failed: %v", err)
+	}
+
+	time.Sleep(2 * time.Second)
+
+	// Premise: the copy must still be running when we cancel. If the
+	// destination has already mounted, the throttle no longer bites and the
+	// rest of this test would prove nothing.
+	premiseCtx, cancelPremise := context.WithTimeout(context.Background(), 5*time.Second)
+	_, premiseErr := destClient.ReadVolumeFileStatus(premiseCtx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
+	cancelPremise()
+	if premiseErr == nil {
+		t.Fatalf("copy of volume %d already completed before the cancel; raise payloadMiB or lower IoBytePerSecond", volumeID)
+	}
+
+	cancelCopy()
+
+	// Well past the ~10s the unabandoned copy would have needed: the volume
+	// must never appear on the destination.
+	deadline := time.Now().Add(25 * time.Second)
+	for time.Now().Before(deadline) {
+		pollCtx, cancelPoll := context.WithTimeout(context.Background(), 5*time.Second)
+		_, statusErr := destClient.ReadVolumeFileStatus(pollCtx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
+		cancelPoll()
+		if statusErr == nil {
+			t.Fatalf("destination mounted volume %d after its VolumeCopy caller cancelled", volumeID)
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+}

test/volume_server/grpc/data_rw_test.go

@@ -0,0 +1,146 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestReadNeedleBlobAndMetaMissingVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{
+		VolumeId: 99111,
+		Offset:   0,
+		Size:     16,
+	})
+	if err == nil {
+		t.Fatalf("ReadNeedleBlob should fail for missing volume")
+	}
+	if !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("ReadNeedleBlob missing volume error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
+		VolumeId: 99112,
+		NeedleId: 1,
+		Offset:   0,
+		Size:     16,
+	})
+	if err == nil {
+		t.Fatalf("ReadNeedleMeta should fail for missing volume")
+	}
+	if !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("ReadNeedleMeta missing volume error mismatch: %v", err)
+	}
+}
+
+func TestWriteNeedleBlobMaintenanceAndMissingVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
+		VolumeId:   99113,
+		NeedleId:   1,
+		NeedleBlob: []byte("abc"),
+		Size:       3,
+	})
+	if err == nil {
+		t.Fatalf("WriteNeedleBlob should fail for missing volume")
+	}
+	if !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("WriteNeedleBlob missing volume error mismatch: %v", err)
+	}
+
+	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	_, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
+		VolumeId:   1,
+		NeedleId:   2,
+		NeedleBlob: []byte("def"),
+		Size:       3,
+	})
+	if err == nil {
+		t.Fatalf("WriteNeedleBlob should fail in maintenance mode")
+	}
+	if !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("WriteNeedleBlob maintenance mode error mismatch: %v", err)
+	}
+}
+
+func TestReadNeedleBlobAndMetaInvalidOffsets(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(92)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 880001, 0xCCDD1122)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("invalid-offset-check"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{
+		VolumeId: volumeID,
+		Offset:   1 << 40,
+		Size:     64,
+	})
+	if err == nil {
+		t.Fatalf("ReadNeedleBlob should fail for invalid offset")
+	}
+	if !strings.Contains(strings.ToLower(err.Error()), "read needle blob") {
+		t.Fatalf("ReadNeedleBlob invalid offset error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
+		VolumeId: volumeID,
+		NeedleId: 880001,
+		Offset:   1 << 40,
+		Size:     64,
+	})
+	if err == nil {
+		t.Fatalf("ReadNeedleMeta should fail for invalid offset")
+	}
+}

test/volume_server/grpc/data_stream_success_test.go

@@ -0,0 +1,428 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func TestReadWriteNeedleBlobAndMetaRoundTrip(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(83)
+	const sourceNeedleID = uint64(333333)
+	const sourceCookie = uint32(0xABCD0102)
+	const clonedNeedleID = uint64(333334)
+
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	payload := []byte("blob-roundtrip-content")
+	fid := framework.NewFileID(volumeID, sourceNeedleID, sourceCookie)
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	fileStatus, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus failed: %v", err)
+	}
+	if fileStatus.GetIdxFileSize() == 0 {
+		t.Fatalf("expected non-zero idx file size after upload")
+	}
+
+	idxBytes := copyFileBytes(t, grpcClient, &volume_server_pb.CopyFileRequest{
+		VolumeId:           volumeID,
+		Ext:                ".idx",
+		CompactionRevision: fileStatus.GetCompactionRevision(),
+		StopOffset:         fileStatus.GetIdxFileSize(),
+	})
+	offset, size := findNeedleOffsetAndSize(t, idxBytes, sourceNeedleID)
+
+	blobResp, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{
+		VolumeId: volumeID,
+		Offset:   offset,
+		Size:     size,
+	})
+	if err != nil {
+		t.Fatalf("ReadNeedleBlob failed: %v", err)
+	}
+	if len(blobResp.GetNeedleBlob()) == 0 {
+		t.Fatalf("ReadNeedleBlob returned empty blob")
+	}
+
+	metaResp, err := grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
+		VolumeId: volumeID,
+		NeedleId: sourceNeedleID,
+		Offset:   offset,
+		Size:     size,
+	})
+	if err != nil {
+		t.Fatalf("ReadNeedleMeta failed: %v", err)
+	}
+	if metaResp.GetCookie() != sourceCookie {
+		t.Fatalf("ReadNeedleMeta cookie mismatch: got %d want %d", metaResp.GetCookie(), sourceCookie)
+	}
+
+	_, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
+		VolumeId:   volumeID,
+		NeedleId:   clonedNeedleID,
+		Size:       size,
+		NeedleBlob: blobResp.GetNeedleBlob(),
+	})
+	if err != nil {
+		t.Fatalf("WriteNeedleBlob failed: %v", err)
+	}
+
+	clonedStatus, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
+		VolumeId: volumeID,
+		NeedleId: clonedNeedleID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeNeedleStatus for cloned needle failed: %v", err)
+	}
+	if clonedStatus.GetNeedleId() != sourceNeedleID {
+		t.Fatalf("cloned needle status id mismatch: got %d want %d", clonedStatus.GetNeedleId(), sourceNeedleID)
+	}
+	if clonedStatus.GetCookie() != sourceCookie {
+		t.Fatalf("cloned needle cookie mismatch: got %d want %d", clonedStatus.GetCookie(), sourceCookie)
+	}
+
+	clonedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, clonedNeedleID, sourceCookie))
+	clonedReadBody := framework.ReadAllAndClose(t, clonedReadResp)
+	if clonedReadResp.StatusCode != 200 {
+		t.Fatalf("cloned needle GET expected 200, got %d", clonedReadResp.StatusCode)
+	}
+	if string(clonedReadBody) != string(payload) {
+		t.Fatalf("cloned needle body mismatch: got %q want %q", string(clonedReadBody), string(payload))
+	}
+}
+
+func TestReadAllNeedlesStreamsUploadedRecords(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(84)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	expected := map[uint64]string{
+		444441: "read-all-needle-one",
+		444442: "read-all-needle-two",
+	}
+	for key, body := range expected {
+		resp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, key, 0xA0B0C0D0), []byte(body))
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != 201 {
+			t.Fatalf("upload for key %d expected 201, got %d", key, resp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{volumeID}})
+	if err != nil {
+		t.Fatalf("ReadAllNeedles start failed: %v", err)
+	}
+
+	seen := map[uint64]string{}
+	for {
+		msg, recvErr := stream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("ReadAllNeedles recv failed: %v", recvErr)
+		}
+		if _, wanted := expected[msg.GetNeedleId()]; wanted {
+			seen[msg.GetNeedleId()] = string(msg.GetNeedleBlob())
+		}
+	}
+
+	for key, body := range expected {
+		got, found := seen[key]
+		if !found {
+			t.Fatalf("ReadAllNeedles missing key %d in stream", key)
+		}
+		if got != body {
+			t.Fatalf("ReadAllNeedles body mismatch for key %d: got %q want %q", key, got, body)
+		}
+	}
+}
+
+func TestReadAllNeedlesExistingThenMissingVolumeAbortsStream(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const existingVolumeID = uint32(85)
+	const missingVolumeID = uint32(98585)
+	const needleID = uint64(445551)
+	framework.AllocateVolume(t, grpcClient, existingVolumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(existingVolumeID, needleID, 0xAA11BB22)
+	payload := "read-all-existing-then-missing"
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte(payload))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{
+		VolumeIds: []uint32{existingVolumeID, missingVolumeID},
+	})
+	if err != nil {
+		t.Fatalf("ReadAllNeedles start failed: %v", err)
+	}
+
+	seenUploadedNeedle := false
+	for {
+		msg, recvErr := stream.Recv()
+		if recvErr == io.EOF {
+			t.Fatalf("ReadAllNeedles expected stream error for missing volume, got EOF")
+		}
+		if recvErr != nil {
+			if !strings.Contains(recvErr.Error(), "not found volume id") {
+				t.Fatalf("ReadAllNeedles missing-volume error mismatch: %v", recvErr)
+			}
+			break
+		}
+		if msg.GetNeedleId() == needleID && string(msg.GetNeedleBlob()) == payload {
+			seenUploadedNeedle = true
+		}
+	}
+
+	if !seenUploadedNeedle {
+		t.Fatalf("ReadAllNeedles should stream entries from existing volume before missing-volume abort")
+	}
+}
+
+func TestReadAllNeedlesPreservesDatOrderAcrossOverwrite(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(86)
+	const firstNeedleID = uint64(444551)
+	const secondNeedleID = uint64(444552)
+	const firstCookie = uint32(0xAA22BB33)
+	const secondCookie = uint32(0xCC44DD55)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	uploads := []struct {
+		fid  string
+		body string
+	}{
+		{fid: framework.NewFileID(volumeID, firstNeedleID, firstCookie), body: "read-all-first"},
+		{fid: framework.NewFileID(volumeID, secondNeedleID, secondCookie), body: "read-all-second"},
+		{fid: framework.NewFileID(volumeID, firstNeedleID, firstCookie), body: "read-all-first-overwrite"},
+	}
+	for _, upload := range uploads {
+		resp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), upload.fid, []byte(upload.body))
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != 201 {
+			t.Fatalf("upload for %s expected 201, got %d", upload.fid, resp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{volumeID}})
+	if err != nil {
+		t.Fatalf("ReadAllNeedles start failed: %v", err)
+	}
+
+	var orderedIDs []uint64
+	var orderedBodies []string
+	for {
+		msg, recvErr := stream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("ReadAllNeedles recv failed: %v", recvErr)
+		}
+		orderedIDs = append(orderedIDs, msg.GetNeedleId())
+		orderedBodies = append(orderedBodies, string(msg.GetNeedleBlob()))
+	}
+
+	wantIDs := []uint64{secondNeedleID, firstNeedleID}
+	wantBodies := []string{"read-all-second", "read-all-first-overwrite"}
+	if !reflect.DeepEqual(orderedIDs, wantIDs) {
+		t.Fatalf("ReadAllNeedles order mismatch: got %v want %v", orderedIDs, wantIDs)
+	}
+	if !reflect.DeepEqual(orderedBodies, wantBodies) {
+		t.Fatalf("ReadAllNeedles bodies mismatch: got %v want %v", orderedBodies, wantBodies)
+	}
+}
+
+func TestReadNeedleMetaDeletedEntryUsesTombstoneMetadata(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(87)
+	const needleID = uint64(444661)
+	const cookie = uint32(0xAB12CD34)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	client := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("read-meta-delete"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	deleteReq, err := http.NewRequest(http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid, nil)
+	if err != nil {
+		t.Fatalf("build delete request: %v", err)
+	}
+	deleteResp := framework.DoRequest(t, client, deleteReq)
+	_ = framework.ReadAllAndClose(t, deleteResp)
+	if deleteResp.StatusCode != http.StatusAccepted {
+		t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	fileStatus, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus after delete failed: %v", err)
+	}
+
+	idxBytes := copyFileBytes(t, grpcClient, &volume_server_pb.CopyFileRequest{
+		VolumeId:           volumeID,
+		Ext:                ".idx",
+		CompactionRevision: fileStatus.GetCompactionRevision(),
+		StopOffset:         fileStatus.GetIdxFileSize(),
+	})
+	offset, size := findLastNeedleOffsetAndSize(t, idxBytes, needleID)
+	if size >= 0 {
+		t.Fatalf("expected deleted idx entry for needle %d, got size %d", needleID, size)
+	}
+
+	metaResp, err := grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
+		VolumeId: volumeID,
+		NeedleId: needleID,
+		Offset:   offset,
+		Size:     size,
+	})
+	if err != nil {
+		t.Fatalf("ReadNeedleMeta deleted-entry failed: %v", err)
+	}
+	if metaResp.GetCookie() != cookie {
+		t.Fatalf("ReadNeedleMeta deleted-entry cookie mismatch: got %d want %d", metaResp.GetCookie(), cookie)
+	}
+	if metaResp.GetAppendAtNs() == 0 {
+		t.Fatalf("ReadNeedleMeta deleted-entry expected non-zero append_at_ns")
+	}
+}
+
+func copyFileBytes(t testing.TB, grpcClient volume_server_pb.VolumeServerClient, req *volume_server_pb.CopyFileRequest) []byte {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.CopyFile(ctx, req)
+	if err != nil {
+		t.Fatalf("CopyFile start failed: %v", err)
+	}
+
+	var out []byte
+	for {
+		msg, recvErr := stream.Recv()
+		if recvErr == io.EOF {
+			return out
+		}
+		if recvErr != nil {
+			t.Fatalf("CopyFile recv failed: %v", recvErr)
+		}
+		out = append(out, msg.GetFileContent()...)
+	}
+}
+
+func findNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64) (offset int64, size int32) {
+	t.Helper()
+
+	for i := 0; i+types.NeedleMapEntrySize <= len(idxBytes); i += types.NeedleMapEntrySize {
+		key, entryOffset, entrySize := idx.IdxFileEntry(idxBytes[i : i+types.NeedleMapEntrySize])
+		if uint64(key) != needleID {
+			continue
+		}
+		if entryOffset.IsZero() || entrySize <= 0 {
+			continue
+		}
+		return entryOffset.ToActualOffset(), int32(entrySize)
+	}
+
+	t.Fatalf("needle id %d not found in idx entries", needleID)
+	return 0, 0
+}
+
+func findLastNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64) (offset int64, size int32) {
+	t.Helper()
+
+	found := false
+	for i := 0; i+types.NeedleMapEntrySize <= len(idxBytes); i += types.NeedleMapEntrySize {
+		key, entryOffset, entrySize := idx.IdxFileEntry(idxBytes[i : i+types.NeedleMapEntrySize])
+		if uint64(key) != needleID {
+			continue
+		}
+		found = true
+		offset = entryOffset.ToActualOffset()
+		size = int32(entrySize)
+	}
+
+	if !found {
+		t.Fatalf("needle id %d not found in idx entries", needleID)
+	}
+	return offset, size
+}

test/volume_server/grpc/ec_decode_ecj_test.go

@@ -0,0 +1,294 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// TestEcDecodePreservesDeletedNeedles verifies that needles deleted via
+// VolumeEcBlobDelete (recorded in .ecj) are correctly excluded from the
+// decoded volume produced by VolumeEcShardsToVolume.
+func TestEcDecodePreservesDeletedNeedles(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const (
+		volumeID = uint32(140)
+		keyA     = uint64(990020)
+		cookieA  = uint32(0xDA001122)
+		keyB     = uint64(990021)
+		cookieB  = uint32(0xDA003344)
+	)
+
+	framework.AllocateVolume(t, client, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fidA := framework.NewFileID(volumeID, keyA, cookieA)
+	fidB := framework.NewFileID(volumeID, keyB, cookieB)
+	payloadA := []byte("needle-A-should-be-deleted-after-decode")
+	payloadB := []byte("needle-B-should-survive-decode")
+
+	// Upload two needles.
+	for _, tc := range []struct {
+		fid     string
+		payload []byte
+	}{
+		{fidA, payloadA},
+		{fidB, payloadB},
+	} {
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), tc.fid, tc.payload)
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s: expected 201, got %d", tc.fid, resp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// EC encode.
+	_, err := client.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate: %v", err)
+	}
+
+	// Mount all data shards so the EC volume is usable.
+	_, err = client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId: volumeID, Collection: "",
+		ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount: %v", err)
+	}
+
+	// Delete needle A via EC path (writes to .ecj).
+	_, err = client.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId: volumeID, Collection: "",
+		FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcBlobDelete needle A: %v", err)
+	}
+
+	// Unmount the normal volume so decode writes fresh files.
+	_, err = client.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeUnmount: %v", err)
+	}
+
+	// Decode EC shards back to a normal volume.
+	_, err = client.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsToVolume: %v", err)
+	}
+
+	// Re-mount the decoded volume.
+	_, err = client.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeMount: %v", err)
+	}
+
+	// Needle A should be gone (deleted via .ecj before decode).
+	respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidA)
+	bodyA := framework.ReadAllAndClose(t, respA)
+	if respA.StatusCode >= 500 {
+		t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
+	}
+	if respA.StatusCode != http.StatusNotFound {
+		t.Fatalf("needle A should be 404 after decode, got %d", respA.StatusCode)
+	}
+
+	// Needle B should still be readable.
+	respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(), fidB)
+	bodyB := framework.ReadAllAndClose(t, respB)
+	if respB.StatusCode != http.StatusOK {
+		t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
+	}
+	if string(bodyB) != string(payloadB) {
+		t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
+	}
+}
+
+// TestEcDecodeCollectsEcjFromPeer verifies that .ecj deletion entries from a
+// peer server that contributes no new data shards are still collected during
+// decode. This is the regression test for the fix in collectEcShards that
+// always copies .ecj from every shard location.
+//
+// Scenario:
+//   - Server 0 holds all 10 data shards (decode target).
+//   - Server 1 holds a copy of shard 0 (no new shards for server 0).
+//   - A needle is deleted ONLY on server 1 (server 0's .ecj is empty).
+//   - During decode on server 0, server 1's .ecj must be collected and applied.
+func TestEcDecodeCollectsEcjFromPeer(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMultiVolumeClusterAuto(t, matrix.P1(), 2)
+	conn0, client0 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, client1 := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	const (
+		volumeID = uint32(141)
+		keyA     = uint64(990030)
+		cookieA  = uint32(0xDB001122)
+		keyB     = uint64(990031)
+		cookieB  = uint32(0xDB003344)
+	)
+
+	// Allocate and upload on server 0.
+	framework.AllocateVolume(t, client0, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fidA := framework.NewFileID(volumeID, keyA, cookieA)
+	fidB := framework.NewFileID(volumeID, keyB, cookieB)
+	payloadB := []byte("needle-B-should-survive-peer-ecj-decode")
+
+	resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA, []byte("needle-A-deleted-on-peer"))
+	_ = framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload A: expected 201, got %d", resp.StatusCode)
+	}
+	resp = framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB, payloadB)
+	_ = framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload B: expected 201, got %d", resp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// EC encode on server 0.
+	_, err := client0.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate on server 0: %v", err)
+	}
+
+	// Build the SourceDataNode address for server 0 (format: host:adminPort.grpcPort).
+	sourceDataNode := cluster.VolumeAdminAddress(0) + "." +
+		strings.Split(cluster.VolumeGRPCAddress(0), ":")[1]
+
+	// Copy shard 0 + ecx + ecj from server 0 → server 1.
+	_, err = client1.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       volumeID,
+		Collection:     "",
+		SourceDataNode: sourceDataNode,
+		ShardIds:       []uint32{0},
+		CopyEcxFile:    true,
+		CopyEcjFile:    true,
+		CopyVifFile:    true,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsCopy 0→1: %v", err)
+	}
+
+	// Mount shard 0 on server 1 so the EC volume can accept deletions.
+	_, err = client1.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId: volumeID, Collection: "",
+		ShardIds: []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount on server 1: %v", err)
+	}
+
+	// Delete needle A on server 1 only (creates .ecj entry on server 1).
+	_, err = client1.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId: volumeID, Collection: "",
+		FileKey: keyA, Version: uint32(needle.GetCurrentVersion()),
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcBlobDelete needle A on server 1: %v", err)
+	}
+
+	// Mount all data shards on server 0 (the decode target).
+	_, err = client0.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId: volumeID, Collection: "",
+		ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount on server 0: %v", err)
+	}
+
+	// Collect .ecj from server 1 → server 0 with NO new shard IDs.
+	// This is the critical path: server 1 has shard 0 which server 0 already
+	// has, so needToCopyShardsInfo would be empty. Before the fix in
+	// collectEcShards, this copy would be skipped entirely, losing server 1's
+	// deletion entries.
+	server1DataNode := cluster.VolumeAdminAddress(1) + "." +
+		strings.Split(cluster.VolumeGRPCAddress(1), ":")[1]
+
+	_, err = client0.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       volumeID,
+		Collection:     "",
+		SourceDataNode: server1DataNode,
+		ShardIds:       []uint32{}, // No new shards — just .ecj.
+		CopyEcxFile:    false,
+		CopyEcjFile:    true,
+		CopyVifFile:    false,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsCopy .ecj from server 1→0: %v", err)
+	}
+
+	// Unmount the normal volume before decode.
+	_, err = client0.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeUnmount on server 0: %v", err)
+	}
+
+	// Decode on server 0. RebuildEcxFile should see needle A's deletion from
+	// the .ecj that was collected from server 1.
+	_, err = client0.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId: volumeID, Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsToVolume on server 0: %v", err)
+	}
+
+	// Re-mount the decoded normal volume.
+	_, err = client0.VolumeMount(ctx, &volume_server_pb.VolumeMountRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeMount on server 0: %v", err)
+	}
+
+	// Needle A should be gone — its deletion was in server 1's .ecj.
+	respA := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidA)
+	bodyA := framework.ReadAllAndClose(t, respA)
+	if respA.StatusCode >= 500 {
+		t.Fatalf("needle A read: server error %d: %s", respA.StatusCode, bodyA)
+	}
+	if respA.StatusCode != http.StatusNotFound {
+		t.Fatalf("needle A should be 404 (ecj from peer), got %d", respA.StatusCode)
+	}
+
+	// Needle B should still be readable.
+	respB := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fidB)
+	bodyB := framework.ReadAllAndClose(t, respB)
+	if respB.StatusCode != http.StatusOK {
+		t.Fatalf("needle B read: expected 200, got %d", respB.StatusCode)
+	}
+	if string(bodyB) != string(payloadB) {
+		t.Fatalf("needle B payload mismatch: got %q, want %q", bodyB, payloadB)
+	}
+}

test/volume_server/grpc/ec_multi_disk_lifecycle_test.go

@@ -0,0 +1,529 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+)
+
+// TestEcLifecycleAcrossMultipleDisks drives encode, mount, read, drop-dat,
+// cross-disk redistribute + restart + reconcile, blob delete, and rebuild on
+// a 2-disk volume server.
+func TestEcLifecycleAcrossMultipleDisks(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	const (
+		dataDirCount = 2
+		volumeID     = uint32(9500)
+		collection   = "ec-multi-disk"
+	)
+
+	clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
+	dataDirs := clusterHarness.VolumeDataDirs()
+	if len(dataDirs) != dataDirCount {
+		t.Fatalf("expected %d data dirs, got %d: %v", dataDirCount, len(dataDirs), dataDirs)
+	}
+
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	framework.AllocateVolume(t, grpcClient, volumeID, collection)
+
+	// populate
+	type needleSpec struct {
+		fid     string
+		payload []byte
+	}
+	needles := []needleSpec{
+		{framework.NewFileID(volumeID, 9501, 0xAA000001), bytesOfLen(11, 0xA1)},
+		{framework.NewFileID(volumeID, 9502, 0xAA000002), bytesOfLen(1024, 0xA2)},
+		{framework.NewFileID(volumeID, 9503, 0xAA000003), bytesOfLen(63*1024+17, 0xA3)},
+		{framework.NewFileID(volumeID, 9504, 0xAA000004), bytesOfLen(2, 0xA4)},
+		{framework.NewFileID(volumeID, 9505, 0xAA000005), bytesOfLen(513*1024, 0xA5)},
+	}
+	httpClient := framework.NewHTTPClient()
+	for _, n := range needles {
+		resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload)
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s expected 201, got %d", n.fid, resp.StatusCode)
+		}
+	}
+
+	// encode — every shard lands on the .dat's disk
+	if _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: collection,
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsGenerate: %v", err)
+	}
+	postGenerateLayout := scanShardLayout(t, dataDirs, collection, volumeID)
+	if len(postGenerateLayout[0]) != erasure_coding.TotalShardsCount {
+		t.Fatalf("post-generate: expected all %d shards on disk 0, got per-disk layout %v",
+			erasure_coding.TotalShardsCount, postGenerateLayout)
+	}
+	if len(postGenerateLayout[1]) != 0 {
+		t.Fatalf("post-generate: disk 1 should be empty, got %v", postGenerateLayout[1])
+	}
+	// .ecj is created lazily on first EC delete
+	for _, side := range []string{".ecx", ".vif"} {
+		if !fileExistsIn(dataDirs[0], collection, volumeID, side) {
+			t.Fatalf("post-generate: expected %s on disk 0", side)
+		}
+	}
+
+	// mount + read via EC
+	if _, err := grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: collection,
+		ShardIds:   dataShardIds(),
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsMount data shards: %v", err)
+	}
+	for _, n := range needles {
+		verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-mount")
+	}
+
+	// drop the original volume so EC shards are the only source
+	if _, err := grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
+		VolumeId: volumeID,
+	}); err != nil {
+		t.Fatalf("VolumeDelete (drop .dat): %v", err)
+	}
+	if fileExistsIn(dataDirs[0], collection, volumeID, ".dat") {
+		t.Fatalf("VolumeDelete left .dat in place on disk 0")
+	}
+	for _, n := range needles {
+		verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-dat-delete")
+	}
+
+	// move half the shards onto disk 1, leaving .ecx on disk 0
+	clusterHarness.StopVolumeServer()
+	const splitAt = 7
+	for shard := 0; shard < splitAt; shard++ {
+		movedFile(t, dataDirs[0], dataDirs[1], collection, volumeID, erasure_coding.ToExt(shard))
+	}
+	postMoveLayout := scanShardLayout(t, dataDirs, collection, volumeID)
+	if len(postMoveLayout[0]) != erasure_coding.TotalShardsCount-splitAt {
+		t.Fatalf("post-move: expected %d shards on disk 0, got %v", erasure_coding.TotalShardsCount-splitAt, postMoveLayout[0])
+	}
+	if len(postMoveLayout[1]) != splitAt {
+		t.Fatalf("post-move: expected %d shards on disk 1, got %v", splitAt, postMoveLayout[1])
+	}
+	if !fileExistsIn(dataDirs[0], collection, volumeID, ".ecx") {
+		t.Fatalf("post-move: .ecx must stay on disk 0 to exercise the cross-disk reconcile path")
+	}
+	if fileExistsIn(dataDirs[1], collection, volumeID, ".ecx") {
+		t.Fatalf("post-move: .ecx leaked onto disk 1; reconcile path would not be exercised")
+	}
+
+	clusterHarness.RestartVolumeServer()
+	conn2, grpcClient2 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn2.Close()
+
+	postReconcileLayout := scanShardLayout(t, dataDirs, collection, volumeID)
+	if got, want := totalShardsInLayout(postReconcileLayout), erasure_coding.TotalShardsCount; got != want {
+		t.Fatalf("post-reconcile: total shards on disk mismatch: got %d, want %d (layout=%v)", got, want, postReconcileLayout)
+	}
+	if got, want := len(postReconcileLayout[0]), erasure_coding.TotalShardsCount-splitAt; got != want {
+		t.Fatalf("post-reconcile: disk 0 shard count drift: got %d, want %d (layout=%v)", got, want, postReconcileLayout)
+	}
+	if got, want := len(postReconcileLayout[1]), splitAt; got != want {
+		t.Fatalf("post-reconcile: disk 1 shard count drift: got %d, want %d (layout=%v)", got, want, postReconcileLayout)
+	}
+	// VolumeEcShardsInfo must walk every DiskLocation and report the full
+	// shard set — the verification step in ec_task.go gates source-volume
+	// deletion on this RPC returning a complete shard inventory.
+	infoResp, err := grpcClient2.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsInfo after redistribute restart: %v", err)
+	}
+	if got, want := len(infoResp.GetEcShardInfos()), erasure_coding.TotalShardsCount; got != want {
+		t.Fatalf("VolumeEcShardsInfo after redistribute restart: got %d shards, want %d (per-disk layout=%v)",
+			got, want, postReconcileLayout)
+	}
+	gotShardIds := make(map[uint32]struct{}, len(infoResp.GetEcShardInfos()))
+	for _, info := range infoResp.GetEcShardInfos() {
+		gotShardIds[info.GetShardId()] = struct{}{}
+	}
+	for shardId := uint32(0); shardId < uint32(erasure_coding.TotalShardsCount); shardId++ {
+		if _, ok := gotShardIds[shardId]; !ok {
+			t.Fatalf("VolumeEcShardsInfo missing shard %d (per-disk layout=%v)", shardId, postReconcileLayout)
+		}
+	}
+	for _, n := range needles {
+		verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-cross-disk-reconcile")
+	}
+
+	// blob delete on a cross-disk layout
+	deletedNeedle := needles[2]
+	deleteReq, err := http.NewRequest(http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+deletedNeedle.fid, nil)
+	if err != nil {
+		t.Fatalf("build delete request: %v", err)
+	}
+	deleteResp := framework.DoRequest(t, httpClient, deleteReq)
+	_ = framework.ReadAllAndClose(t, deleteResp)
+	if deleteResp.StatusCode != http.StatusAccepted {
+		t.Fatalf("HTTP DELETE %s expected 202, got %d", deletedNeedle.fid, deleteResp.StatusCode)
+	}
+	readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), deletedNeedle.fid)
+	_ = framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode == http.StatusOK {
+		t.Fatalf("HTTP GET on deleted EC needle expected non-200, got %d", readResp.StatusCode)
+	}
+	for _, n := range needles {
+		if n.fid == deletedNeedle.fid {
+			continue
+		}
+		verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-blob-delete")
+	}
+
+	// rebuild a shard whose inputs are split across two disks
+	clusterHarness.StopVolumeServer()
+	const repairTargetShard = 9 // sits on disk 0 after the split
+	shardPath := filepath.Join(dataDirs[0], shardFileName(collection, volumeID, repairTargetShard))
+	if _, statErr := os.Stat(shardPath); statErr != nil {
+		t.Fatalf("repair setup: shard %d not on disk 0 (%s): %v", repairTargetShard, shardPath, statErr)
+	}
+	if err := os.Remove(shardPath); err != nil {
+		t.Fatalf("repair setup: remove shard %d on disk 0: %v", repairTargetShard, err)
+	}
+	clusterHarness.RestartVolumeServer()
+	conn3, grpcClient3 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn3.Close()
+
+	rebuildResp, err := grpcClient3.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{
+		VolumeId:   volumeID,
+		Collection: collection,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsRebuild after cross-disk shard loss: %v", err)
+	}
+	if !containsUint32(rebuildResp.GetRebuiltShardIds(), repairTargetShard) {
+		t.Fatalf("VolumeEcShardsRebuild expected to rebuild shard %d, got %v",
+			repairTargetShard, rebuildResp.GetRebuiltShardIds())
+	}
+	// The rebuilder picks whichever disk hosts the most shards plus a
+	// matching .ecx. After the boot-time mirror runs, every shard-
+	// bearing disk owns its own sidecars, so rebuild can legitimately
+	// restore the shard on either disk — accept both.
+	rebuiltOn := -1
+	for i, dir := range dataDirs {
+		candidate := filepath.Join(dir, shardFileName(collection, volumeID, repairTargetShard))
+		if _, statErr := os.Stat(candidate); statErr == nil {
+			rebuiltOn = i
+			shardPath = candidate
+			break
+		}
+	}
+	if rebuiltOn < 0 {
+		t.Fatalf("rebuild did not restore shard %d on any disk (checked %v)", repairTargetShard, dataDirs)
+	}
+
+	if _, err := grpcClient3.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: collection,
+		ShardIds:   []uint32{repairTargetShard},
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsMount rebuilt shard %d: %v", repairTargetShard, err)
+	}
+	for _, n := range needles {
+		if n.fid == deletedNeedle.fid {
+			continue
+		}
+		verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload, "after-repair")
+	}
+}
+
+// TestEcPartialShardsOnSiblingDiskCleanedUpOnRestart seeds a healthy .dat on
+// disk 0, plants the on-disk footprint of an interrupted EC encode on disk 1
+// (one shard plus .ecx / .ecj / .vif, no .dat), and asserts the startup prune
+// wipes disk 1 without touching disk 0.
+func TestEcPartialShardsOnSiblingDiskCleanedUpOnRestart(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	const (
+		dataDirCount = 2
+		volumeID     = uint32(9501)
+		collection   = "ec-9478"
+		partialShard = 1
+	)
+
+	clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
+	dataDirs := clusterHarness.VolumeDataDirs()
+	if len(dataDirs) != dataDirCount {
+		t.Fatalf("expected %d data dirs, got %d: %v", dataDirCount, len(dataDirs), dataDirs)
+	}
+
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	framework.AllocateVolume(t, grpcClient, volumeID, collection)
+
+	httpClient := framework.NewHTTPClient()
+	needleFid := framework.NewFileID(volumeID, 7777, 0xCAFEBABE)
+	needlePayload := bytesOfLen(4096, 0xC7)
+	upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), needleFid, needlePayload)
+	_ = framework.ReadAllAndClose(t, upResp)
+	if upResp.StatusCode != http.StatusCreated {
+		t.Fatalf("seed upload expected 201, got %d", upResp.StatusCode)
+	}
+
+	datPath := filepath.Join(dataDirs[0], datFileName(collection, volumeID))
+	datInfo, err := os.Stat(datPath)
+	if err != nil {
+		t.Fatalf("seed setup: .dat must exist on disk 0 (%s): %v", datPath, err)
+	}
+	// prune's size guard refuses cleanup if the sibling .dat looks truncated
+	if datInfo.Size() == 0 {
+		t.Fatalf("seed setup: .dat on disk 0 is zero bytes")
+	}
+
+	clusterHarness.StopVolumeServer()
+	plantPartialEc(t, dataDirs[1], collection, volumeID, partialShard, datInfo.Size())
+
+	preRestartLayout := scanShardLayout(t, dataDirs, collection, volumeID)
+	if len(preRestartLayout[0]) != 0 {
+		t.Fatalf("seed setup: disk 0 should hold no EC shards before restart, got %v", preRestartLayout[0])
+	}
+	if len(preRestartLayout[1]) != 1 {
+		t.Fatalf("seed setup: disk 1 should hold exactly one planted shard, got %v", preRestartLayout[1])
+	}
+
+	clusterHarness.RestartVolumeServer()
+
+	conn2, grpcClient2 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn2.Close()
+
+	// the volume admin port answers before the startup prune finishes
+	postPruneLayout := waitForLayout(t, dataDirs, collection, volumeID, func(layout map[int][]int) bool {
+		return len(layout[1]) == 0
+	}, 10*time.Second)
+
+	if len(postPruneLayout[1]) != 0 {
+		t.Fatalf("partial EC shards survived restart on disk 1: %v", postPruneLayout[1])
+	}
+	// .vif is intentionally left behind: with no .ecx / shard to anchor it,
+	// the next startup ignores it.
+	for _, side := range []string{".ecx", ".ecj"} {
+		if fileExistsIn(dataDirs[1], collection, volumeID, side) {
+			t.Fatalf("%s survived prune on disk 1", side)
+		}
+	}
+	if !fileExistsIn(dataDirs[0], collection, volumeID, ".dat") {
+		t.Fatalf("prune wiped the healthy .dat on disk 0")
+	}
+	if !fileExistsIn(dataDirs[0], collection, volumeID, ".idx") {
+		t.Fatalf("prune wiped the healthy .idx on disk 0")
+	}
+	if _, err = grpcClient2.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+		VolumeId: volumeID,
+	}); err == nil {
+		t.Fatalf("VolumeEcShardsInfo returned success after prune; the partial EC mount should be gone")
+	}
+	verifyHTTPRead(t, httpClient, clusterHarness.VolumeAdminURL(), needleFid, needlePayload, "after-prune")
+}
+
+func bytesOfLen(n int, seed byte) []byte {
+	out := make([]byte, n)
+	for i := range out {
+		out[i] = byte(int(seed) + i*31)
+	}
+	return out
+}
+
+func dataShardIds() []uint32 {
+	ids := make([]uint32, erasure_coding.DataShardsCount)
+	for i := range ids {
+		ids[i] = uint32(i)
+	}
+	return ids
+}
+
+func datFileName(collection string, volumeID uint32) string {
+	if collection == "" {
+		return fmt.Sprintf("%d.dat", volumeID)
+	}
+	return fmt.Sprintf("%s_%d.dat", collection, volumeID)
+}
+
+func shardFileName(collection string, volumeID uint32, shardID int) string {
+	if collection == "" {
+		return fmt.Sprintf("%d%s", volumeID, erasure_coding.ToExt(shardID))
+	}
+	return fmt.Sprintf("%s_%d%s", collection, volumeID, erasure_coding.ToExt(shardID))
+}
+
+func sideFileName(collection string, volumeID uint32, ext string) string {
+	if collection == "" {
+		return fmt.Sprintf("%d%s", volumeID, ext)
+	}
+	return fmt.Sprintf("%s_%d%s", collection, volumeID, ext)
+}
+
+func fileExistsIn(dir, collection string, volumeID uint32, ext string) bool {
+	_, err := os.Stat(filepath.Join(dir, sideFileName(collection, volumeID, ext)))
+	return err == nil
+}
+
+func scanShardLayout(t testing.TB, dataDirs []string, collection string, volumeID uint32) map[int][]int {
+	t.Helper()
+	out := make(map[int][]int, len(dataDirs))
+	for diskIdx, dir := range dataDirs {
+		out[diskIdx] = nil
+		for shard := 0; shard < erasure_coding.TotalShardsCount; shard++ {
+			if _, err := os.Stat(filepath.Join(dir, shardFileName(collection, volumeID, shard))); err == nil {
+				out[diskIdx] = append(out[diskIdx], shard)
+			}
+		}
+		sort.Ints(out[diskIdx])
+	}
+	return out
+}
+
+func waitForLayout(t testing.TB, dataDirs []string, collection string, volumeID uint32, pred func(map[int][]int) bool, timeout time.Duration) map[int][]int {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	var layout map[int][]int
+	for time.Now().Before(deadline) {
+		layout = scanShardLayout(t, dataDirs, collection, volumeID)
+		if pred(layout) {
+			return layout
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return layout
+}
+
+func movedFile(t testing.TB, fromDir, toDir, collection string, volumeID uint32, ext string) {
+	t.Helper()
+	name := sideFileName(collection, volumeID, ext)
+	src := filepath.Join(fromDir, name)
+	dst := filepath.Join(toDir, name)
+	if err := os.Rename(src, dst); err != nil {
+		t.Fatalf("move %s → %s: %v", src, dst, err)
+	}
+}
+
+func totalShardsInLayout(layout map[int][]int) int {
+	n := 0
+	for _, ids := range layout {
+		n += len(ids)
+	}
+	return n
+}
+
+func containsUint32(s []uint32, v uint32) bool {
+	for _, x := range s {
+		if x == v {
+			return true
+		}
+	}
+	return false
+}
+
+func verifyHTTPRead(t testing.TB, client *http.Client, base, fid string, want []byte, stage string) {
+	t.Helper()
+	resp := framework.ReadBytes(t, client, base, fid)
+	body := framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("[%s] GET %s expected 200, got %d", stage, fid, resp.StatusCode)
+	}
+	if len(body) != len(want) {
+		t.Fatalf("[%s] GET %s size mismatch: got %d, want %d", stage, fid, len(body), len(want))
+	}
+	for i := range want {
+		if body[i] != want[i] {
+			t.Fatalf("[%s] GET %s byte %d mismatch: got %d, want %d", stage, fid, i, body[i], want[i])
+		}
+	}
+}
+
+// plantPartialEc writes the footprint of an interrupted EC encode (one shard
+// sized for datFileSize, plus .ecx / .ecj / .vif, no .dat) into dir.
+func plantPartialEc(t testing.TB, dir, collection string, volumeID uint32, shardID int, datFileSize int64) {
+	t.Helper()
+	shardPath := filepath.Join(dir, shardFileName(collection, volumeID, shardID))
+	if f, err := os.Create(shardPath); err == nil {
+		expectedShardSize := plantedShardSize(datFileSize)
+		if err := f.Truncate(expectedShardSize); err != nil {
+			f.Close()
+			t.Fatalf("truncate planted shard: %v", err)
+		}
+		f.Close()
+	} else {
+		t.Fatalf("create planted shard %s: %v", shardPath, err)
+	}
+	for _, side := range []struct {
+		ext     string
+		content []byte
+	}{
+		{".ecx", []byte("dummy ecx")},
+		{".ecj", nil},
+	} {
+		p := filepath.Join(dir, sideFileName(collection, volumeID, side.ext))
+		f, err := os.Create(p)
+		if err != nil {
+			t.Fatalf("create planted %s: %v", side.ext, err)
+		}
+		if side.content != nil {
+			if _, err := f.Write(side.content); err != nil {
+				f.Close()
+				t.Fatalf("write planted %s: %v", side.ext, err)
+			}
+		}
+		f.Close()
+	}
+	// Record the encode-time source size so the prune's byte-exact gate
+	// recognizes the sibling .dat as the source, as a real encoded volume does.
+	vifPath := filepath.Join(dir, sideFileName(collection, volumeID, ".vif"))
+	if err := volume_info.SaveVolumeInfo(vifPath, &volume_server_pb.VolumeInfo{
+		Version:     uint32(needle.Version3),
+		DatFileSize: datFileSize,
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards:   uint32(erasure_coding.DataShardsCount),
+			ParityShards: uint32(erasure_coding.ParityShardsCount),
+		},
+	}); err != nil {
+		t.Fatalf("save planted .vif: %v", err)
+	}
+}
+
+// mirrors calculateExpectedShardSize in weed/storage/disk_location_ec.go
+func plantedShardSize(datFileSize int64) int64 {
+	const dataShards = int64(erasure_coding.DataShardsCount)
+	largeBatch := int64(erasure_coding.ErasureCodingLargeBlockSize) * dataShards
+	numLarge := datFileSize / largeBatch
+	shardSize := numLarge * int64(erasure_coding.ErasureCodingLargeBlockSize)
+	remaining := datFileSize - numLarge*largeBatch
+	if remaining > 0 {
+		smallBatch := int64(erasure_coding.ErasureCodingSmallBlockSize) * dataShards
+		numSmall := (remaining + smallBatch - 1) / smallBatch
+		shardSize += numSmall * int64(erasure_coding.ErasureCodingSmallBlockSize)
+	}
+	return shardSize
+}

test/volume_server/grpc/ec_receive_disk_placement_test.go

@@ -0,0 +1,178 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+// Sends EC shards with disk_id={1, 2, 0} and verifies each lands on the
+// requested disk (0 → auto-select → disk 0 for a fresh volume).
+func TestReceiveFileEcShardHonorsDiskID(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	const dataDirCount = 3
+	clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	dataDirs := clusterHarness.VolumeDataDirs()
+	if len(dataDirs) != dataDirCount {
+		t.Fatalf("expected %d data dirs, got %d: %v", dataDirCount, len(dataDirs), dataDirs)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+
+	const volumeID = uint32(9184)
+	const collection = "ec-disk-placement"
+
+	// DiskId=0 triggers auto-select. Since this test never calls
+	// VolumeEcShardsMount, loc.FindEcVolume returns false on every disk, so
+	// the "already holds shards" preference is skipped and auto-select falls
+	// through to the first HDD (dir 0). If shards get mounted between uploads
+	// in a future revision, the expected dir for DiskId=0 becomes the first
+	// disk that already holds a shard.
+	shards := []struct {
+		shardID        uint32
+		requestedDisk  uint32
+		expectedDirIdx int
+	}{
+		{shardID: 0, requestedDisk: 1, expectedDirIdx: 1},
+		{shardID: 5, requestedDisk: 2, expectedDirIdx: 2},
+		{shardID: 13, requestedDisk: 0, expectedDirIdx: 0},
+	}
+	shardExt := func(id uint32) string { return fmt.Sprintf(".ec%02d", id) }
+
+	for _, s := range shards {
+		payload := []byte(fmt.Sprintf("ec-shard-payload-%02d", s.shardID))
+		stream, err := grpcClient.ReceiveFile(ctx)
+		if err != nil {
+			t.Fatalf("ReceiveFile stream create for shard %d: %v", s.shardID, err)
+		}
+		if err = stream.Send(&volume_server_pb.ReceiveFileRequest{
+			Data: &volume_server_pb.ReceiveFileRequest_Info{
+				Info: &volume_server_pb.ReceiveFileInfo{
+					VolumeId:   volumeID,
+					Ext:        shardExt(s.shardID),
+					Collection: collection,
+					IsEcVolume: true,
+					ShardId:    s.shardID,
+					FileSize:   uint64(len(payload)),
+					DiskId:     s.requestedDisk,
+				},
+			},
+		}); err != nil {
+			t.Fatalf("ReceiveFile send info for shard %d: %v", s.shardID, err)
+		}
+		if err = stream.Send(&volume_server_pb.ReceiveFileRequest{
+			Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payload},
+		}); err != nil {
+			t.Fatalf("ReceiveFile send content for shard %d: %v", s.shardID, err)
+		}
+		resp, err := stream.CloseAndRecv()
+		if err != nil {
+			t.Fatalf("ReceiveFile close for shard %d: %v", s.shardID, err)
+		}
+		if resp.GetError() != "" {
+			t.Fatalf("ReceiveFile shard %d error response: %s", s.shardID, resp.GetError())
+		}
+		if resp.GetBytesWritten() != uint64(len(payload)) {
+			t.Fatalf("ReceiveFile shard %d bytes_written mismatch: got %d want %d",
+				s.shardID, resp.GetBytesWritten(), len(payload))
+		}
+	}
+
+	// Scan every data dir and record which shard files live where.
+	perDirShards := make(map[int][]uint32)
+	for dirIdx, dir := range dataDirs {
+		for _, s := range shards {
+			shardPath := filepath.Join(dir, fmt.Sprintf("%s_%d%s", collection, volumeID, shardExt(s.shardID)))
+			if _, err := os.Stat(shardPath); err == nil {
+				perDirShards[dirIdx] = append(perDirShards[dirIdx], s.shardID)
+			} else if !os.IsNotExist(err) {
+				t.Fatalf("unexpected stat error for %s: %v", shardPath, err)
+			}
+		}
+	}
+
+	for _, s := range shards {
+		found := false
+		for _, got := range perDirShards[s.expectedDirIdx] {
+			if got == s.shardID {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Fatalf("shard %d (requestedDisk=%d) expected on dir[%d], full layout: %v",
+				s.shardID, s.requestedDisk, s.expectedDirIdx, perDirShards)
+		}
+		for dirIdx, ids := range perDirShards {
+			if dirIdx == s.expectedDirIdx {
+				continue
+			}
+			for _, id := range ids {
+				if id == s.shardID {
+					t.Fatalf("shard %d leaked onto dir[%d]: full layout: %v", s.shardID, dirIdx, perDirShards)
+				}
+			}
+		}
+	}
+
+	t.Logf("disk_id honored: shard placement = %v", perDirShards)
+}
+
+func TestReceiveFileEcShardRejectsInvalidDiskID(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	const dataDirCount = 2
+	clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	const volumeID = uint32(91840)
+	const collection = "ec-invalid-disk"
+	payload := []byte("invalid-disk-id-payload")
+
+	stream, err := grpcClient.ReceiveFile(ctx)
+	if err != nil {
+		t.Fatalf("ReceiveFile stream create: %v", err)
+	}
+	if err = stream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_Info{
+			Info: &volume_server_pb.ReceiveFileInfo{
+				VolumeId:   volumeID,
+				Ext:        ".ec00",
+				Collection: collection,
+				IsEcVolume: true,
+				ShardId:    0,
+				FileSize:   uint64(len(payload)),
+				DiskId:     99,
+			},
+		},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send info: %v", err)
+	}
+	resp, err := stream.CloseAndRecv()
+	if err != nil {
+		t.Fatalf("ReceiveFile close: %v", err)
+	}
+	if resp.GetError() == "" {
+		t.Fatalf("expected invalid disk_id rejection, got success response: %+v", resp)
+	}
+}

test/volume_server/grpc/ec_receive_truncates_mounted_test.go

@@ -0,0 +1,186 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestReceiveFileRejectsOverwriteOfMountedEcShard(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(91841)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 918401, 0x9184CAFE)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
+		[]byte("ec-receive-overwrite-content-for-issue-9184-repro"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsGenerate: %v", err)
+	}
+
+	if _, err := grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0},
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsMount: %v", err)
+	}
+
+	dataDir := filepath.Join(clusterHarness.BaseDir(), "volume")
+	shardPath := filepath.Join(dataDir, fmt.Sprintf("%d.ec00", volumeID))
+	origInfo, err := os.Stat(shardPath)
+	if err != nil {
+		t.Fatalf("stat mounted shard %s: %v", shardPath, err)
+	}
+	origSize := origInfo.Size()
+	if origSize == 0 {
+		t.Fatalf("mounted shard %s unexpectedly empty", shardPath)
+	}
+
+	readStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
+		VolumeId: volumeID,
+		ShardId:  0,
+		Offset:   0,
+		Size:     1,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardRead (pre): %v", err)
+	}
+	if _, err := readStream.Recv(); err != nil {
+		t.Fatalf("VolumeEcShardRead Recv (pre): %v", err)
+	}
+
+	overwritePayload := []byte("bug-9184-overwrite")
+	receiveStream, err := grpcClient.ReceiveFile(ctx)
+	if err != nil {
+		t.Fatalf("ReceiveFile stream create: %v", err)
+	}
+	if err = receiveStream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_Info{
+			Info: &volume_server_pb.ReceiveFileInfo{
+				VolumeId:   volumeID,
+				Ext:        ".ec00",
+				Collection: "",
+				IsEcVolume: true,
+				ShardId:    0,
+				FileSize:   uint64(len(overwritePayload)),
+			},
+		},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send info: %v", err)
+	}
+	resp, err := receiveStream.CloseAndRecv()
+	if err != nil {
+		t.Logf("ReceiveFile rejected at stream level: %v", err)
+	} else {
+		if resp.GetError() == "" {
+			t.Fatalf("expected ReceiveFile to reject overwrite of mounted shard, got success: %+v", resp)
+		}
+		if !strings.Contains(resp.GetError(), "mounted") {
+			t.Fatalf("expected error to mention mounted; got: %s", resp.GetError())
+		}
+	}
+
+	afterInfo, err := os.Stat(shardPath)
+	if err != nil {
+		t.Fatalf("stat shard after rejected overwrite: %v", err)
+	}
+	if afterInfo.Size() != origSize {
+		t.Fatalf("shard %s was modified despite rejection: size was %d, now %d",
+			shardPath, origSize, afterInfo.Size())
+	}
+
+	postStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
+		VolumeId: volumeID,
+		ShardId:  0,
+		Offset:   0,
+		Size:     1,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardRead (post): %v", err)
+	}
+	if _, err := postStream.Recv(); err != nil {
+		t.Fatalf("VolumeEcShardRead Recv (post): %v", err)
+	}
+
+	t.Logf("ReceiveFile correctly refused overwrite; mounted shard intact at %d bytes", afterInfo.Size())
+}
+
+func TestReceiveFileAllowsEcShardWhenNoMount(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	const volumeID = uint32(91843)
+	const collection = "ec-receive-no-mount"
+	payload := []byte("ok-to-receive-not-mounted")
+
+	stream, err := grpcClient.ReceiveFile(ctx)
+	if err != nil {
+		t.Fatalf("ReceiveFile stream create: %v", err)
+	}
+	if err = stream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_Info{
+			Info: &volume_server_pb.ReceiveFileInfo{
+				VolumeId:   volumeID,
+				Ext:        ".ec00",
+				Collection: collection,
+				IsEcVolume: true,
+				ShardId:    0,
+				FileSize:   uint64(len(payload)),
+			},
+		},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send info: %v", err)
+	}
+	if err = stream.Send(&volume_server_pb.ReceiveFileRequest{
+		Data: &volume_server_pb.ReceiveFileRequest_FileContent{FileContent: payload},
+	}); err != nil {
+		t.Fatalf("ReceiveFile send content: %v", err)
+	}
+	resp, err := stream.CloseAndRecv()
+	if err != nil {
+		t.Fatalf("ReceiveFile close: %v", err)
+	}
+	if resp.GetError() != "" {
+		t.Fatalf("expected success on unmounted volume, got error: %s", resp.GetError())
+	}
+	if resp.GetBytesWritten() != uint64(len(payload)) {
+		t.Fatalf("bytes_written mismatch: got %d want %d", resp.GetBytesWritten(), len(payload))
+	}
+}

test/volume_server/grpc/ec_verify_multi_disk_test.go

@@ -0,0 +1,182 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"net/http"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// TestVolumeEcShardsInfoReturnsAllShardsAcrossDisks drives the full path
+// behind the ec.encode source-deletion gate. A multi-disk volume server
+// ends up with EC shards split across disks (each registers its own
+// EcVolume entry in DiskLocation.ecVolumes), and the volume server's
+// VolumeEcShardsInfo RPC must walk every DiskLocation rather than
+// reporting whichever disk Store.FindEcVolume picks first.
+//
+// Pre-fix, verifyEcShardsBeforeDelete refused to delete source volumes —
+// the shard-bitmap union across destinations fell short of dataShards +
+// parityShards because each destination only reported shards on one of
+// its disks. With the handler fix, the same VerifyShardsAcrossServers
+// call returns a complete bitmap and the gate opens.
+func TestVolumeEcShardsInfoReturnsAllShardsAcrossDisks(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	const (
+		dataDirCount = 2
+		volumeID     = uint32(9558)
+		collection   = "ec-multi-disk-verify"
+	)
+
+	clusterHarness := framework.StartSingleVolumeClusterWithDataDirs(t, matrix.P1(), dataDirCount)
+	dataDirs := clusterHarness.VolumeDataDirs()
+	if len(dataDirs) != dataDirCount {
+		t.Fatalf("expected %d data dirs, got %d: %v", dataDirCount, len(dataDirs), dataDirs)
+	}
+
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	framework.AllocateVolume(t, grpcClient, volumeID, collection)
+
+	httpClient := framework.NewHTTPClient()
+	needles := []struct {
+		fid     string
+		payload []byte
+	}{
+		{framework.NewFileID(volumeID, 9559, 0xC0FFEE01), bytesOfLen(64, 0xB1)},
+		{framework.NewFileID(volumeID, 9560, 0xC0FFEE02), bytesOfLen(8192, 0xB2)},
+		{framework.NewFileID(volumeID, 9561, 0xC0FFEE03), bytesOfLen(131072, 0xB3)},
+	}
+	for _, n := range needles {
+		resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload)
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s expected 201, got %d", n.fid, resp.StatusCode)
+		}
+	}
+
+	if _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: collection,
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsGenerate: %v", err)
+	}
+
+	// Generate places every shard plus the .ecx/.ecj/.vif on the .dat's
+	// disk (disk 0). Mount all 14 there first so the next step's restart
+	// has a steady starting state.
+	allShards := make([]uint32, erasure_coding.TotalShardsCount)
+	for i := range allShards {
+		allShards[i] = uint32(i)
+	}
+	if _, err := grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: collection,
+		ShardIds:   allShards,
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsMount all shards: %v", err)
+	}
+
+	// Drop the .dat so the EC shards are the only data path — mirrors the
+	// real ec.encode flow before verifyEcShardsBeforeDelete fires.
+	if _, err := grpcClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
+		VolumeId: volumeID,
+	}); err != nil {
+		t.Fatalf("VolumeDelete (drop .dat): %v", err)
+	}
+
+	// Move half the shards onto disk 1, leaving .ecx on disk 0. After
+	// restart, the cross-disk reconcile path attaches each disk's shards
+	// against its own EcVolume entry — the exact in-memory shape the bug
+	// reporter saw on a multi-disk destination.
+	clusterHarness.StopVolumeServer()
+	const splitAt = 7
+	for shard := 0; shard < splitAt; shard++ {
+		movedFile(t, dataDirs[0], dataDirs[1], collection, volumeID, erasure_coding.ToExt(shard))
+	}
+	if fileExistsIn(dataDirs[1], collection, volumeID, ".ecx") {
+		t.Fatalf("setup: .ecx must stay on disk 0 to exercise the multi-disk path")
+	}
+
+	clusterHarness.RestartVolumeServer()
+	conn2, grpcClient2 := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn2.Close()
+
+	postReconcileLayout := scanShardLayout(t, dataDirs, collection, volumeID)
+	if got, want := totalShardsInLayout(postReconcileLayout), erasure_coding.TotalShardsCount; got != want {
+		t.Fatalf("post-reconcile: total shards on disk mismatch: got %d, want %d (layout=%v)", got, want, postReconcileLayout)
+	}
+	if len(postReconcileLayout[0]) == 0 || len(postReconcileLayout[1]) == 0 {
+		t.Fatalf("post-reconcile: expected shards on BOTH disks, got per-disk layout %v", postReconcileLayout)
+	}
+
+	// Direct RPC assertion: VolumeEcShardsInfo must report every shard
+	// the server holds, not just the ones registered against the first
+	// matching DiskLocation.
+	infoResp, err := grpcClient2.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsInfo: %v", err)
+	}
+	gotShardIds := make([]int, 0, len(infoResp.GetEcShardInfos()))
+	for _, info := range infoResp.GetEcShardInfos() {
+		if info.GetVolumeId() != volumeID {
+			t.Errorf("EcShardInfo VolumeId=%d, want %d", info.GetVolumeId(), volumeID)
+		}
+		gotShardIds = append(gotShardIds, int(info.GetShardId()))
+	}
+	sort.Ints(gotShardIds)
+	wantShardIds := make([]int, erasure_coding.TotalShardsCount)
+	for i := range wantShardIds {
+		wantShardIds[i] = i
+	}
+	if len(gotShardIds) != len(wantShardIds) {
+		t.Fatalf("VolumeEcShardsInfo returned %d shards (ids=%v), want %d (ids=%v) — per-disk layout=%v",
+			len(gotShardIds), gotShardIds, len(wantShardIds), wantShardIds, postReconcileLayout)
+	}
+	for i, sid := range wantShardIds {
+		if gotShardIds[i] != sid {
+			t.Fatalf("VolumeEcShardsInfo shard ids=%v, want %v (per-disk layout=%v)",
+				gotShardIds, wantShardIds, postReconcileLayout)
+		}
+	}
+
+	// End-to-end assertion via the same helper the worker uses to gate
+	// source-volume deletion (weed/worker/tasks/erasure_coding/ec_task.go
+	// verifyEcShardsBeforeDelete). The union across destinations is what
+	// RequireRecoverableShardSet measures; with one destination that holds
+	// every shard, the union must cover dataShards + parityShards without
+	// being reported as degraded.
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	servers := []string{clusterHarness.VolumeServerAddress()}
+	union, perServer := erasure_coding.VerifyShardsAcrossServers(ctx, volumeID, servers, dialOption)
+	degraded, err := erasure_coding.RequireRecoverableShardSet(
+		volumeID, union, erasure_coding.DataShardsCount, erasure_coding.TotalShardsCount)
+	if err != nil {
+		t.Fatalf("verifyEcShardsBeforeDelete-equivalent gate failed: %v\nper-server inventory: %s\nper-disk layout: %v",
+			err, erasure_coding.SummarizeShardInventory(perServer), postReconcileLayout)
+	}
+	if degraded {
+		t.Fatalf("verifyEcShardsBeforeDelete-equivalent gate reported a degraded shard set\nper-server inventory: %s\nper-disk layout: %v",
+			erasure_coding.SummarizeShardInventory(perServer), postReconcileLayout)
+	}
+	if got, want := union.Count(), erasure_coding.TotalShardsCount; got != want {
+		t.Fatalf("VerifyShardsAcrossServers union covered %d/%d shards (per-server=%s, layout=%v)",
+			got, want, erasure_coding.SummarizeShardInventory(perServer), postReconcileLayout)
+	}
+}

test/volume_server/grpc/erasure_coding_test.go

@@ -0,0 +1,871 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"io"
+	"math"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func TestEcMaintenanceModeRejections(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	// Encode a volume before entering maintenance mode so the shard delete
+	// below has real shards to remove.
+	const ecVolumeID = uint32(1)
+	framework.AllocateVolume(t, grpcClient, ecVolumeID, "")
+	httpClient := framework.NewHTTPClient()
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(ecVolumeID, 990001, 0x11223344), []byte("ec-maintenance-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	if _, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{VolumeId: ecVolumeID, Collection: ""}); err != nil {
+		t.Fatalf("VolumeEcShardsGenerate before maintenance failed: %v", err)
+	}
+
+	framework.EnableMaintenanceMode(t, ctx, grpcClient)
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{VolumeId: 1, Collection: ""})
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeEcShardsGenerate maintenance error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       1,
+		Collection:     "",
+		SourceDataNode: "127.0.0.1:1",
+		ShardIds:       []uint32{0},
+	})
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeEcShardsCopy maintenance error mismatch: %v", err)
+	}
+
+	// Deleting shards removes data rather than adding it, and is how evacuating
+	// EC shards off a server in maintenance mode finishes (issue #11066).
+	_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+		VolumeId:   ecVolumeID,
+		Collection: "",
+		ShardIds:   []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsDelete should succeed in maintenance mode, got: %v", err)
+	}
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   ecVolumeID,
+		Collection: "",
+		ShardIds:   []uint32{0},
+	})
+	if err == nil {
+		t.Fatalf("VolumeEcShardsMount should fail once shard 0 has been deleted")
+	}
+
+	_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId:   1,
+		Collection: "",
+		FileKey:    1,
+		Version:    3,
+	})
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeEcBlobDelete maintenance error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId:   1,
+		Collection: "",
+	})
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeEcShardsToVolume maintenance error mismatch: %v", err)
+	}
+}
+
+func TestEcMissingInvalidAndNoopPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   98791,
+		Collection: "",
+	})
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeEcShardsGenerate missing-volume error mismatch: %v", err)
+	}
+
+	rebuildResp, err := grpcClient.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{
+		VolumeId:   98792,
+		Collection: "ec-rebuild",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsRebuild missing-volume should return empty success, got: %v", err)
+	}
+	if len(rebuildResp.GetRebuiltShardIds()) != 0 {
+		t.Fatalf("VolumeEcShardsRebuild expected no rebuilt shards for missing volume, got %v", rebuildResp.GetRebuiltShardIds())
+	}
+
+	_, err = grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       98793,
+		Collection:     "ec-copy",
+		SourceDataNode: "127.0.0.1:1",
+		ShardIds:       []uint32{0},
+		DiskId:         99,
+	})
+	if err == nil || !strings.Contains(err.Error(), "invalid disk_id") {
+		t.Fatalf("VolumeEcShardsCopy invalid-disk error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+		VolumeId:   98794,
+		Collection: "ec-delete",
+		ShardIds:   []uint32{0, 1},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsDelete missing-volume should be no-op success, got: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   98795,
+		Collection: "ec-mount",
+		ShardIds:   []uint32{0},
+	})
+	if err == nil {
+		t.Fatalf("VolumeEcShardsMount should fail for missing EC shards")
+	}
+
+	_, err = grpcClient.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
+		VolumeId: 98796,
+		ShardIds: []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsUnmount missing shards should be no-op success, got: %v", err)
+	}
+
+	readStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
+		VolumeId: 98797,
+		ShardId:  0,
+		Offset:   0,
+		Size:     1,
+	})
+	if err == nil {
+		_, err = readStream.Recv()
+	}
+	if err == nil || err == io.EOF {
+		t.Fatalf("VolumeEcShardRead should fail for missing EC volume")
+	}
+
+	_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId:   98798,
+		Collection: "ec-blob",
+		FileKey:    1,
+		Version:    3,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcBlobDelete missing local EC volume should be no-op success, got: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId:   98799,
+		Collection: "ec-to-volume",
+	})
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeEcShardsToVolume missing-volume error mismatch: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+		VolumeId: 98800,
+	})
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeEcShardsInfo missing-volume error mismatch: %v", err)
+	}
+}
+
+func TestEcGenerateMountInfoUnmountLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(115)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 990001, 0x1234ABCD)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-generate-lifecycle-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate success path failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount success path failed: %v", err)
+	}
+
+	infoResp, err := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsInfo after mount failed: %v", err)
+	}
+	if len(infoResp.GetEcShardInfos()) == 0 {
+		t.Fatalf("VolumeEcShardsInfo expected non-empty shard infos after mount")
+	}
+	if infoResp.GetVolumeSize() == 0 {
+		t.Fatalf("VolumeEcShardsInfo expected non-zero volume size after mount")
+	}
+
+	_, err = grpcClient.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
+		VolumeId: volumeID,
+		ShardIds: []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsUnmount success path failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+		VolumeId: volumeID,
+	})
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeEcShardsInfo after unmount expected not-found error, got: %v", err)
+	}
+}
+
+func TestEcShardReadAndBlobDeleteLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(116)
+	const fileKey = uint64(990002)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, fileKey, 0x2233CCDD)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-shard-read-delete-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount failed: %v", err)
+	}
+
+	readStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
+		VolumeId: volumeID,
+		ShardId:  0,
+		Offset:   0,
+		Size:     1,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardRead start failed: %v", err)
+	}
+	firstChunk, err := readStream.Recv()
+	if err != nil {
+		t.Fatalf("VolumeEcShardRead recv failed: %v", err)
+	}
+	if len(firstChunk.GetData()) == 0 {
+		t.Fatalf("VolumeEcShardRead expected non-empty data chunk before deletion")
+	}
+
+	_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		FileKey:    fileKey,
+		Version:    uint32(needle.GetCurrentVersion()),
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcBlobDelete first delete failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcBlobDelete(ctx, &volume_server_pb.VolumeEcBlobDeleteRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		FileKey:    fileKey,
+		Version:    uint32(needle.GetCurrentVersion()),
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcBlobDelete second delete should be idempotent success, got: %v", err)
+	}
+
+	deletedStream, err := grpcClient.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
+		VolumeId: volumeID,
+		ShardId:  0,
+		FileKey:  fileKey,
+		Offset:   0,
+		Size:     1,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardRead deleted-check start failed: %v", err)
+	}
+	deletedMsg, err := deletedStream.Recv()
+	if err != nil {
+		t.Fatalf("VolumeEcShardRead deleted-check recv failed: %v", err)
+	}
+	if !deletedMsg.GetIsDeleted() {
+		t.Fatalf("VolumeEcShardRead expected IsDeleted=true after blob delete")
+	}
+	_, err = deletedStream.Recv()
+	if err != io.EOF {
+		t.Fatalf("VolumeEcShardRead deleted-check expected EOF after deleted marker, got: %v", err)
+	}
+}
+
+func TestEcRebuildMissingShardLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(117)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 990003, 0x3344DDEE)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-rebuild-shard-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsDelete shard 0 failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0},
+	})
+	if err == nil {
+		t.Fatalf("VolumeEcShardsMount should fail when shard 0 has been deleted")
+	}
+
+	rebuildResp, err := grpcClient.VolumeEcShardsRebuild(ctx, &volume_server_pb.VolumeEcShardsRebuildRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsRebuild failed: %v", err)
+	}
+	if len(rebuildResp.GetRebuiltShardIds()) == 0 {
+		t.Fatalf("VolumeEcShardsRebuild expected rebuilt shard ids")
+	}
+	foundShard0 := false
+	for _, shardID := range rebuildResp.GetRebuiltShardIds() {
+		if shardID == 0 {
+			foundShard0 = true
+			break
+		}
+	}
+	if !foundShard0 {
+		t.Fatalf("VolumeEcShardsRebuild expected shard 0 to be rebuilt, got %v", rebuildResp.GetRebuiltShardIds())
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount shard 0 after rebuild failed: %v", err)
+	}
+}
+
+func TestEcShardsToVolumeMissingShardAndNoLiveEntries(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	httpClient := framework.NewHTTPClient()
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	t.Run("missing shard returns error", func(t *testing.T) {
+		const volumeID = uint32(118)
+		framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+		fid := framework.NewFileID(volumeID, 990004, 0x4455EEFF)
+		uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-to-volume-missing-shard-content"))
+		_ = framework.ReadAllAndClose(t, uploadResp)
+		if uploadResp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+		}
+
+		_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+			VolumeId:   volumeID,
+			Collection: "",
+		})
+		if err != nil {
+			t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+		}
+
+		_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+			VolumeId:   volumeID,
+			Collection: "",
+			ShardIds:   []uint32{0},
+		})
+		if err != nil {
+			t.Fatalf("VolumeEcShardsDelete shard 0 failed: %v", err)
+		}
+
+		_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+			VolumeId:   volumeID,
+			Collection: "",
+			ShardIds:   []uint32{1},
+		})
+		if err != nil {
+			t.Fatalf("VolumeEcShardsMount shard 1 failed: %v", err)
+		}
+
+		_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+			VolumeId:   volumeID,
+			Collection: "",
+		})
+		if err == nil || !strings.Contains(err.Error(), "missing shard 0") {
+			t.Fatalf("VolumeEcShardsToVolume missing-shard error mismatch: %v", err)
+		}
+	})
+
+	t.Run("no live entries returns failed precondition", func(t *testing.T) {
+		const volumeID = uint32(119)
+		const needleID = uint64(990005)
+		const cookie = uint32(0x5566FF11)
+		framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+		fid := framework.NewFileID(volumeID, needleID, cookie)
+		uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-no-live-entries-content"))
+		_ = framework.ReadAllAndClose(t, uploadResp)
+		if uploadResp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+		}
+
+		deleteResp := framework.DoRequest(t, httpClient, mustNewRequest(t, http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid))
+		_ = framework.ReadAllAndClose(t, deleteResp)
+		if deleteResp.StatusCode != http.StatusAccepted {
+			t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode)
+		}
+
+		_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+			VolumeId:   volumeID,
+			Collection: "",
+		})
+		if err != nil {
+			t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+		}
+
+		_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+			VolumeId:   volumeID,
+			Collection: "",
+			ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+		})
+		if err != nil {
+			t.Fatalf("VolumeEcShardsMount data shards failed: %v", err)
+		}
+
+		_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+			VolumeId:   volumeID,
+			Collection: "",
+		})
+		if err == nil {
+			t.Fatalf("VolumeEcShardsToVolume expected failed-precondition error when no live entries")
+		}
+		if status.Code(err) != codes.FailedPrecondition {
+			t.Fatalf("VolumeEcShardsToVolume no-live-entries expected FailedPrecondition, got %v (%v)", status.Code(err), err)
+		}
+		if !strings.Contains(err.Error(), erasure_coding.EcNoLiveEntriesSubstring) {
+			t.Fatalf("VolumeEcShardsToVolume no-live-entries error should mention %q, got %v", erasure_coding.EcNoLiveEntriesSubstring, err)
+		}
+	})
+}
+
+func TestEcShardsToVolumeSuccessRoundTrip(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(120)
+	const needleID = uint64(990006)
+	const cookie = uint32(0x66771122)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	payload := []byte("ec-shards-to-volume-success-roundtrip-content")
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount data shards failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsToVolume(ctx, &volume_server_pb.VolumeEcShardsToVolumeRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsToVolume success path failed: %v", err)
+	}
+
+	readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid)
+	readBody := framework.ReadAllAndClose(t, readResp)
+	if readResp.StatusCode != http.StatusOK {
+		t.Fatalf("post-conversion read expected 200, got %d", readResp.StatusCode)
+	}
+	if string(readBody) != string(payload) {
+		t.Fatalf("post-conversion payload mismatch: got %q want %q", string(readBody), string(payload))
+	}
+}
+
+func TestEcShardsDeleteLastShardRemovesEcx(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(121)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 990007, 0x77882233)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-delete-all-shards-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	// Verify .ecx is present before deleting all shards.
+	ecxBeforeDelete, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:           volumeID,
+		Collection:         "",
+		IsEcVolume:         true,
+		Ext:                ".ecx",
+		CompactionRevision: math.MaxUint32,
+		StopOffset:         1,
+	})
+	if err != nil {
+		t.Fatalf("CopyFile .ecx before shard deletion start failed: %v", err)
+	}
+	if _, err = ecxBeforeDelete.Recv(); err != nil {
+		t.Fatalf("CopyFile .ecx before shard deletion recv failed: %v", err)
+	}
+
+	_, err = grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsDelete all shards failed: %v", err)
+	}
+
+	ecxAfterDelete, err := grpcClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+		VolumeId:           volumeID,
+		Collection:         "",
+		IsEcVolume:         true,
+		Ext:                ".ecx",
+		CompactionRevision: math.MaxUint32,
+		StopOffset:         1,
+	})
+	if err == nil {
+		_, err = ecxAfterDelete.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found ec volume id") {
+		t.Fatalf("CopyFile .ecx after deleting all shards should fail not-found, got: %v", err)
+	}
+}
+
+func TestEcShardsCopyFromPeerSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
+	sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer sourceConn.Close()
+	destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
+	defer destConn.Close()
+
+	const volumeID = uint32(122)
+	framework.AllocateVolume(t, sourceClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 990008, 0x88993344)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, []byte("ec-copy-from-peer-content"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("source upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := sourceClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("source VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	sourceDataNode := clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1]
+	_, err = destClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       volumeID,
+		Collection:     "",
+		SourceDataNode: sourceDataNode,
+		ShardIds:       []uint32{0},
+		CopyEcxFile:    true,
+		CopyVifFile:    true,
+	})
+	if err != nil {
+		t.Fatalf("destination VolumeEcShardsCopy success path failed: %v", err)
+	}
+
+	for _, ext := range []string{".ec00", ".ecx", ".vif"} {
+		copyStream, copyErr := destClient.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+			VolumeId:           volumeID,
+			Collection:         "",
+			IsEcVolume:         true,
+			Ext:                ext,
+			CompactionRevision: math.MaxUint32,
+			StopOffset:         1,
+		})
+		if copyErr != nil {
+			t.Fatalf("destination CopyFile %s start failed: %v", ext, copyErr)
+		}
+		if _, copyErr = copyStream.Recv(); copyErr != nil {
+			t.Fatalf("destination CopyFile %s recv failed: %v", ext, copyErr)
+		}
+	}
+}
+
+// TestEcIndexConsistencyAfterEncode verifies that every needle indexed in .ecx
+// can be read back correctly from EC shards after VolumeEcShardsGenerate.
+// This catches the race condition fixed in this PR where .ecx could reference
+// data not present in EC shards.
+func TestEcIndexConsistencyAfterEncode(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(130)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+
+	// Upload multiple needles of varying sizes
+	type testNeedle struct {
+		fid     string
+		payload []byte
+	}
+	needles := []testNeedle{
+		{framework.NewFileID(volumeID, 1001, 0xAABB0001), []byte("small-needle-1")},
+		{framework.NewFileID(volumeID, 1002, 0xAABB0002), make([]byte, 1024)},     // 1KB
+		{framework.NewFileID(volumeID, 1003, 0xAABB0003), make([]byte, 64*1024)},  // 64KB
+		{framework.NewFileID(volumeID, 1004, 0xAABB0004), make([]byte, 256*1024)}, // 256KB
+		{framework.NewFileID(volumeID, 1005, 0xAABB0005), []byte("small-needle-2")},
+	}
+
+	// Fill larger payloads with recognizable data
+	for i := range needles {
+		for j := range needles[i].payload {
+			needles[i].payload[j] = byte(i*37 + j%251)
+		}
+	}
+
+	for _, n := range needles {
+		resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid, n.payload)
+		_ = framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s expected 201, got %d", n.fid, resp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// EC encode
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	// Mount all data shards so reads go through the EC path
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount failed: %v", err)
+	}
+
+	// Read every needle back from EC shards and verify payload
+	for _, n := range needles {
+		readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid)
+		readBody := framework.ReadAllAndClose(t, readResp)
+		if readResp.StatusCode != http.StatusOK {
+			t.Fatalf("EC read %s expected 200, got %d", n.fid, readResp.StatusCode)
+		}
+		if string(readBody) != string(n.payload) {
+			t.Fatalf("EC read %s payload mismatch: got %d bytes, want %d bytes", n.fid, len(readBody), len(n.payload))
+		}
+	}
+}
+
+func TestEcShardsCopyFailsWhenSourceUnavailable(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+		VolumeId:       12345,
+		Collection:     "",
+		SourceDataNode: "127.0.0.1:1.1",
+		ShardIds:       []uint32{0},
+		CopyEcxFile:    true,
+	})
+	if err == nil || !strings.Contains(err.Error(), "VolumeEcShardsCopy volume") {
+		t.Fatalf("VolumeEcShardsCopy source-unavailable error mismatch: %v", err)
+	}
+}

test/volume_server/grpc/fetch_remote_s3_test.go

@@ -0,0 +1,266 @@
+package volume_server_grpc_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+// findAvailablePort finds a free TCP port on localhost.
+
+// startWeedMini starts a weed mini subprocess and returns the S3 endpoint and cleanup func.
+func startWeedMini(t *testing.T) (s3Endpoint string, cleanup func()) {
+	t.Helper()
+
+	weedBin, err := exec.LookPath("weed")
+	if err != nil {
+		weedBin = filepath.Join("..", "..", "..", "weed", "weed_binary")
+		if _, err := os.Stat(weedBin); os.IsNotExist(err) {
+			t.Skip("weed binary not found, skipping S3 remote storage test")
+		}
+	}
+
+	miniPorts := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3"})
+	miniMasterPort := miniPorts[0]
+	miniVolumePort := miniPorts[1]
+	miniFilerPort := miniPorts[2]
+	miniS3Port := miniPorts[3]
+	miniDir := t.TempDir()
+	os.WriteFile(filepath.Join(miniDir, "security.toml"), []byte("# empty\n"), 0644)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	miniCmd := exec.CommandContext(ctx, weedBin, "mini",
+		fmt.Sprintf("-dir=%s", miniDir),
+		fmt.Sprintf("-master.port=%d", miniMasterPort),
+		fmt.Sprintf("-volume.port=%d", miniVolumePort),
+		fmt.Sprintf("-filer.port=%d", miniFilerPort),
+		fmt.Sprintf("-s3.port=%d", miniS3Port),
+	)
+	miniCmd.Env = append(os.Environ(), "AWS_ACCESS_KEY_ID=admin", "AWS_SECRET_ACCESS_KEY=admin")
+	miniCmd.Dir = miniDir
+	logFile, _ := os.CreateTemp("", "weed-mini-*.log")
+	miniCmd.Stdout = logFile
+	miniCmd.Stderr = logFile
+	t.Logf("weed mini logs at %s", logFile.Name())
+
+	if err := miniCmd.Start(); err != nil {
+		cancel()
+		logFile.Close()
+		t.Fatalf("start weed mini: %v", err)
+	}
+
+	if !testutil.WaitForPort(miniS3Port, 30*time.Second) {
+		cancel()
+		miniCmd.Wait()
+		logFile.Close()
+		t.Fatalf("weed mini S3 not ready on port %d", miniS3Port)
+	}
+	t.Logf("weed mini S3 ready on port %d", miniS3Port)
+
+	return fmt.Sprintf("http://127.0.0.1:%d", miniS3Port), func() {
+		cancel()
+		miniCmd.Wait()
+		logFile.Close()
+	}
+}
+
+func newS3Client(endpoint string) *s3.S3 {
+	sess, _ := session.NewSession(&aws.Config{
+		Region:           aws.String("us-east-1"),
+		Endpoint:         aws.String(endpoint),
+		Credentials:      credentials.NewStaticCredentials("admin", "admin", ""),
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+	})
+	return s3.New(sess)
+}
+
+// TestFetchAndWriteNeedleFromS3 tests the full FetchAndWriteNeedle flow:
+// 1. Start a weed mini instance as S3 backend
+// 2. Upload a test object to it via S3 API
+// 3. Call FetchAndWriteNeedle on the volume server to fetch from S3
+// 4. Verify the response contains a valid e_tag
+func TestFetchAndWriteNeedleFromS3(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	s3Endpoint, cleanupMini := startWeedMini(t)
+	defer cleanupMini()
+
+	s3Client := newS3Client(s3Endpoint)
+
+	// Create bucket and upload test data
+	bucket := "test-remote-fetch"
+	s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
+
+	testData := []byte("Hello from S3 remote storage! This is test data for FetchAndWriteNeedle.")
+	testKey := "test-object.dat"
+	_, err := s3Client.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(testKey),
+		Body:   bytes.NewReader(testData),
+	})
+	if err != nil {
+		t.Fatalf("put object: %v", err)
+	}
+	t.Logf("uploaded %d bytes to s3://%s/%s", len(testData), bucket, testKey)
+
+	// Start volume server
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(99)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	grpcCtx, grpcCancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer grpcCancel()
+
+	// FetchAndWriteNeedle from S3
+	resp, err := grpcClient.FetchAndWriteNeedle(grpcCtx, &volume_server_pb.FetchAndWriteNeedleRequest{
+		VolumeId: volumeID,
+		NeedleId: 42,
+		Cookie:   12345,
+		Offset:   0,
+		Size:     int64(len(testData)),
+		RemoteConf: &remote_pb.RemoteConf{
+			Name:             "test-s3",
+			Type:             "s3",
+			S3AccessKey:      "admin",
+			S3SecretKey:      "admin",
+			S3Region:         "us-east-1",
+			S3Endpoint:       s3Endpoint,
+			S3ForcePathStyle: true,
+		},
+		RemoteLocation: &remote_pb.RemoteStorageLocation{
+			Name:   "test-s3",
+			Bucket: bucket,
+			Path:   "/" + testKey,
+		},
+	})
+	if err != nil {
+		t.Fatalf("FetchAndWriteNeedle failed: %v", err)
+	}
+	if resp.GetETag() == "" {
+		t.Fatal("FetchAndWriteNeedle returned empty e_tag")
+	}
+	t.Logf("FetchAndWriteNeedle success: e_tag=%s", resp.GetETag())
+}
+
+// TestFetchAndWriteNeedleFromS3WithPartialRead tests reading a byte range from S3.
+func TestFetchAndWriteNeedleFromS3WithPartialRead(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	s3Endpoint, cleanupMini := startWeedMini(t)
+	defer cleanupMini()
+
+	s3Client := newS3Client(s3Endpoint)
+
+	bucket := "partial-read-test"
+	s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
+
+	// Upload 1KB of data
+	fullData := make([]byte, 1024)
+	for i := range fullData {
+		fullData[i] = byte(i % 256)
+	}
+	s3Client.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String("big.dat"),
+		Body: bytes.NewReader(fullData),
+	})
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	framework.AllocateVolume(t, grpcClient, 98, "")
+
+	grpcCtx, grpcCancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer grpcCancel()
+
+	// Fetch only bytes 100-199 (100 bytes) from the 1KB object
+	resp, err := grpcClient.FetchAndWriteNeedle(grpcCtx, &volume_server_pb.FetchAndWriteNeedleRequest{
+		VolumeId: 98, NeedleId: 7, Cookie: 999,
+		Offset: 100, Size: 100,
+		RemoteConf: &remote_pb.RemoteConf{
+			Name: "test-s3-partial", Type: "s3",
+			S3AccessKey: "admin", S3SecretKey: "admin",
+			S3Region: "us-east-1", S3Endpoint: s3Endpoint, S3ForcePathStyle: true,
+		},
+		RemoteLocation: &remote_pb.RemoteStorageLocation{
+			Name: "test-s3-partial", Bucket: bucket, Path: "/big.dat",
+		},
+	})
+	if err != nil {
+		t.Fatalf("FetchAndWriteNeedle partial read failed: %v", err)
+	}
+	if resp.GetETag() == "" {
+		t.Fatal("empty e_tag for partial read")
+	}
+	t.Logf("FetchAndWriteNeedle partial read success: e_tag=%s", resp.GetETag())
+}
+
+// TestFetchAndWriteNeedleS3NotFound tests that fetching a non-existent S3 object returns an error.
+func TestFetchAndWriteNeedleS3NotFound(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	s3Endpoint, cleanupMini := startWeedMini(t)
+	defer cleanupMini()
+
+	s3Client := newS3Client(s3Endpoint)
+
+	bucket := "notfound-test"
+	s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	framework.AllocateVolume(t, grpcClient, 97, "")
+
+	grpcCtx, grpcCancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer grpcCancel()
+
+	_, err := grpcClient.FetchAndWriteNeedle(grpcCtx, &volume_server_pb.FetchAndWriteNeedleRequest{
+		VolumeId: 97, NeedleId: 1, Cookie: 1,
+		Offset: 0, Size: 100,
+		RemoteConf: &remote_pb.RemoteConf{
+			Name: "test-s3-nf", Type: "s3",
+			S3AccessKey: "admin", S3SecretKey: "admin",
+			S3Region: "us-east-1", S3Endpoint: s3Endpoint, S3ForcePathStyle: true,
+		},
+		RemoteLocation: &remote_pb.RemoteStorageLocation{
+			Name: "test-s3-nf", Bucket: bucket, Path: "/does-not-exist.dat",
+		},
+	})
+	if err == nil {
+		t.Fatal("FetchAndWriteNeedle should fail for non-existent object")
+	}
+	if !strings.Contains(err.Error(), "read from remote") {
+		t.Fatalf("expected 'read from remote' error, got: %v", err)
+	}
+	t.Logf("correctly got error for non-existent object: %v", err)
+}

test/volume_server/grpc/health_state_test.go

@@ -0,0 +1,139 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestStateAndStatusRPCs(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	initialState, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	if initialState.GetState() == nil {
+		t.Fatalf("GetState returned nil state")
+	}
+
+	setResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{
+			Maintenance: true,
+			Version:     initialState.GetState().GetVersion(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("SetState(maintenance=true) failed: %v", err)
+	}
+	if !setResp.GetState().GetMaintenance() {
+		t.Fatalf("expected maintenance=true after SetState")
+	}
+
+	setResp, err = client.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{
+			Maintenance: false,
+			Version:     setResp.GetState().GetVersion(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("SetState(maintenance=false) failed: %v", err)
+	}
+	if setResp.GetState().GetMaintenance() {
+		t.Fatalf("expected maintenance=false after SetState")
+	}
+
+	statusResp, err := client.VolumeServerStatus(ctx, &volume_server_pb.VolumeServerStatusRequest{})
+	if err != nil {
+		t.Fatalf("VolumeServerStatus failed: %v", err)
+	}
+	if statusResp.GetVersion() == "" {
+		t.Fatalf("VolumeServerStatus returned empty version")
+	}
+	if len(statusResp.GetDiskStatuses()) == 0 {
+		t.Fatalf("VolumeServerStatus returned no disk statuses")
+	}
+	if statusResp.GetState() == nil {
+		t.Fatalf("VolumeServerStatus returned nil state")
+	}
+	if statusResp.GetMemoryStatus() == nil {
+		t.Fatalf("VolumeServerStatus returned nil memory status")
+	}
+	if statusResp.GetMemoryStatus().GetGoroutines() <= 0 {
+		t.Fatalf("VolumeServerStatus memory status should report goroutines, got %d", statusResp.GetMemoryStatus().GetGoroutines())
+	}
+
+	pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{})
+	if err != nil {
+		t.Fatalf("Ping failed: %v", err)
+	}
+	if pingResp.GetStartTimeNs() == 0 || pingResp.GetStopTimeNs() == 0 {
+		t.Fatalf("Ping timestamps should be non-zero: %+v", pingResp)
+	}
+	if pingResp.GetStopTimeNs() < pingResp.GetStartTimeNs() {
+		t.Fatalf("Ping stop time should be >= start time: %+v", pingResp)
+	}
+}
+
+func TestSetStateVersionMismatchAndNilStateNoop(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartVolumeCluster(t, matrix.P1())
+	conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	initialState, err := client.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	initialVersion := initialState.GetState().GetVersion()
+
+	staleResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{
+			Maintenance: true,
+			Version:     initialVersion + 1,
+		},
+	})
+	if err == nil {
+		t.Fatalf("SetState with stale version should fail")
+	}
+	if !strings.Contains(err.Error(), "version mismatch") {
+		t.Fatalf("SetState stale version error mismatch: %v", err)
+	}
+	if staleResp.GetState().GetVersion() != initialVersion {
+		t.Fatalf("SetState stale version should not mutate server version: got %d want %d", staleResp.GetState().GetVersion(), initialVersion)
+	}
+	if staleResp.GetState().GetMaintenance() != initialState.GetState().GetMaintenance() {
+		t.Fatalf("SetState stale version should not mutate maintenance flag")
+	}
+
+	nilResp, err := client.SetState(ctx, &volume_server_pb.SetStateRequest{})
+	if err != nil {
+		t.Fatalf("SetState nil-state request should be no-op success: %v", err)
+	}
+	if nilResp.GetState().GetVersion() != initialVersion {
+		t.Fatalf("SetState nil-state should keep version unchanged: got %d want %d", nilResp.GetState().GetVersion(), initialVersion)
+	}
+	if nilResp.GetState().GetMaintenance() != initialState.GetState().GetMaintenance() {
+		t.Fatalf("SetState nil-state should keep maintenance unchanged")
+	}
+}

test/volume_server/grpc/mixed_balance_test.go

@@ -0,0 +1,567 @@
+package volume_server_grpc_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+// deleteAndWaitForTombstone issues an HTTP DELETE for the given fid on
+// volumeURL, asserts a successful response, then polls with GET until
+// the file returns 404 (tombstone visible) or the timeout elapses.
+func deleteAndWaitForTombstone(t *testing.T, httpClient *http.Client, volumeURL, fid string) {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/%s", volumeURL, fid), nil)
+	if err != nil {
+		t.Fatalf("build delete request for %s: %v", fid, err)
+	}
+	resp := framework.DoRequest(t, httpClient, req)
+	framework.ReadAllAndClose(t, resp)
+	if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK {
+		t.Fatalf("delete %s: expected 200 or 202, got %d", fid, resp.StatusCode)
+	}
+	// Poll until GET returns 404 (tombstone flushed to disk).
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		getResp := framework.ReadBytes(t, httpClient, volumeURL, fid)
+		status := getResp.StatusCode
+		framework.ReadAllAndClose(t, getResp)
+		if status == http.StatusNotFound {
+			return
+		}
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Fatalf("delete %s: tombstone not visible after 5s", fid)
+}
+
+// TestMixedBalanceCopyGoToRust verifies that VolumeCopy works from a Go volume
+// server to a Rust volume server. This is the core operation behind volume
+// balancing in a mixed Go+Rust cluster.
+func TestMixedBalanceCopyGoToRust(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
+
+	// server 0 = Go, server 1 = Rust
+	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	httpClient := framework.NewHTTPClient()
+
+	const volumeID = uint32(50)
+
+	// Allocate volume on Go server and upload test data
+	framework.AllocateVolume(t, goClient, volumeID, "")
+
+	testFiles := []struct {
+		key    uint64
+		cookie uint32
+		data   []byte
+	}{
+		{1, 0xAABBCCDD, []byte("hello from Go server")},
+		{2, 0x11223344, []byte("second file for balance test")},
+		{3, 0xDEADBEEF, make([]byte, 4096)}, // larger file
+	}
+
+	for _, f := range testFiles {
+		fid := framework.NewFileID(volumeID, f.key, f.cookie)
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, f.data)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s: expected 201, got %d: %s", fid, resp.StatusCode, body)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// Read source volume status before copy
+	sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Go server: %v", err)
+	}
+	t.Logf("Source: dat=%d idx=%d files=%d version=%d",
+		sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
+		sourceStatus.GetFileCount(), sourceStatus.GetVersion())
+
+	// Copy volume from Go (server 0) to Rust (server 1)
+	copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		SourceDataNode: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy call failed: %v", err)
+	}
+
+	var lastAppendAtNs uint64
+	for {
+		resp, recvErr := copyStream.Recv()
+		if recvErr != nil {
+			if recvErr != io.EOF {
+				t.Fatalf("VolumeCopy recv error: %v", recvErr)
+			}
+			break
+		}
+		if resp.GetLastAppendAtNs() != 0 {
+			lastAppendAtNs = resp.GetLastAppendAtNs()
+		}
+	}
+	t.Logf("VolumeCopy completed, lastAppendAtNs=%d", lastAppendAtNs)
+
+	// Verify: read volume status from Rust server
+	targetStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Rust server: %v", err)
+	}
+	t.Logf("Target: dat=%d idx=%d files=%d version=%d",
+		targetStatus.GetDatFileSize(), targetStatus.GetIdxFileSize(),
+		targetStatus.GetFileCount(), targetStatus.GetVersion())
+
+	if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
+		t.Fatalf("dat file size mismatch: source=%d target=%d",
+			sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
+	}
+	if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
+		t.Fatalf("idx file size mismatch: source=%d target=%d",
+			sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
+	}
+	if sourceStatus.GetFileCount() != targetStatus.GetFileCount() {
+		t.Fatalf("file count mismatch: source=%d target=%d",
+			sourceStatus.GetFileCount(), targetStatus.GetFileCount())
+	}
+
+	// Verify data can be read from Rust server
+	for _, f := range testFiles {
+		fid := framework.NewFileID(volumeID, f.key, f.cookie)
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("read %s from Rust server: expected 200, got %d", fid, resp.StatusCode)
+		}
+		if !bytes.Equal(body, f.data) {
+			t.Fatalf("read %s from Rust server: content mismatch (got %d bytes, want %d)", fid, len(body), len(f.data))
+		}
+	}
+}
+
+// TestMixedBalanceCopyRustToGo verifies that VolumeCopy works from a Rust
+// volume server to a Go volume server.
+func TestMixedBalanceCopyRustToGo(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
+
+	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	httpClient := framework.NewHTTPClient()
+
+	const volumeID = uint32(51)
+
+	// Allocate volume on Rust server and upload test data
+	framework.AllocateVolume(t, rustClient, volumeID, "")
+
+	testFiles := []struct {
+		key    uint64
+		cookie uint32
+		data   []byte
+	}{
+		{1, 0xAABBCCDD, []byte("hello from Rust server")},
+		{2, 0x11223344, []byte("second file for reverse balance")},
+	}
+
+	for _, f := range testFiles {
+		fid := framework.NewFileID(volumeID, f.key, f.cookie)
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid, f.data)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload %s: expected 201, got %d: %s", fid, resp.StatusCode, body)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	sourceStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Rust server: %v", err)
+	}
+
+	// Copy volume from Rust (server 1) to Go (server 0)
+	copyStream, err := goClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		SourceDataNode: cluster.VolumeServerAddress(1),
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy call failed: %v", err)
+	}
+	for {
+		_, recvErr := copyStream.Recv()
+		if recvErr != nil {
+			if recvErr != io.EOF {
+				t.Fatalf("VolumeCopy recv error: %v", recvErr)
+			}
+			break
+		}
+	}
+
+	targetStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Go server: %v", err)
+	}
+
+	if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
+		t.Fatalf("dat file size mismatch: source=%d target=%d",
+			sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
+	}
+	if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
+		t.Fatalf("idx file size mismatch: source=%d target=%d",
+			sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
+	}
+	if sourceStatus.GetFileCount() != targetStatus.GetFileCount() {
+		t.Fatalf("file count mismatch: source=%d target=%d",
+			sourceStatus.GetFileCount(), targetStatus.GetFileCount())
+	}
+
+	// Verify data can be read from Go server
+	for _, f := range testFiles {
+		fid := framework.NewFileID(volumeID, f.key, f.cookie)
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("read %s from Go server: expected 200, got %d", fid, resp.StatusCode)
+		}
+		if !bytes.Equal(body, f.data) {
+			t.Fatalf("read %s from Go server: content mismatch (got %d bytes, want %d)", fid, len(body), len(f.data))
+		}
+	}
+}
+
+// TestMixedBalanceCopyWithDeletes verifies that VolumeCopy correctly handles
+// volumes that have both active and deleted needles.
+func TestMixedBalanceCopyWithDeletes(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
+
+	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	httpClient := framework.NewHTTPClient()
+
+	const volumeID = uint32(52)
+	framework.AllocateVolume(t, goClient, volumeID, "")
+
+	// Upload files
+	for i := uint64(1); i <= 5; i++ {
+		fid := framework.NewFileID(volumeID, i, 0x12340000+uint32(i))
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, []byte(fmt.Sprintf("file-%d", i)))
+		framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload file %d: expected 201, got %d", i, resp.StatusCode)
+		}
+	}
+
+	// Delete some files and wait for tombstones to be visible
+	for _, key := range []uint64{2, 4} {
+		fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
+		deleteAndWaitForTombstone(t, httpClient, cluster.VolumeAdminURL(0), fid)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus: %v", err)
+	}
+	t.Logf("Source after deletes: dat=%d idx=%d files=%d",
+		sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
+		sourceStatus.GetFileCount())
+
+	// Copy volume from Go to Rust
+	copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		SourceDataNode: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy: %v", err)
+	}
+	var lastAppendAtNs uint64
+	for {
+		resp, recvErr := copyStream.Recv()
+		if recvErr != nil {
+			if recvErr != io.EOF {
+				t.Fatalf("VolumeCopy recv error: %v", recvErr)
+			}
+			break
+		}
+		if resp.GetLastAppendAtNs() != 0 {
+			lastAppendAtNs = resp.GetLastAppendAtNs()
+		}
+	}
+	if lastAppendAtNs == 0 {
+		t.Fatalf("VolumeCopy did not return a lastAppendAtNs timestamp")
+	}
+	t.Logf("VolumeCopy completed, lastAppendAtNs=%d", lastAppendAtNs)
+
+	targetStatusAfterCopy, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus on Rust: %v", err)
+	}
+	t.Logf("Target after copy: dat=%d idx=%d files=%d",
+		targetStatusAfterCopy.GetDatFileSize(), targetStatusAfterCopy.GetIdxFileSize(),
+		targetStatusAfterCopy.GetFileCount())
+
+	if sourceStatus.GetDatFileSize() != targetStatusAfterCopy.GetDatFileSize() {
+		t.Fatalf("dat file size mismatch: source=%d target=%d",
+			sourceStatus.GetDatFileSize(), targetStatusAfterCopy.GetDatFileSize())
+	}
+	if sourceStatus.GetIdxFileSize() != targetStatusAfterCopy.GetIdxFileSize() {
+		t.Fatalf("idx file size mismatch: source=%d target=%d",
+			sourceStatus.GetIdxFileSize(), targetStatusAfterCopy.GetIdxFileSize())
+	}
+
+	// Tail from the copy checkpoint — source is unchanged (deletes happened
+	// before copy), so tailing should not append any data.
+	_, err = rustClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
+		VolumeId:           volumeID,
+		SinceNs:            lastAppendAtNs,
+		IdleTimeoutSeconds: 3,
+		SourceVolumeServer: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeTailReceiver: %v", err)
+	}
+
+	targetStatusAfterTail, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus after tail: %v", err)
+	}
+	if targetStatusAfterTail.GetDatFileSize() != targetStatusAfterCopy.GetDatFileSize() {
+		t.Fatalf("dat grew after tail: before=%d after=%d",
+			targetStatusAfterCopy.GetDatFileSize(), targetStatusAfterTail.GetDatFileSize())
+	}
+	if targetStatusAfterTail.GetIdxFileSize() != targetStatusAfterCopy.GetIdxFileSize() {
+		t.Fatalf("idx grew after tail: before=%d after=%d",
+			targetStatusAfterCopy.GetIdxFileSize(), targetStatusAfterTail.GetIdxFileSize())
+	}
+
+	// Verify surviving files are readable from Rust
+	for _, key := range []uint64{1, 3, 5} {
+		fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("read surviving file %d from Rust: expected 200, got %d", key, resp.StatusCode)
+		}
+		expected := fmt.Sprintf("file-%d", key)
+		if string(body) != expected {
+			t.Fatalf("data mismatch for file %d: got %q want %q", key, body, expected)
+		}
+	}
+
+	// Verify deleted files return 404
+	for _, key := range []uint64{2, 4} {
+		fid := framework.NewFileID(volumeID, key, 0x12340000+uint32(key))
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusNotFound {
+			t.Fatalf("read deleted file %d from Rust: expected 404, got %d", key, resp.StatusCode)
+		}
+	}
+}
+
+// TestMixedBalanceFullMoveGoToRust exercises the complete volume balance move
+// flow: mark readonly → copy → tail → verify sizes → delete source.
+// This mirrors the steps in balance_task.go Execute().
+func TestMixedBalanceFullMoveGoToRust(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMixedVolumeCluster(t, matrix.P1(), 1, 1)
+
+	conn0, goClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer conn0.Close()
+	conn1, rustClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer conn1.Close()
+
+	httpClient := framework.NewHTTPClient()
+
+	const volumeID = uint32(53)
+	framework.AllocateVolume(t, goClient, volumeID, "")
+
+	// Upload test data
+	for i := uint64(1); i <= 5; i++ {
+		fid := framework.NewFileID(volumeID, i, 0xABCD0000+uint32(i))
+		resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, []byte(fmt.Sprintf("balance-move-file-%d", i)))
+		framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload file %d: expected 201, got %d", i, resp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// Step 1: Copy volume to Rust server (source stays writable so we can
+	// delete after copy to exercise the tail tombstone path)
+	copyStream, err := rustClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		SourceDataNode: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy: %v", err)
+	}
+	var lastAppendAtNs uint64
+	for {
+		resp, recvErr := copyStream.Recv()
+		if recvErr != nil {
+			if recvErr != io.EOF {
+				t.Fatalf("VolumeCopy recv: %v", recvErr)
+			}
+			break
+		}
+		if resp.GetLastAppendAtNs() != 0 {
+			lastAppendAtNs = resp.GetLastAppendAtNs()
+		}
+	}
+	t.Logf("Copy done, lastAppendAtNs=%d", lastAppendAtNs)
+
+	// Step 2: Delete file 3 on the source AFTER copy. This creates a
+	// tombstone needle that the tail step must propagate to the Rust server.
+	deleteAndWaitForTombstone(t, httpClient, cluster.VolumeAdminURL(0),
+		framework.NewFileID(volumeID, 3, 0xABCD0003))
+
+	// Step 3: Mark source readonly so no further writes arrive during tail
+	_, err = goClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("mark readonly: %v", err)
+	}
+
+	// Read source status (the reference for post-tail verification)
+	sourceStatus, err := goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("read source status: %v", err)
+	}
+	t.Logf("Source: dat=%d idx=%d files=%d",
+		sourceStatus.GetDatFileSize(), sourceStatus.GetIdxFileSize(),
+		sourceStatus.GetFileCount())
+
+	// Step 4: Tail for updates — this must pick up the delete tombstone
+	_, err = rustClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
+		VolumeId:           volumeID,
+		SinceNs:            lastAppendAtNs,
+		IdleTimeoutSeconds: 5,
+		SourceVolumeServer: cluster.VolumeServerAddress(0),
+	})
+	if err != nil {
+		t.Fatalf("VolumeTailReceiver: %v", err)
+	}
+
+	// Step 5: Verify dat/idx sizes match after tail.
+	// We compare file sizes (byte-level correctness) rather than file_count
+	// because the tail writes the tombstone via the write path (which
+	// increments file_count) while the source used the delete path (which
+	// only increments deletion_count). This is consistent with Go behavior.
+	targetStatus, err := rustClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("read target status: %v", err)
+	}
+	t.Logf("Target: dat=%d idx=%d files=%d",
+		targetStatus.GetDatFileSize(), targetStatus.GetIdxFileSize(),
+		targetStatus.GetFileCount())
+
+	if sourceStatus.GetDatFileSize() != targetStatus.GetDatFileSize() {
+		t.Fatalf("dat size mismatch after tail: source=%d target=%d",
+			sourceStatus.GetDatFileSize(), targetStatus.GetDatFileSize())
+	}
+	if sourceStatus.GetIdxFileSize() != targetStatus.GetIdxFileSize() {
+		t.Fatalf("idx size mismatch after tail: source=%d target=%d",
+			sourceStatus.GetIdxFileSize(), targetStatus.GetIdxFileSize())
+	}
+
+	// Step 6: Delete volume from source
+	_, err = goClient.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
+		VolumeId: volumeID,
+	})
+	if err != nil {
+		t.Fatalf("delete source volume: %v", err)
+	}
+
+	// Verify source volume is gone
+	_, err = goClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+		VolumeId: volumeID,
+	})
+	if err == nil {
+		t.Fatalf("expected error reading deleted source volume, got nil")
+	}
+	t.Logf("Source volume deleted successfully (error as expected: %v)", err)
+
+	// Verify all surviving data is readable from Rust (the new home)
+	for _, key := range []uint64{1, 2, 4, 5} {
+		fid := framework.NewFileID(volumeID, key, 0xABCD0000+uint32(key))
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("read file %d from Rust after move: expected 200, got %d", key, resp.StatusCode)
+		}
+		expected := fmt.Sprintf("balance-move-file-%d", key)
+		if string(body) != expected {
+			t.Fatalf("data mismatch for file %d after move: got %q want %q", key, body, expected)
+		}
+	}
+
+	// Verify deleted file is 404 on Rust (tombstone propagated via tail)
+	{
+		fid := framework.NewFileID(volumeID, 3, 0xABCD0003)
+		resp := framework.ReadBytes(t, httpClient, cluster.VolumeAdminURL(1), fid)
+		framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusNotFound {
+			t.Fatalf("deleted file 3 on Rust: expected 404, got %d", resp.StatusCode)
+		}
+	}
+
+	t.Logf("Full balance move completed: volume %d moved from Go to Rust, source purged", volumeID)
+}

test/volume_server/grpc/move_tail_timestamp_test.go

@@ -0,0 +1,285 @@
+package volume_server_grpc_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestVolumeCopyReturnsPreciseLastAppendTimestamp(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMultiVolumeClusterAuto(t, matrix.P1(), 2)
+	sourceConn, sourceClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer sourceConn.Close()
+	destConn, destClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer destConn.Close()
+
+	const volumeID = uint32(999)
+	framework.AllocateVolume(t, sourceClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 1, 0x42)
+	payload := []byte("move-tail-timestamp-payload")
+	uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("source upload failed: status %d", uploadResp.StatusCode)
+	}
+
+	sourceDir := filepath.Join(cluster.BaseDir(), "volume0")
+	datPath := storage.VolumeFileName(sourceDir, "", int(volumeID)) + ".dat"
+	futureTime := time.Now().Add(2 * time.Hour)
+	if err := os.Chtimes(datPath, futureTime, futureTime); err != nil {
+		t.Fatalf("set future dat timestamp: %v", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	sourceDataNode := cluster.VolumeAdminAddress(0) + "." + strings.Split(cluster.VolumeGRPCAddress(0), ":")[1]
+	copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		SourceDataNode: sourceDataNode,
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy start failed: %v", err)
+	}
+
+	var lastAppendAtNs uint64
+	for {
+		resp, recvErr := copyStream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("VolumeCopy recv failed: %v", recvErr)
+		}
+		if ts := resp.GetLastAppendAtNs(); ts > 0 {
+			lastAppendAtNs = ts
+		}
+	}
+	if lastAppendAtNs == 0 {
+		t.Fatalf("volume copy did not return a last append timestamp")
+	}
+
+	destDir := filepath.Join(cluster.BaseDir(), "volume1")
+	actualLastAppend := readLastAppendAtNs(t, destDir, volumeID)
+	if actualLastAppend == 0 {
+		t.Fatalf("failed to compute last append timestamp from destination files")
+	}
+
+	if lastAppendAtNs != actualLastAppend {
+		t.Fatalf("last append timestamp mismatch: got %d, actual %d", lastAppendAtNs, actualLastAppend)
+	}
+}
+
+func readLastAppendAtNs(t testing.TB, volumeDir string, volumeID uint32) uint64 {
+	t.Helper()
+
+	baseName := storage.VolumeFileName(volumeDir, "", int(volumeID))
+	idxPath := baseName + ".idx"
+	idxFile, err := os.Open(idxPath)
+	if err != nil {
+		t.Fatalf("open idx file %s: %v", idxPath, err)
+	}
+	defer idxFile.Close()
+
+	stat, err := idxFile.Stat()
+	if err != nil {
+		t.Fatalf("stat idx file %s: %v", idxPath, err)
+	}
+	if stat.Size() == 0 {
+		return 0
+	}
+	if stat.Size()%int64(types.NeedleMapEntrySize) != 0 {
+		t.Fatalf("unexpected idx file size %d", stat.Size())
+	}
+
+	buf := make([]byte, types.NeedleMapEntrySize)
+	if _, err := idxFile.ReadAt(buf, stat.Size()-int64(types.NeedleMapEntrySize)); err != nil {
+		t.Fatalf("read idx entry: %v", err)
+	}
+	_, offset, _ := idx.IdxFileEntry(buf)
+	if offset.IsZero() {
+		return 0
+	}
+
+	datPath := baseName + ".dat"
+	datFile, err := os.Open(datPath)
+	if err != nil {
+		t.Fatalf("open dat file %s: %v", datPath, err)
+	}
+	defer datFile.Close()
+
+	datBackend := backend.NewDiskFile(datFile)
+	n, _, _, err := needle.ReadNeedleHeader(datBackend, needle.GetCurrentVersion(), offset.ToActualOffset())
+	if err != nil {
+		t.Fatalf("read needle header: %v", err)
+	}
+
+	tailOffset := offset.ToActualOffset() + int64(types.NeedleHeaderSize) + int64(n.Size)
+	tail := make([]byte, needle.NeedleChecksumSize+types.TimestampSize)
+	readCount, readErr := datBackend.ReadAt(tail, tailOffset)
+	if readErr == io.EOF && readCount == len(tail) {
+		readErr = nil
+	}
+	if readErr != nil {
+		t.Fatalf("read needle tail: %v", readErr)
+	}
+
+	return util.BytesToUint64(tail[needle.NeedleChecksumSize : needle.NeedleChecksumSize+types.TimestampSize])
+}
+
+func TestVolumeMoveHandlesInFlightWrites(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	cluster := framework.StartMultiVolumeClusterAuto(t, matrix.P1(), 2)
+	sourceConn, sourceClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
+	defer sourceConn.Close()
+	destConn, destClient := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
+	defer destConn.Close()
+
+	const volumeID = uint32(988)
+	framework.AllocateVolume(t, sourceClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 7, 0xABCDEF01)
+	payload := []byte("volume-move-live-payload")
+	uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("initial upload failed: %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	sourceDataNode := cluster.VolumeAdminAddress(0) + "." + strings.Split(cluster.VolumeGRPCAddress(0), ":")[1]
+	copyStream, err := destClient.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+		VolumeId:       volumeID,
+		SourceDataNode: sourceDataNode,
+	})
+	if err != nil {
+		t.Fatalf("VolumeCopy start failed: %v", err)
+	}
+
+	var lastAppendAtNs uint64
+	for {
+		resp, recvErr := copyStream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("VolumeCopy recv failed: %v", recvErr)
+		}
+		if ts := resp.GetLastAppendAtNs(); ts > 0 {
+			lastAppendAtNs = ts
+		}
+	}
+	if lastAppendAtNs == 0 {
+		t.Fatalf("volume copy did not return a last append timestamp")
+	}
+
+	type written struct {
+		fid  string
+		data []byte
+	}
+
+	var writesMu sync.Mutex
+	var writes []written
+	var uploadErr error
+	writeCtx, writeCancel := context.WithCancel(context.Background())
+	var writerWG sync.WaitGroup
+	writerWG.Add(1)
+	go func() {
+		defer writerWG.Done()
+		client := framework.NewHTTPClient()
+		for i := 0; i < 12; i++ {
+			select {
+			case <-writeCtx.Done():
+				return
+			default:
+			}
+			livePayload := []byte("live-data-" + fmt.Sprintf("%02d", i))
+			liveFid := framework.NewFileID(volumeID, uint64(2000+i), 0xAAAA1111+uint32(i))
+			resp := framework.UploadBytes(t, client, cluster.VolumeAdminURL(0), liveFid, livePayload)
+			_ = framework.ReadAllAndClose(t, resp)
+			if resp.StatusCode != http.StatusCreated {
+				uploadErr = fmt.Errorf("live upload failed: %d", resp.StatusCode)
+				return
+			}
+			writesMu.Lock()
+			writes = append(writes, written{fid: liveFid, data: livePayload})
+			writesMu.Unlock()
+			time.Sleep(250 * time.Millisecond)
+		}
+	}()
+
+	tailCtx, tailCancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer tailCancel()
+	_, err = destClient.VolumeTailReceiver(tailCtx, &volume_server_pb.VolumeTailReceiverRequest{
+		VolumeId:           volumeID,
+		SourceVolumeServer: sourceDataNode,
+		SinceNs:            lastAppendAtNs,
+		IdleTimeoutSeconds: 3,
+	})
+	if err != nil {
+		writeCancel()
+		writerWG.Wait()
+		t.Fatalf("VolumeTailReceiver failed: %v", err)
+	}
+
+	writeCancel()
+	writerWG.Wait()
+
+	if uploadErr != nil {
+		t.Fatalf("upload goroutine error: %v", uploadErr)
+	}
+
+	writesMu.Lock()
+	sampleCount := len(writes)
+	if sampleCount == 0 {
+		writesMu.Unlock()
+		t.Fatal("no live writes captured")
+	}
+	sample := writes
+	if sampleCount > 3 {
+		sample = writes[sampleCount-3:]
+	}
+	writesMu.Unlock()
+
+	httpCheckClient := framework.NewHTTPClient()
+	for _, w := range sample {
+		resp := framework.ReadBytes(t, httpCheckClient, cluster.VolumeAdminURL(1), w.fid)
+		body := framework.ReadAllAndClose(t, resp)
+		if resp.StatusCode != http.StatusOK {
+			t.Fatalf("dest read %s status %d", w.fid, resp.StatusCode)
+		}
+		if !bytes.Equal(body, w.data) {
+			t.Fatalf("dest read body mismatch for %s: %q vs %q", w.fid, string(body), string(w.data))
+		}
+	}
+}

test/volume_server/grpc/production_features_test.go

@@ -0,0 +1,338 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func TestScrubVolumeDetectsHealthyData(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(101)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	needles := []struct {
+		needleID uint64
+		cookie   uint32
+		body     string
+	}{
+		{needleID: 1010001, cookie: 0xAA000001, body: "scrub-healthy-needle-one"},
+		{needleID: 1010002, cookie: 0xAA000002, body: "scrub-healthy-needle-two"},
+		{needleID: 1010003, cookie: 0xAA000003, body: "scrub-healthy-needle-three"},
+	}
+	for _, n := range needles {
+		fid := framework.NewFileID(volumeID, n.needleID, n.cookie)
+		uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte(n.body))
+		_ = framework.ReadAllAndClose(t, uploadResp)
+		if uploadResp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload needle %d expected 201, got %d", n.needleID, uploadResp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	scrubResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume FULL mode failed: %v", err)
+	}
+	if scrubResp.GetTotalVolumes() != 1 {
+		t.Fatalf("ScrubVolume expected total_volumes=1, got %d", scrubResp.GetTotalVolumes())
+	}
+	if scrubResp.GetTotalFiles() < 3 {
+		t.Fatalf("ScrubVolume expected total_files >= 3, got %d", scrubResp.GetTotalFiles())
+	}
+	if len(scrubResp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("ScrubVolume expected no broken volumes for healthy data, got %v: %v", scrubResp.GetBrokenVolumeIds(), scrubResp.GetDetails())
+	}
+}
+
+func TestScrubVolumeLocalModeWithMultipleVolumes(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeIDA = uint32(102)
+	const volumeIDB = uint32(103)
+	framework.AllocateVolume(t, grpcClient, volumeIDA, "")
+	framework.AllocateVolume(t, grpcClient, volumeIDB, "")
+
+	httpClient := framework.NewHTTPClient()
+
+	fidA := framework.NewFileID(volumeIDA, 1020001, 0xBB000001)
+	uploadA := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fidA, []byte("scrub-local-vol-a"))
+	_ = framework.ReadAllAndClose(t, uploadA)
+	if uploadA.StatusCode != http.StatusCreated {
+		t.Fatalf("upload to volume A expected 201, got %d", uploadA.StatusCode)
+	}
+
+	fidB := framework.NewFileID(volumeIDB, 1030001, 0xBB000002)
+	uploadB := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fidB, []byte("scrub-local-vol-b"))
+	_ = framework.ReadAllAndClose(t, uploadB)
+	if uploadB.StatusCode != http.StatusCreated {
+		t.Fatalf("upload to volume B expected 201, got %d", uploadB.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	scrubResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		Mode: volume_server_pb.VolumeScrubMode_LOCAL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume LOCAL auto-select failed: %v", err)
+	}
+	if scrubResp.GetTotalVolumes() < 2 {
+		t.Fatalf("ScrubVolume LOCAL expected total_volumes >= 2, got %d", scrubResp.GetTotalVolumes())
+	}
+	if len(scrubResp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("ScrubVolume LOCAL expected no broken volumes, got %v: %v", scrubResp.GetBrokenVolumeIds(), scrubResp.GetDetails())
+	}
+}
+
+func TestVolumeServerStatusReturnsRealDiskStats(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	statusResp, err := grpcClient.VolumeServerStatus(ctx, &volume_server_pb.VolumeServerStatusRequest{})
+	if err != nil {
+		t.Fatalf("VolumeServerStatus failed: %v", err)
+	}
+
+	diskStatuses := statusResp.GetDiskStatuses()
+	if len(diskStatuses) == 0 {
+		t.Fatalf("VolumeServerStatus expected non-empty disk_statuses")
+	}
+
+	foundValid := false
+	for _, ds := range diskStatuses {
+		if ds.GetDir() != "" && ds.GetAll() > 0 && ds.GetFree() > 0 {
+			foundValid = true
+			break
+		}
+	}
+	if !foundValid {
+		t.Fatalf("VolumeServerStatus expected at least one disk status with Dir, All > 0, Free > 0; got %v", diskStatuses)
+	}
+}
+
+func TestReadNeedleBlobAndMetaVerifiesCookie(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(104)
+	const needleID = uint64(1040001)
+	const cookie = uint32(0xCC000001)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	payload := []byte("read-needle-blob-meta-verify")
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, payload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	fileStatus, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("ReadVolumeFileStatus failed: %v", err)
+	}
+	if fileStatus.GetIdxFileSize() == 0 {
+		t.Fatalf("expected non-zero idx file size after upload")
+	}
+
+	idxBytes := prodCopyFileBytes(t, grpcClient, &volume_server_pb.CopyFileRequest{
+		VolumeId:           volumeID,
+		Ext:                ".idx",
+		CompactionRevision: fileStatus.GetCompactionRevision(),
+		StopOffset:         fileStatus.GetIdxFileSize(),
+	})
+	offset, size := prodFindNeedleOffsetAndSize(t, idxBytes, needleID)
+
+	blobResp, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{
+		VolumeId: volumeID,
+		Offset:   offset,
+		Size:     size,
+	})
+	if err != nil {
+		t.Fatalf("ReadNeedleBlob failed: %v", err)
+	}
+	if len(blobResp.GetNeedleBlob()) == 0 {
+		t.Fatalf("ReadNeedleBlob returned empty blob")
+	}
+
+	metaResp, err := grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
+		VolumeId: volumeID,
+		NeedleId: needleID,
+		Offset:   offset,
+		Size:     size,
+	})
+	if err != nil {
+		t.Fatalf("ReadNeedleMeta failed: %v", err)
+	}
+	if metaResp.GetCookie() != cookie {
+		t.Fatalf("ReadNeedleMeta cookie mismatch: got %d want %d", metaResp.GetCookie(), cookie)
+	}
+	if metaResp.GetCrc() == 0 {
+		t.Fatalf("ReadNeedleMeta expected non-zero CRC")
+	}
+}
+
+func TestBatchDeleteMultipleNeedles(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(105)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	type needle struct {
+		needleID uint64
+		cookie   uint32
+		body     string
+		fid      string
+	}
+	needles := []needle{
+		{needleID: 1050001, cookie: 0xDD000001, body: "batch-del-needle-one"},
+		{needleID: 1050002, cookie: 0xDD000002, body: "batch-del-needle-two"},
+		{needleID: 1050003, cookie: 0xDD000003, body: "batch-del-needle-three"},
+	}
+	fids := make([]string, len(needles))
+	for i := range needles {
+		needles[i].fid = framework.NewFileID(volumeID, needles[i].needleID, needles[i].cookie)
+		fids[i] = needles[i].fid
+		uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), needles[i].fid, []byte(needles[i].body))
+		_ = framework.ReadAllAndClose(t, uploadResp)
+		if uploadResp.StatusCode != http.StatusCreated {
+			t.Fatalf("upload needle %d expected 201, got %d", needles[i].needleID, uploadResp.StatusCode)
+		}
+	}
+
+	// Verify all needles are readable before delete
+	for _, n := range needles {
+		readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid)
+		_ = framework.ReadAllAndClose(t, readResp)
+		if readResp.StatusCode != http.StatusOK {
+			t.Fatalf("pre-delete read of %s expected 200, got %d", n.fid, readResp.StatusCode)
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	deleteResp, err := grpcClient.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{
+		FileIds: fids,
+	})
+	if err != nil {
+		t.Fatalf("BatchDelete failed: %v", err)
+	}
+	if len(deleteResp.GetResults()) != 3 {
+		t.Fatalf("BatchDelete expected 3 results, got %d", len(deleteResp.GetResults()))
+	}
+	for i, result := range deleteResp.GetResults() {
+		if result.GetStatus() != http.StatusAccepted {
+			t.Fatalf("BatchDelete result[%d] expected status 202, got %d (error: %s)", i, result.GetStatus(), result.GetError())
+		}
+		if result.GetSize() <= 0 {
+			t.Fatalf("BatchDelete result[%d] expected size > 0, got %d", i, result.GetSize())
+		}
+	}
+
+	// Verify all needles return 404 after delete
+	for _, n := range needles {
+		readResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), n.fid)
+		_ = framework.ReadAllAndClose(t, readResp)
+		if readResp.StatusCode != http.StatusNotFound {
+			t.Fatalf("post-delete read of %s expected 404, got %d", n.fid, readResp.StatusCode)
+		}
+	}
+}
+
+// prodCopyFileBytes streams a CopyFile response into a byte slice.
+func prodCopyFileBytes(t testing.TB, grpcClient volume_server_pb.VolumeServerClient, req *volume_server_pb.CopyFileRequest) []byte {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.CopyFile(ctx, req)
+	if err != nil {
+		t.Fatalf("CopyFile start failed: %v", err)
+	}
+
+	var out []byte
+	for {
+		msg, recvErr := stream.Recv()
+		if recvErr == io.EOF {
+			return out
+		}
+		if recvErr != nil {
+			t.Fatalf("CopyFile recv failed: %v", recvErr)
+		}
+		out = append(out, msg.GetFileContent()...)
+	}
+}
+
+// prodFindNeedleOffsetAndSize scans idx bytes for a needle's offset and size.
+func prodFindNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64) (offset int64, size int32) {
+	t.Helper()
+
+	for i := 0; i+types.NeedleMapEntrySize <= len(idxBytes); i += types.NeedleMapEntrySize {
+		key, entryOffset, entrySize := idx.IdxFileEntry(idxBytes[i : i+types.NeedleMapEntrySize])
+		if uint64(key) != needleID {
+			continue
+		}
+		if entryOffset.IsZero() || entrySize <= 0 {
+			continue
+		}
+		return entryOffset.ToActualOffset(), int32(entrySize)
+	}
+
+	t.Fatalf("needle id %d not found in idx entries", needleID)
+	return 0, 0
+}

test/volume_server/grpc/scrub_integration_test.go

@@ -0,0 +1,501 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+// --- Normal volume scrub tests ---
+
+func TestScrubVolumeFullHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(200)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("data-one")))
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 2, 2), []byte("data-two")))
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 3, 3), []byte("data-three")))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume FULL on healthy volume failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 1 {
+		t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
+	}
+	if resp.GetTotalFiles() != 3 {
+		t.Fatalf("expected total_files=3, got %d", resp.GetTotalFiles())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
+	}
+}
+
+func TestScrubVolumeFullCorruptData(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(201)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("important data")))
+
+	framework.CorruptDatFile(t, clusterHarness.BaseDir(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume FULL on corrupt volume failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after data corruption, got none")
+	}
+	if len(resp.GetDetails()) == 0 {
+		t.Fatalf("expected error details for corrupt volume")
+	}
+}
+
+func TestScrubVolumeMixedHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const healthyVol = uint32(202)
+	const corruptVol = uint32(203)
+	framework.AllocateVolume(t, grpcClient, healthyVol, "")
+	framework.AllocateVolume(t, grpcClient, corruptVol, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(healthyVol, 1, 1), []byte("healthy")))
+	framework.ReadAllAndClose(t, framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(corruptVol, 1, 1), []byte("will corrupt")))
+
+	framework.CorruptIndexFile(t, clusterHarness.BaseDir(), corruptVol)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{healthyVol, corruptVol},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume INDEX on mixed volumes failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 2 {
+		t.Fatalf("expected total_volumes=2, got %d", resp.GetTotalVolumes())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 1 {
+		t.Fatalf("expected exactly 1 broken volume, got %v", resp.GetBrokenVolumeIds())
+	}
+	if resp.GetBrokenVolumeIds()[0] != corruptVol {
+		t.Fatalf("expected broken volume %d, got %d", corruptVol, resp.GetBrokenVolumeIds()[0])
+	}
+}
+
+func TestScrubVolumeMissingVolumeReturnsError(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{99999},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err == nil {
+		t.Fatalf("ScrubVolume should fail for missing volume")
+	}
+}
+
+// --- EC volume scrub tests ---
+
+// ecSetup creates a volume, uploads data, generates EC shards, and mounts all of them.
+func ecSetup(t *testing.T, grpcClient volume_server_pb.VolumeServerClient, httpClient *http.Client, volumeURL string, volumeID uint32) {
+	t.Helper()
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	fid := framework.NewFileID(volumeID, 1, 0xABCD0001)
+	uploadResp := framework.UploadBytes(t, httpClient, volumeURL, fid, []byte("ec-scrub-test-data-payload"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	allShards := make([]uint32, erasure_coding.TotalShardsCount)
+	for i := range allShards {
+		allShards[i] = uint32(i)
+	}
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   allShards,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount all shards failed: %v", err)
+	}
+}
+
+func TestScrubEcVolumeIndexHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(210)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume INDEX on healthy volume failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 1 {
+		t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
+	}
+}
+
+func TestScrubEcVolumeLocalHealthy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(211)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_LOCAL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume LOCAL on healthy volume failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 1 {
+		t.Fatalf("expected total_volumes=1, got %d", resp.GetTotalVolumes())
+	}
+	if resp.GetTotalFiles() != 1 {
+		t.Fatalf("expected total_files=1, got %d", resp.GetTotalFiles())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v: %v", resp.GetBrokenVolumeIds(), resp.GetDetails())
+	}
+	if len(resp.GetBrokenShardInfos()) != 0 {
+		t.Fatalf("expected no broken shards, got %v", resp.GetBrokenShardInfos())
+	}
+}
+
+func TestScrubEcVolumeLocalCorruptShard(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(212)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	// Corrupt shard 0 by truncating it.
+	framework.CorruptEcShardFile(t, clusterHarness.BaseDir(), volumeID, 0)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_LOCAL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume LOCAL on corrupt shard failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after shard corruption")
+	}
+	if len(resp.GetBrokenShardInfos()) == 0 {
+		t.Fatalf("expected broken shard info after shard corruption")
+	}
+	// Verify all reported broken shards belong to the corrupted volume.
+	for _, si := range resp.GetBrokenShardInfos() {
+		if si.GetVolumeId() != volumeID {
+			t.Fatalf("broken shard info for unexpected volume %d, want %d", si.GetVolumeId(), volumeID)
+		}
+	}
+}
+
+func TestScrubEcVolumeAutoSelectWithEcPresent(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeIDA = uint32(213)
+	const volumeIDB = uint32(214)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeIDA)
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeIDB)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// Auto-select (empty VolumeIds) should find both EC volumes.
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		Mode: volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume auto-select failed: %v", err)
+	}
+	if resp.GetTotalVolumes() < 2 {
+		t.Fatalf("expected at least 2 EC volumes via auto-select, got %d", resp.GetTotalVolumes())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v", resp.GetBrokenVolumeIds())
+	}
+}
+
+func TestScrubEcVolumeUnsupportedMode(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(215)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode(99),
+	})
+	if err == nil {
+		t.Fatalf("ScrubEcVolume should fail for unsupported mode")
+	}
+}
+
+func TestScrubEcVolumeIndexCorruptEcx(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(216)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 1, 0xABCD0001)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-ecx-corrupt-test"))
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	// Generate EC shards (this creates the .ecx file on disk).
+	_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
+	}
+
+	// Corrupt the .ecx file BEFORE mounting, so the corrupted size is loaded.
+	framework.CorruptEcxFile(t, clusterHarness.BaseDir(), volumeID)
+
+	// Now mount shards - the ecx file size will reflect the corruption.
+	allShards := make([]uint32, erasure_coding.TotalShardsCount)
+	for i := range allShards {
+		allShards[i] = uint32(i)
+	}
+	_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+		VolumeId:   volumeID,
+		Collection: "",
+		ShardIds:   allShards,
+	})
+	if err != nil {
+		t.Fatalf("VolumeEcShardsMount failed: %v", err)
+	}
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume INDEX on corrupt ecx failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after ECX corruption")
+	}
+}
+
+// scrubEcUntilLocated runs an EC scrub, retrying while the server is still waiting on
+// the master for the shard locations a distributed scrub needs.
+func scrubEcUntilLocated(t *testing.T, ctx context.Context, grpcClient volume_server_pb.VolumeServerClient, volumeID uint32, mode volume_server_pb.VolumeScrubMode) *volume_server_pb.ScrubEcVolumeResponse {
+	t.Helper()
+	deadline := time.Now().Add(60 * time.Second)
+	for {
+		resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+			VolumeIds: []uint32{volumeID},
+			Mode:      mode,
+		})
+		if err != nil {
+			t.Fatalf("ScrubEcVolume %s failed: %v", mode, err)
+		}
+		waiting := false
+		for _, d := range resp.GetDetails() {
+			if strings.Contains(d, "failed to locate shard via master grpc") {
+				waiting = true
+				break
+			}
+		}
+		if !waiting {
+			return resp
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("master never reported EC shard locations for volume %d: %v", volumeID, resp.GetDetails())
+		}
+		time.Sleep(time.Second)
+	}
+}
+
+// With a shard gone, FULL cannot read the intervals that lived on it, while READS
+// rebuilds them from parity. Either way the missing shard has to be reported: a
+// volume that scrubs clean is a volume nobody repairs.
+func TestScrubEcVolumeReadsRecoversMissingShard(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(216)
+	const missingShard = uint32(0)
+	httpClient := framework.NewHTTPClient()
+	ecSetup(t, grpcClient, httpClient, clusterHarness.VolumeAdminURL(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
+	defer cancel()
+
+	if _, err := grpcClient.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
+		VolumeId: volumeID,
+		ShardIds: []uint32{missingShard},
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsUnmount shard %d failed: %v", missingShard, err)
+	}
+	if _, err := grpcClient.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+		VolumeId: volumeID,
+		ShardIds: []uint32{missingShard},
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsDelete shard %d failed: %v", missingShard, err)
+	}
+
+	fullResp := scrubEcUntilLocated(t, ctx, grpcClient, volumeID, volume_server_pb.VolumeScrubMode_FULL)
+	assertOnlyBrokenShard(t, "FULL", fullResp, missingShard)
+	if len(fullResp.GetDetails()) == 0 {
+		t.Fatalf("FULL should report the needles it could not read")
+	}
+
+	readsResp := scrubEcUntilLocated(t, ctx, grpcClient, volumeID, volume_server_pb.VolumeScrubMode_READS)
+	assertOnlyBrokenShard(t, "READS", readsResp, missingShard)
+	if len(readsResp.GetDetails()) != 0 {
+		t.Fatalf("READS should rebuild every needle from parity, got: %v", readsResp.GetDetails())
+	}
+}
+
+func assertOnlyBrokenShard(t *testing.T, mode string, resp *volume_server_pb.ScrubEcVolumeResponse, shardID uint32) {
+	t.Helper()
+	infos := resp.GetBrokenShardInfos()
+	if len(infos) != 1 || infos[0].GetShardId() != shardID {
+		t.Fatalf("%s reported broken shards %v, want only shard %d (details: %v)", mode, infos, shardID, resp.GetDetails())
+	}
+}

test/volume_server/grpc/scrub_query_test.go

@@ -0,0 +1,544 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestScrubVolumeIndexAndUnsupportedMode(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(61)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	indexResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume index mode failed: %v", err)
+	}
+	if indexResp.GetTotalVolumes() != 1 {
+		t.Fatalf("ScrubVolume expected total_volumes=1, got %d", indexResp.GetTotalVolumes())
+	}
+
+	_, err = grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode(99),
+	})
+	if err == nil {
+		t.Fatalf("ScrubVolume should fail for unsupported mode")
+	}
+	if !strings.Contains(err.Error(), "unsupported volume scrub mode") {
+		t.Fatalf("ScrubVolume unsupported mode error mismatch: %v", err)
+	}
+}
+
+func TestScrubEcVolumeMissingVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		VolumeIds: []uint32{98765},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err == nil {
+		t.Fatalf("ScrubEcVolume should fail for missing EC volume")
+	}
+	if !strings.Contains(err.Error(), "EC volume id") {
+		t.Fatalf("ScrubEcVolume missing-volume error mismatch: %v", err)
+	}
+}
+
+func TestScrubEcVolumeAutoSelectNoEcVolumes(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubEcVolume(ctx, &volume_server_pb.ScrubEcVolumeRequest{
+		Mode: volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubEcVolume auto-select failed: %v", err)
+	}
+	if resp.GetTotalVolumes() != 0 {
+		t.Fatalf("ScrubEcVolume auto-select expected total_volumes=0 without EC data, got %d", resp.GetTotalVolumes())
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("ScrubEcVolume auto-select expected no broken volumes, got %v", resp.GetBrokenVolumeIds())
+	}
+}
+
+func TestQueryInvalidAndMissingFileIDPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	invalidStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
+		FromFileIds: []string{"bad-fid"},
+		Selections:  []string{"name"},
+		Filter:      &volume_server_pb.QueryRequest_Filter{},
+		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
+			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{},
+		},
+	})
+	if err == nil {
+		_, err = invalidStream.Recv()
+	}
+	if err == nil {
+		t.Fatalf("Query should fail for invalid file id")
+	}
+
+	missingFid := framework.NewFileID(98766, 1, 1)
+	missingStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
+		FromFileIds: []string{missingFid},
+		Selections:  []string{"name"},
+		Filter:      &volume_server_pb.QueryRequest_Filter{},
+		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
+			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{},
+		},
+	})
+	if err == nil {
+		_, err = missingStream.Recv()
+	}
+	if err == nil {
+		t.Fatalf("Query should fail for missing file id volume")
+	}
+}
+
+func TestScrubVolumeAutoSelectAndAllModes(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeIDA = uint32(62)
+	const volumeIDB = uint32(63)
+	framework.AllocateVolume(t, grpcClient, volumeIDA, "")
+	framework.AllocateVolume(t, grpcClient, volumeIDB, "")
+
+	// upload some data so index files are not zero-sized
+	httpClient := framework.NewHTTPClient()
+	framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeIDA, 1, 1), []byte("test data A"))
+	framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeIDB, 2, 2), []byte("test data B"))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	autoResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		Mode: volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume auto-select failed: %v", err)
+	}
+	if autoResp.GetTotalVolumes() < 2 {
+		t.Fatalf("ScrubVolume auto-select expected at least 2 volumes, got %d", autoResp.GetTotalVolumes())
+	}
+
+	localResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeIDA},
+		Mode:      volume_server_pb.VolumeScrubMode_LOCAL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume local mode failed: %v", err)
+	}
+	if localResp.GetTotalVolumes() != 1 {
+		t.Fatalf("ScrubVolume local mode expected total_volumes=1, got %d", localResp.GetTotalVolumes())
+	}
+	if len(localResp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("ScrubVolume local mode expected no broken volumes, got %v: %v", localResp.GetBrokenVolumeIds(), localResp.GetDetails())
+	}
+
+	fullResp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeIDA},
+		Mode:      volume_server_pb.VolumeScrubMode_FULL,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume full mode failed: %v", err)
+	}
+	if fullResp.GetTotalVolumes() != 1 {
+		t.Fatalf("ScrubVolume full mode expected total_volumes=1, got %d", fullResp.GetTotalVolumes())
+	}
+	if len(fullResp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("ScrubVolume full mode expected no broken volumes, got %v: %v", fullResp.GetBrokenVolumeIds(), fullResp.GetDetails())
+	}
+}
+
+func TestQueryJsonSuccessAndCsvNoOutput(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(64)
+	const needleID = uint64(777001)
+	const cookie = uint32(0xAABBCCDD)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	jsonLines := []byte("{\"score\":3}\n{\"score\":12}\n{\"score\":18}\n")
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	queryStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
+		FromFileIds: []string{fid},
+		Selections:  []string{"score"},
+		Filter: &volume_server_pb.QueryRequest_Filter{
+			Field:   "score",
+			Operand: ">",
+			Value:   "10",
+		},
+		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
+			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Query json start failed: %v", err)
+	}
+
+	firstStripe, err := queryStream.Recv()
+	if err != nil {
+		t.Fatalf("Query json recv failed: %v", err)
+	}
+	records := string(firstStripe.GetRecords())
+	if !strings.Contains(records, "score:12") || !strings.Contains(records, "score:18") {
+		t.Fatalf("Query json records missing expected filtered scores: %q", records)
+	}
+	if strings.Contains(records, "score:3") {
+		t.Fatalf("Query json records should not include filtered-out score: %q", records)
+	}
+	_, err = queryStream.Recv()
+	if err != io.EOF {
+		t.Fatalf("Query json expected EOF after first stripe, got: %v", err)
+	}
+
+	csvStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
+		FromFileIds: []string{fid},
+		Selections:  []string{"score"},
+		Filter:      &volume_server_pb.QueryRequest_Filter{},
+		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
+			CsvInput: &volume_server_pb.QueryRequest_InputSerialization_CSVInput{},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Query csv start failed: %v", err)
+	}
+	_, err = csvStream.Recv()
+	if err != io.EOF {
+		t.Fatalf("Query csv expected EOF with no rows, got: %v", err)
+	}
+}
+
+func TestQueryJsonNoMatchReturnsEmptyStripe(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(65)
+	const needleID = uint64(777002)
+	const cookie = uint32(0xABABCDCD)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	jsonLines := []byte("{\"score\":1}\n{\"score\":2}\n")
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	queryStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
+		FromFileIds: []string{fid},
+		Selections:  []string{"score"},
+		Filter: &volume_server_pb.QueryRequest_Filter{
+			Field:   "score",
+			Operand: ">",
+			Value:   "100",
+		},
+		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
+			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Query json no-match start failed: %v", err)
+	}
+
+	firstStripe, err := queryStream.Recv()
+	if err != nil {
+		t.Fatalf("Query json no-match recv failed: %v", err)
+	}
+	if len(firstStripe.GetRecords()) != 0 {
+		t.Fatalf("Query json no-match expected empty records stripe, got: %q", string(firstStripe.GetRecords()))
+	}
+
+	_, err = queryStream.Recv()
+	if err != io.EOF {
+		t.Fatalf("Query json no-match expected EOF after first empty stripe, got: %v", err)
+	}
+}
+
+func TestQueryCookieMismatchReturnsEOFNoResults(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(66)
+	const needleID = uint64(777003)
+	const cookie = uint32(0xCDCDABAB)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	jsonLines := []byte("{\"score\":7}\n{\"score\":8}\n")
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, needleID, cookie)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, jsonLines)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != 201 {
+		t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	wrongCookieFid := framework.NewFileID(volumeID, needleID, cookie+1)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
+		FromFileIds: []string{wrongCookieFid},
+		Selections:  []string{"score"},
+		Filter: &volume_server_pb.QueryRequest_Filter{
+			Field:   "score",
+			Operand: ">",
+			Value:   "0",
+		},
+		InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
+			JsonInput: &volume_server_pb.QueryRequest_InputSerialization_JSONInput{Type: "LINES"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Query start for cookie mismatch should not fail immediately, got: %v", err)
+	}
+
+	_, err = stream.Recv()
+	if err != io.EOF {
+		t.Fatalf("Query cookie mismatch expected EOF with no streamed records, got: %v", err)
+	}
+}
+
+func TestScrubVolumeMarkBrokenReadonlyHealthyVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(76)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("healthy data"))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds:                 []uint32{volumeID},
+		Mode:                      volume_server_pb.VolumeScrubMode_INDEX,
+		MarkBrokenVolumesReadonly: true,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume with MarkBrokenVolumesReadonly on healthy volume failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) != 0 {
+		t.Fatalf("expected no broken volumes, got %v", resp.GetBrokenVolumeIds())
+	}
+
+	statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus failed: %v", err)
+	}
+	if statusResp.GetIsReadOnly() {
+		t.Fatalf("healthy volume should not be read-only after scrub with MarkBrokenVolumesReadonly")
+	}
+}
+
+func TestScrubVolumeMarkBrokenReadonlyCorruptVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(77)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("test data"))
+
+	framework.CorruptIndexFile(t, clusterHarness.BaseDir(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	// scrub without the flag: broken volume reported but not marked read-only
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds: []uint32{volumeID},
+		Mode:      volume_server_pb.VolumeScrubMode_INDEX,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume without flag failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after corruption")
+	}
+
+	statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus after scrub without flag failed: %v", err)
+	}
+	if statusResp.GetIsReadOnly() {
+		t.Fatalf("volume should not be read-only when MarkBrokenVolumesReadonly is false")
+	}
+
+	// scrub with the flag: broken volume should now be marked read-only
+	resp, err = grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds:                 []uint32{volumeID},
+		Mode:                      volume_server_pb.VolumeScrubMode_INDEX,
+		MarkBrokenVolumesReadonly: true,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume with MarkBrokenVolumesReadonly failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after corruption with flag")
+	}
+
+	found := false
+	for _, d := range resp.GetDetails() {
+		if strings.Contains(d, "is now read-only") {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Fatalf("expected 'is now read-only' in details, got: %v", resp.GetDetails())
+	}
+
+	statusResp, err = grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus after MarkBrokenVolumesReadonly scrub failed: %v", err)
+	}
+	if !statusResp.GetIsReadOnly() {
+		t.Fatalf("broken volume should be read-only after MarkBrokenVolumesReadonly scrub")
+	}
+}
+
+func TestScrubVolumeMarkBrokenReadonlyInMaintenanceMode(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(78)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, 1, 1), []byte("test data"))
+
+	framework.CorruptIndexFile(t, clusterHarness.BaseDir(), volumeID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	framework.EnableMaintenanceMode(t, ctx, grpcClient)
+
+	// marking a volume readonly only restricts a server that is already
+	// read-only, so maintenance mode does not block fencing broken volumes
+	resp, err := grpcClient.ScrubVolume(ctx, &volume_server_pb.ScrubVolumeRequest{
+		VolumeIds:                 []uint32{volumeID},
+		Mode:                      volume_server_pb.VolumeScrubMode_INDEX,
+		MarkBrokenVolumesReadonly: true,
+	})
+	if err != nil {
+		t.Fatalf("ScrubVolume with MarkBrokenVolumesReadonly in maintenance mode failed: %v", err)
+	}
+	if len(resp.GetBrokenVolumeIds()) == 0 {
+		t.Fatalf("expected broken volume after corruption")
+	}
+
+	statusResp, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VolumeStatus after scrub in maintenance mode failed: %v", err)
+	}
+	if !statusResp.GetIsReadOnly() {
+		t.Fatalf("broken volume should be read-only after MarkBrokenVolumesReadonly scrub in maintenance mode")
+	}
+}

test/volume_server/grpc/tail_test.go

@@ -0,0 +1,206 @@
+package volume_server_grpc_test
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestVolumeTailSenderMissingVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{VolumeId: 77777, SinceNs: 0, IdleTimeoutSeconds: 1})
+	if err == nil {
+		_, err = stream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found volume") {
+		t.Fatalf("VolumeTailSender missing-volume error mismatch: %v", err)
+	}
+}
+
+func TestVolumeTailSenderHeartbeatThenEOF(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(71)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{
+		VolumeId:           volumeID,
+		SinceNs:            0,
+		IdleTimeoutSeconds: 1,
+	})
+	if err != nil {
+		t.Fatalf("VolumeTailSender start failed: %v", err)
+	}
+
+	msg, err := stream.Recv()
+	if err != nil {
+		t.Fatalf("VolumeTailSender first recv failed: %v", err)
+	}
+	if !msg.GetIsLastChunk() {
+		t.Fatalf("expected first tail message to be heartbeat IsLastChunk=true")
+	}
+
+	_, err = stream.Recv()
+	if err != io.EOF {
+		t.Fatalf("expected EOF after idle timeout drain, got: %v", err)
+	}
+}
+
+func TestVolumeTailReceiverMissingVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{VolumeId: 88888, SourceVolumeServer: clusterHarness.VolumeServerAddress(), SinceNs: 0, IdleTimeoutSeconds: 1})
+	if err == nil || !strings.Contains(err.Error(), "receiver not found volume") {
+		t.Fatalf("VolumeTailReceiver missing-volume error mismatch: %v", err)
+	}
+}
+
+func TestVolumeTailReceiverReplicatesSourceUpdates(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartDualVolumeCluster(t, matrix.P1())
+	sourceConn, sourceClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(0))
+	defer sourceConn.Close()
+	destConn, destClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress(1))
+	defer destConn.Close()
+
+	const volumeID = uint32(72)
+	framework.AllocateVolume(t, sourceClient, volumeID, "")
+	framework.AllocateVolume(t, destClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 880003, 0x3456789A)
+	payload := []byte("tail-receiver-replicates-source-updates")
+
+	sourceUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(0), fid, payload)
+	_ = framework.ReadAllAndClose(t, sourceUploadResp)
+	if sourceUploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("source upload expected 201, got %d", sourceUploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	_, err := destClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
+		VolumeId:           volumeID,
+		SourceVolumeServer: clusterHarness.VolumeAdminAddress(0) + "." + strings.Split(clusterHarness.VolumeGRPCAddress(0), ":")[1],
+		SinceNs:            0,
+		IdleTimeoutSeconds: 1,
+	})
+	if err != nil {
+		t.Fatalf("VolumeTailReceiver success path failed: %v", err)
+	}
+
+	destReadResp := framework.ReadBytes(t, httpClient, clusterHarness.VolumeAdminURL(1), fid)
+	destReadBody := framework.ReadAllAndClose(t, destReadResp)
+	if destReadResp.StatusCode != http.StatusOK {
+		t.Fatalf("destination read after tail receive expected 200, got %d", destReadResp.StatusCode)
+	}
+	if string(destReadBody) != string(payload) {
+		t.Fatalf("destination tail-received payload mismatch: got %q want %q", string(destReadBody), string(payload))
+	}
+}
+
+func TestVolumeTailSenderLargeNeedleChunking(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(73)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	httpClient := framework.NewHTTPClient()
+	fid := framework.NewFileID(volumeID, 880004, 0x456789AB)
+	largePayload := bytes.Repeat([]byte("L"), 2*1024*1024+128*1024)
+	uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, largePayload)
+	_ = framework.ReadAllAndClose(t, uploadResp)
+	if uploadResp.StatusCode != http.StatusCreated {
+		t.Fatalf("large upload expected 201, got %d", uploadResp.StatusCode)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{
+		VolumeId:           volumeID,
+		SinceNs:            0,
+		IdleTimeoutSeconds: 1,
+	})
+	if err != nil {
+		t.Fatalf("VolumeTailSender start failed: %v", err)
+	}
+
+	dataChunkCount := 0
+	sawNonLastDataChunk := false
+	sawLastDataChunk := false
+	for {
+		msg, recvErr := stream.Recv()
+		if recvErr == io.EOF {
+			break
+		}
+		if recvErr != nil {
+			t.Fatalf("VolumeTailSender recv failed: %v", recvErr)
+		}
+		if len(msg.GetNeedleBody()) == 0 {
+			continue
+		}
+		dataChunkCount++
+		if msg.GetIsLastChunk() {
+			sawLastDataChunk = true
+		} else {
+			sawNonLastDataChunk = true
+		}
+	}
+
+	if dataChunkCount < 2 {
+		t.Fatalf("VolumeTailSender expected multiple chunks for large needle, got %d", dataChunkCount)
+	}
+	if !sawNonLastDataChunk {
+		t.Fatalf("VolumeTailSender expected at least one non-last data chunk")
+	}
+	if !sawLastDataChunk {
+		t.Fatalf("VolumeTailSender expected a final data chunk marked IsLastChunk=true")
+	}
+}

test/volume_server/grpc/tiering_remote_test.go

@@ -0,0 +1,236 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestFetchAndWriteNeedleMaintenanceAndMissingVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{
+		VolumeId: 98781,
+		NeedleId: 1,
+	})
+	if err == nil || !strings.Contains(err.Error(), "not found volume id") {
+		t.Fatalf("FetchAndWriteNeedle missing-volume error mismatch: %v", err)
+	}
+
+	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{
+			Maintenance: true,
+			Version:     stateResp.GetState().GetVersion(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	_, err = grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{
+		VolumeId: 1,
+		NeedleId: 1,
+	})
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("FetchAndWriteNeedle maintenance error mismatch: %v", err)
+	}
+}
+
+func TestFetchAndWriteNeedleInvalidRemoteConfig(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(88)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	_, err := grpcClient.FetchAndWriteNeedle(ctx, &volume_server_pb.FetchAndWriteNeedleRequest{
+		VolumeId: volumeID,
+		NeedleId: 1,
+		Cookie:   1,
+		Size:     1,
+		RemoteConf: &remote_pb.RemoteConf{
+			Name: "it-invalid-remote",
+			Type: "does-not-exist",
+		},
+		RemoteLocation: &remote_pb.RemoteStorageLocation{
+			Name: "it-invalid-remote",
+			Path: "/test",
+		},
+	})
+	if err == nil || !strings.Contains(err.Error(), "get remote client") {
+		t.Fatalf("FetchAndWriteNeedle invalid-remote error mismatch: %v", err)
+	}
+}
+
+func TestVolumeTierMoveDatToRemoteErrorPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(85)
+	const collection = "tier-collection"
+	framework.AllocateVolume(t, grpcClient, volumeID, collection)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	missingStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{
+		VolumeId:               98782,
+		Collection:             collection,
+		DestinationBackendName: "dummy",
+	})
+	if err == nil {
+		_, err = missingStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeTierMoveDatToRemote missing-volume error mismatch: %v", err)
+	}
+
+	mismatchStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{
+		VolumeId:               volumeID,
+		Collection:             "wrong-collection",
+		DestinationBackendName: "dummy",
+	})
+	if err == nil {
+		_, err = mismatchStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "unexpected input") {
+		t.Fatalf("VolumeTierMoveDatToRemote collection mismatch error mismatch: %v", err)
+	}
+
+	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{
+			Maintenance: true,
+			Version:     stateResp.GetState().GetVersion(),
+		},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	maintenanceStream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{
+		VolumeId:               volumeID,
+		Collection:             collection,
+		DestinationBackendName: "dummy",
+	})
+	if err == nil {
+		_, err = maintenanceStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "maintenance mode") {
+		t.Fatalf("VolumeTierMoveDatToRemote maintenance error mismatch: %v", err)
+	}
+}
+
+func TestVolumeTierMoveDatToRemoteMissingBackend(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(89)
+	const collection = "tier-missing-backend"
+	framework.AllocateVolume(t, grpcClient, volumeID, collection)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stream, err := grpcClient.VolumeTierMoveDatToRemote(ctx, &volume_server_pb.VolumeTierMoveDatToRemoteRequest{
+		VolumeId:               volumeID,
+		Collection:             collection,
+		DestinationBackendName: "definitely-missing-backend",
+	})
+	if err == nil {
+		_, err = stream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "destination definitely-missing-backend not found") {
+		t.Fatalf("VolumeTierMoveDatToRemote missing-backend error mismatch: %v", err)
+	}
+}
+
+func TestVolumeTierMoveDatFromRemoteErrorPaths(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(86)
+	const collection = "tier-download-collection"
+	framework.AllocateVolume(t, grpcClient, volumeID, collection)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	missingStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{
+		VolumeId:   98783,
+		Collection: collection,
+	})
+	if err == nil {
+		_, err = missingStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "not found") {
+		t.Fatalf("VolumeTierMoveDatFromRemote missing-volume error mismatch: %v", err)
+	}
+
+	mismatchStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{
+		VolumeId:   volumeID,
+		Collection: "wrong-collection",
+	})
+	if err == nil {
+		_, err = mismatchStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "unexpected input") {
+		t.Fatalf("VolumeTierMoveDatFromRemote collection mismatch error mismatch: %v", err)
+	}
+
+	localDiskStream, err := grpcClient.VolumeTierMoveDatFromRemote(ctx, &volume_server_pb.VolumeTierMoveDatFromRemoteRequest{
+		VolumeId:   volumeID,
+		Collection: collection,
+	})
+	if err == nil {
+		_, err = localDiskStream.Recv()
+	}
+	if err == nil || !strings.Contains(err.Error(), "already on local disk") {
+		t.Fatalf("VolumeTierMoveDatFromRemote local-disk error mismatch: %v", err)
+	}
+}

test/volume_server/grpc/vacuum_test.go

@@ -0,0 +1,87 @@
+package volume_server_grpc_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestVacuumVolumeCheckSuccessAndMissingVolume(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	const volumeID = uint32(31)
+	framework.AllocateVolume(t, grpcClient, volumeID, "")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	resp, err := grpcClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{VolumeId: volumeID})
+	if err != nil {
+		t.Fatalf("VacuumVolumeCheck existing volume failed: %v", err)
+	}
+	if resp.GetGarbageRatio() < 0 || resp.GetGarbageRatio() > 1 {
+		t.Fatalf("unexpected garbage ratio: %f", resp.GetGarbageRatio())
+	}
+
+	_, err = grpcClient.VacuumVolumeCheck(ctx, &volume_server_pb.VacuumVolumeCheckRequest{VolumeId: 99999})
+	if err == nil {
+		t.Fatalf("VacuumVolumeCheck should fail for missing volume")
+	}
+}
+
+func TestVacuumMaintenanceModeRejections(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test in short mode")
+	}
+
+	clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
+	conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	stateResp, err := grpcClient.GetState(ctx, &volume_server_pb.GetStateRequest{})
+	if err != nil {
+		t.Fatalf("GetState failed: %v", err)
+	}
+	_, err = grpcClient.SetState(ctx, &volume_server_pb.SetStateRequest{
+		State: &volume_server_pb.VolumeServerState{Maintenance: true, Version: stateResp.GetState().GetVersion()},
+	})
+	if err != nil {
+		t.Fatalf("SetState maintenance=true failed: %v", err)
+	}
+
+	assertMaintenanceErr := func(name string, err error) {
+		t.Helper()
+		if err == nil {
+			t.Fatalf("%s should fail in maintenance mode", name)
+		}
+		if !strings.Contains(err.Error(), "maintenance mode") {
+			t.Fatalf("%s expected maintenance mode error, got: %v", name, err)
+		}
+	}
+
+	compactStream, err := grpcClient.VacuumVolumeCompact(ctx, &volume_server_pb.VacuumVolumeCompactRequest{VolumeId: 31})
+	if err == nil {
+		_, err = compactStream.Recv()
+	}
+	assertMaintenanceErr("VacuumVolumeCompact", err)
+
+	_, err = grpcClient.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{VolumeId: 31})
+	assertMaintenanceErr("VacuumVolumeCommit", err)
+
+	_, err = grpcClient.VacuumVolumeCleanup(ctx, &volume_server_pb.VacuumVolumeCleanupRequest{VolumeId: 31})
+	assertMaintenanceErr("VacuumVolumeCleanup", err)
+}