snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/plugin_workers/erasure_coding/detection_test.go

@@ -0,0 +1,288 @@
+package erasure_coding_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	pluginworkers "github.com/seaweedfs/seaweedfs/test/plugin_workers"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+	ecstorage "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/protobuf/proto"
+)
+
+type topologySpec struct {
+	name         string
+	dataCenters  int
+	racksPerDC   int
+	nodesPerRack int
+	diskTypes    []string
+	replicas     int
+	collection   string
+}
+
+type detectionCase struct {
+	name                  string
+	topology              topologySpec
+	adminCollectionFilter string
+	expectProposals       bool
+}
+
+func TestErasureCodingDetectionAcrossTopologies(t *testing.T) {
+	cases := []detectionCase{
+		{
+			name: "single-dc-multi-rack",
+			topology: topologySpec{
+				name:         "single-dc-multi-rack",
+				dataCenters:  1,
+				racksPerDC:   2,
+				nodesPerRack: 7,
+				diskTypes:    []string{"hdd"},
+				replicas:     1,
+				collection:   "ec-test",
+			},
+			expectProposals: true,
+		},
+		{
+			name: "multi-dc",
+			topology: topologySpec{
+				name:         "multi-dc",
+				dataCenters:  2,
+				racksPerDC:   1,
+				nodesPerRack: 7,
+				diskTypes:    []string{"hdd"},
+				replicas:     1,
+				collection:   "ec-test",
+			},
+			expectProposals: true,
+		},
+		{
+			name: "multi-dc-multi-rack",
+			topology: topologySpec{
+				name:         "multi-dc-multi-rack",
+				dataCenters:  2,
+				racksPerDC:   2,
+				nodesPerRack: 4,
+				diskTypes:    []string{"hdd"},
+				replicas:     1,
+				collection:   "ec-test",
+			},
+			expectProposals: true,
+		},
+		{
+			name: "mixed-disk-types",
+			topology: topologySpec{
+				name:         "mixed-disk-types",
+				dataCenters:  1,
+				racksPerDC:   2,
+				nodesPerRack: 7,
+				diskTypes:    []string{"hdd", "ssd"},
+				replicas:     1,
+				collection:   "ec-test",
+			},
+			expectProposals: true,
+		},
+		{
+			name: "multi-replica-volume",
+			topology: topologySpec{
+				name:         "multi-replica-volume",
+				dataCenters:  1,
+				racksPerDC:   2,
+				nodesPerRack: 7,
+				diskTypes:    []string{"hdd"},
+				replicas:     3,
+				collection:   "ec-test",
+			},
+			expectProposals: true,
+		},
+		{
+			name: "collection-filter-match",
+			topology: topologySpec{
+				name:         "collection-filter-match",
+				dataCenters:  1,
+				racksPerDC:   2,
+				nodesPerRack: 7,
+				diskTypes:    []string{"hdd"},
+				replicas:     1,
+				collection:   "filtered",
+			},
+			adminCollectionFilter: "filtered",
+			expectProposals:       true,
+		},
+		{
+			name: "collection-filter-mismatch",
+			topology: topologySpec{
+				name:         "collection-filter-mismatch",
+				dataCenters:  1,
+				racksPerDC:   2,
+				nodesPerRack: 7,
+				diskTypes:    []string{"hdd"},
+				replicas:     1,
+				collection:   "filtered",
+			},
+			adminCollectionFilter: "other",
+			expectProposals:       false,
+		},
+		{
+			name: "insufficient-disks",
+			topology: topologySpec{
+				name:         "insufficient-disks",
+				dataCenters:  1,
+				racksPerDC:   1,
+				nodesPerRack: 2,
+				diskTypes:    []string{"hdd"},
+				replicas:     1,
+				collection:   "ec-test",
+			},
+			expectProposals: false,
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			volumeID := uint32(7)
+			response := buildVolumeListResponse(t, tc.topology, volumeID)
+			master := pluginworkers.NewMasterServer(t, response)
+
+			dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+			handler := erasure_coding.NewErasureCodingHandler(dialOption, t.TempDir())
+			harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
+				WorkerOptions: pluginworker.WorkerOptions{
+					GrpcDialOption: dialOption,
+				},
+				Handlers: []pluginworker.JobHandler{handler},
+			})
+			harness.WaitForJobType("erasure_coding")
+
+			if tc.adminCollectionFilter != "" {
+				err := harness.Plugin().SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
+					JobType: "erasure_coding",
+					AdminConfigValues: map[string]*plugin_pb.ConfigValue{
+						"collection_filter": {
+							Kind: &plugin_pb.ConfigValue_StringValue{StringValue: tc.adminCollectionFilter},
+						},
+					},
+				})
+				require.NoError(t, err)
+			}
+
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+
+			proposals, err := harness.Plugin().RunDetection(ctx, "erasure_coding", &plugin_pb.ClusterContext{
+				MasterGrpcAddresses: []string{master.Address()},
+			}, 10)
+			require.NoError(t, err)
+
+			if !tc.expectProposals {
+				require.Empty(t, proposals)
+				return
+			}
+
+			require.NotEmpty(t, proposals)
+
+			proposal := proposals[0]
+			require.Equal(t, "erasure_coding", proposal.JobType)
+			paramsValue := proposal.Parameters["task_params_pb"]
+			require.NotNil(t, paramsValue)
+
+			params := &worker_pb.TaskParams{}
+			require.NoError(t, proto.Unmarshal(paramsValue.GetBytesValue(), params))
+			require.NotEmpty(t, params.Sources)
+			require.Len(t, params.Targets, ecstorage.TotalShardsCount)
+		})
+	}
+}
+
+func buildVolumeListResponse(t *testing.T, spec topologySpec, volumeID uint32) *master_pb.VolumeListResponse {
+	t.Helper()
+
+	volumeSizeLimitMB := uint64(100)
+	// Exceed the default fullness (0.95) and quiet (1h) thresholds so volumes are
+	// EC-eligible.
+	volumeSize := uint64(96) * 1024 * 1024
+	volumeModifiedAt := time.Now().Add(-2 * time.Hour).Unix()
+
+	diskTypes := spec.diskTypes
+	if len(diskTypes) == 0 {
+		diskTypes = []string{"hdd"}
+	}
+	replicas := spec.replicas
+	if replicas <= 0 {
+		replicas = 1
+	}
+	collection := spec.collection
+	if collection == "" {
+		collection = "ec-test"
+	}
+
+	var dataCenters []*master_pb.DataCenterInfo
+	nodeIndex := 0
+	replicasPlaced := 0
+
+	for dc := 0; dc < spec.dataCenters; dc++ {
+		var racks []*master_pb.RackInfo
+		for rack := 0; rack < spec.racksPerDC; rack++ {
+			var nodes []*master_pb.DataNodeInfo
+			for n := 0; n < spec.nodesPerRack; n++ {
+				nodeIndex++
+				address := fmt.Sprintf("127.0.0.1:%d", 20000+nodeIndex)
+				diskType := diskTypes[(nodeIndex-1)%len(diskTypes)]
+
+				diskInfo := &master_pb.DiskInfo{
+					DiskId:         0,
+					MaxVolumeCount: 100,
+					VolumeCount:    0,
+					VolumeInfos:    []*master_pb.VolumeInformationMessage{},
+				}
+
+				if replicasPlaced < replicas {
+					diskInfo.VolumeCount = 1
+					diskInfo.VolumeInfos = append(diskInfo.VolumeInfos, &master_pb.VolumeInformationMessage{
+						Id:               volumeID,
+						Collection:       collection,
+						DiskId:           0,
+						Size:             volumeSize,
+						DeletedByteCount: 0,
+						ModifiedAtSecond: volumeModifiedAt,
+						ReplicaPlacement: 1,
+						ReadOnly:         false,
+					})
+					replicasPlaced++
+				}
+
+				nodes = append(nodes, &master_pb.DataNodeInfo{
+					Id:        address,
+					Address:   address,
+					DiskInfos: map[string]*master_pb.DiskInfo{diskType: diskInfo},
+				})
+			}
+
+			racks = append(racks, &master_pb.RackInfo{
+				Id:            fmt.Sprintf("rack-%d", rack+1),
+				DataNodeInfos: nodes,
+			})
+		}
+
+		dataCenters = append(dataCenters, &master_pb.DataCenterInfo{
+			Id:        fmt.Sprintf("dc-%d", dc+1),
+			RackInfos: racks,
+		})
+	}
+
+	return &master_pb.VolumeListResponse{
+		VolumeSizeLimitMb: volumeSizeLimitMB,
+		TopologyInfo: &master_pb.TopologyInfo{
+			DataCenterInfos: dataCenters,
+		},
+	}
+}

test/plugin_workers/erasure_coding/execution_test.go

@@ -0,0 +1,84 @@
+package erasure_coding_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	pluginworkers "github.com/seaweedfs/seaweedfs/test/plugin_workers"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+	ecstorage "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func TestErasureCodingExecutionEncodesShards(t *testing.T) {
+	volumeID := uint32(123)
+	datSize := 1 * 1024 * 1024
+
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	handler := erasure_coding.NewErasureCodingHandler(dialOption, t.TempDir())
+	harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
+		WorkerOptions: pluginworker.WorkerOptions{
+			GrpcDialOption: dialOption,
+		},
+		Handlers: []pluginworker.JobHandler{handler},
+	})
+	harness.WaitForJobType("erasure_coding")
+
+	sourceServer := pluginworkers.NewVolumeServer(t, "")
+	pluginworkers.WriteTestVolumeFiles(t, sourceServer.BaseDir(), volumeID, datSize)
+
+	targetServers := make([]*pluginworkers.VolumeServer, 0, ecstorage.TotalShardsCount)
+	targetAddresses := make([]string, 0, ecstorage.TotalShardsCount)
+	for i := 0; i < ecstorage.TotalShardsCount; i++ {
+		target := pluginworkers.NewVolumeServer(t, "")
+		targetServers = append(targetServers, target)
+		targetAddresses = append(targetAddresses, target.Address())
+	}
+
+	job := &plugin_pb.JobSpec{
+		JobId:   fmt.Sprintf("ec-job-%d", volumeID),
+		JobType: "erasure_coding",
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"volume_id": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(volumeID)},
+			},
+			"collection": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "ec-test"},
+			},
+			"source_server": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceServer.Address()},
+			},
+			"target_servers": {
+				Kind: &plugin_pb.ConfigValue_StringList{StringList: &plugin_pb.StringList{Values: targetAddresses}},
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	result, err := harness.Plugin().ExecuteJob(ctx, job, nil, 1)
+	require.NoError(t, err)
+	require.NotNil(t, result)
+	require.True(t, result.Success)
+
+	require.GreaterOrEqual(t, sourceServer.MarkReadonlyCount(), 1)
+	require.GreaterOrEqual(t, len(sourceServer.DeleteRequests()), 1)
+
+	for shardID := 0; shardID < ecstorage.TotalShardsCount; shardID++ {
+		targetIndex := shardID % len(targetServers)
+		target := targetServers[targetIndex]
+		expected := filepath.Join(target.BaseDir(), fmt.Sprintf("%d.ec%02d", volumeID, shardID))
+		info, err := os.Stat(expected)
+		require.NoErrorf(t, err, "missing shard file %s", expected)
+		require.Greater(t, info.Size(), int64(0))
+	}
+}

test/plugin_workers/erasure_coding/large_topology_test.go

@@ -0,0 +1,126 @@
+package erasure_coding_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	pluginworkers "github.com/seaweedfs/seaweedfs/test/plugin_workers"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func TestErasureCodingDetectionLargeTopology(t *testing.T) {
+	const (
+		rackCount       = 100
+		serverCount     = 1000
+		volumesPerNode  = 300
+		volumeSizeLimit = uint64(100)
+	)
+
+	if serverCount%rackCount != 0 {
+		t.Fatalf("serverCount (%d) must be divisible by rackCount (%d)", serverCount, rackCount)
+	}
+
+	nodesPerRack := serverCount / rackCount
+	// Eligible volumes must exceed the default fullness (0.95) and quiet (1h)
+	// thresholds; ineligible ones fall below the fullness threshold.
+	eligibleSize := uint64(96) * 1024 * 1024
+	ineligibleSize := uint64(10) * 1024 * 1024
+	modifiedAt := time.Now().Add(-2 * time.Hour).Unix()
+
+	volumeID := uint32(1)
+	dataCenters := make([]*master_pb.DataCenterInfo, 0, 1)
+
+	racks := make([]*master_pb.RackInfo, 0, rackCount)
+	for rack := 0; rack < rackCount; rack++ {
+		nodes := make([]*master_pb.DataNodeInfo, 0, nodesPerRack)
+		for node := 0; node < nodesPerRack; node++ {
+			address := fmt.Sprintf("10.0.%d.%d:8080", rack, node+1)
+			volumes := make([]*master_pb.VolumeInformationMessage, 0, volumesPerNode)
+			for v := 0; v < volumesPerNode; v++ {
+				size := ineligibleSize
+				if volumeID%2 == 0 {
+					size = eligibleSize
+				}
+				volumes = append(volumes, &master_pb.VolumeInformationMessage{
+					Id:               volumeID,
+					Collection:       "ec-bulk",
+					DiskId:           0,
+					Size:             size,
+					DeletedByteCount: 0,
+					ModifiedAtSecond: modifiedAt,
+					ReplicaPlacement: 1,
+					ReadOnly:         false,
+				})
+				volumeID++
+			}
+
+			diskInfo := &master_pb.DiskInfo{
+				DiskId:         0,
+				MaxVolumeCount: int64(volumesPerNode + 10),
+				VolumeCount:    int64(volumesPerNode),
+				VolumeInfos:    volumes,
+			}
+
+			nodes = append(nodes, &master_pb.DataNodeInfo{
+				Id:        address,
+				Address:   address,
+				DiskInfos: map[string]*master_pb.DiskInfo{"hdd": diskInfo},
+			})
+		}
+
+		racks = append(racks, &master_pb.RackInfo{
+			Id:            fmt.Sprintf("rack-%d", rack+1),
+			DataNodeInfos: nodes,
+		})
+	}
+
+	dataCenters = append(dataCenters, &master_pb.DataCenterInfo{
+		Id:        "dc-1",
+		RackInfos: racks,
+	})
+
+	response := &master_pb.VolumeListResponse{
+		VolumeSizeLimitMb: volumeSizeLimit,
+		TopologyInfo: &master_pb.TopologyInfo{
+			DataCenterInfos: dataCenters,
+		},
+	}
+
+	master := pluginworkers.NewMasterServer(t, response)
+
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	handler := erasure_coding.NewErasureCodingHandler(dialOption, t.TempDir())
+	harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
+		WorkerOptions: pluginworker.WorkerOptions{
+			GrpcDialOption: dialOption,
+		},
+		Handlers: []pluginworker.JobHandler{handler},
+	})
+	harness.WaitForJobType("erasure_coding")
+
+	totalVolumes := serverCount * volumesPerNode
+	expectedEligible := totalVolumes / 2
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+
+	start := time.Now()
+	proposals, err := harness.Plugin().RunDetection(ctx, "erasure_coding", &plugin_pb.ClusterContext{
+		MasterGrpcAddresses: []string{master.Address()},
+	}, 0)
+	duration := time.Since(start)
+	require.NoError(t, err)
+	require.GreaterOrEqual(t, len(proposals), 10, "should detect at least some proposals")
+	t.Logf("large topology detection completed in %s (proposals=%d, eligible=%d)", duration, len(proposals), expectedEligible)
+	if len(proposals) < expectedEligible {
+		t.Logf("large topology detection stopped early: %d proposals vs %d eligible", len(proposals), expectedEligible)
+	}
+}

test/plugin_workers/fake_master.go

@@ -0,0 +1,90 @@
+package pluginworkers
+
+import (
+	"context"
+	"net"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/protobuf/proto"
+	"testing"
+)
+
+// MasterServer provides a stub master gRPC service for topology responses.
+type MasterServer struct {
+	master_pb.UnimplementedSeaweedServer
+
+	t *testing.T
+
+	server   *grpc.Server
+	listener net.Listener
+	address  string
+
+	mu       sync.RWMutex
+	response *master_pb.VolumeListResponse
+}
+
+// NewMasterServer starts a stub master server that serves the provided response.
+func NewMasterServer(t *testing.T, response *master_pb.VolumeListResponse) *MasterServer {
+	t.Helper()
+
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen master: %v", err)
+	}
+
+	server := pb.NewGrpcServer()
+	ms := &MasterServer{
+		t:        t,
+		server:   server,
+		listener: listener,
+		address:  listener.Addr().String(),
+		response: response,
+	}
+
+	master_pb.RegisterSeaweedServer(server, ms)
+	go func() {
+		_ = server.Serve(listener)
+	}()
+
+	t.Cleanup(func() {
+		ms.Shutdown()
+	})
+
+	return ms
+}
+
+// Address returns the gRPC address of the master server.
+func (m *MasterServer) Address() string {
+	return m.address
+}
+
+// SetVolumeListResponse updates the response served by VolumeList.
+func (m *MasterServer) SetVolumeListResponse(response *master_pb.VolumeListResponse) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.response = response
+}
+
+// VolumeList returns the configured topology response.
+func (m *MasterServer) VolumeList(ctx context.Context, req *master_pb.VolumeListRequest) (*master_pb.VolumeListResponse, error) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	if m.response == nil {
+		return &master_pb.VolumeListResponse{}, nil
+	}
+	return proto.Clone(m.response).(*master_pb.VolumeListResponse), nil
+}
+
+// Shutdown stops the master gRPC server.
+func (m *MasterServer) Shutdown() {
+	if m.server != nil {
+		m.server.GracefulStop()
+	}
+	if m.listener != nil {
+		_ = m.listener.Close()
+	}
+}

test/plugin_workers/fake_volume_server.go

@@ -0,0 +1,569 @@
+package pluginworkers
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// VolumeServer provides a minimal volume server for erasure coding tests.
+type VolumeServer struct {
+	volume_server_pb.UnimplementedVolumeServerServer
+
+	t *testing.T
+
+	server   *grpc.Server
+	listener net.Listener
+	address  string
+	baseDir  string
+
+	mu                   sync.Mutex
+	receivedFiles        map[string]uint64
+	readonlyVolumes      map[uint32]bool
+	mountRequests        []*volume_server_pb.VolumeEcShardsMountRequest
+	deleteRequests       []*volume_server_pb.VolumeDeleteRequest
+	markReadonlyCalls    int
+	markWritableCalls    int
+	readFileStatusCalls  int
+	vacuumGarbageRatio   float64
+	vacuumCommitReadOnly bool
+	vacuumCheckCalls     int
+	vacuumCompactCalls   int
+	vacuumCommitCalls    int
+	vacuumCleanupCalls   int
+	volumeCopyCalls      int
+	volumeMountCalls     int
+	tailReceiverCalls    int
+}
+
+// NewVolumeServer starts a test volume server using the provided base directory.
+func NewVolumeServer(t *testing.T, baseDir string) *VolumeServer {
+	t.Helper()
+
+	if baseDir == "" {
+		baseDir = t.TempDir()
+	}
+	if err := os.MkdirAll(baseDir, 0755); err != nil {
+		t.Fatalf("create volume base dir: %v", err)
+	}
+
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen volume server: %v", err)
+	}
+
+	grpcPort := listener.Addr().(*net.TCPAddr).Port
+	server := pb.NewGrpcServer()
+	vs := &VolumeServer{
+		t:               t,
+		server:          server,
+		listener:        listener,
+		address:         fmt.Sprintf("127.0.0.1:0.%d", grpcPort),
+		baseDir:         baseDir,
+		receivedFiles:   make(map[string]uint64),
+		readonlyVolumes: make(map[uint32]bool),
+	}
+
+	volume_server_pb.RegisterVolumeServerServer(server, vs)
+	go func() {
+		_ = server.Serve(listener)
+	}()
+
+	t.Cleanup(func() {
+		vs.Shutdown()
+	})
+
+	return vs
+}
+
+// Address returns the gRPC address of the volume server.
+func (v *VolumeServer) Address() string {
+	return v.address
+}
+
+// BaseDir returns the base directory used by the server.
+func (v *VolumeServer) BaseDir() string {
+	return v.baseDir
+}
+
+// ReceivedFiles returns a snapshot of received files and byte counts.
+func (v *VolumeServer) ReceivedFiles() map[string]uint64 {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+
+	out := make(map[string]uint64, len(v.receivedFiles))
+	for key, value := range v.receivedFiles {
+		out[key] = value
+	}
+	return out
+}
+
+// SetVacuumGarbageRatio sets the garbage ratio returned by VacuumVolumeCheck.
+func (v *VolumeServer) SetVacuumGarbageRatio(ratio float64) {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	v.vacuumGarbageRatio = ratio
+}
+
+// SetVacuumCommitReadOnly sets the IsReadOnly value returned by VacuumVolumeCommit.
+func (v *VolumeServer) SetVacuumCommitReadOnly(readOnly bool) {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	v.vacuumCommitReadOnly = readOnly
+}
+
+// VacuumStats returns the vacuum RPC call counts.
+func (v *VolumeServer) VacuumStats() (check, compact, commit, cleanup int) {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	return v.vacuumCheckCalls, v.vacuumCompactCalls, v.vacuumCommitCalls, v.vacuumCleanupCalls
+}
+
+// BalanceStats returns the balance RPC call counts.
+func (v *VolumeServer) BalanceStats() (copyCalls, mountCalls, tailCalls int) {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	return v.volumeCopyCalls, v.volumeMountCalls, v.tailReceiverCalls
+}
+
+// MountRequests returns recorded mount requests.
+func (v *VolumeServer) MountRequests() []*volume_server_pb.VolumeEcShardsMountRequest {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+
+	out := make([]*volume_server_pb.VolumeEcShardsMountRequest, len(v.mountRequests))
+	copy(out, v.mountRequests)
+	return out
+}
+
+// DeleteRequests returns recorded delete requests.
+func (v *VolumeServer) DeleteRequests() []*volume_server_pb.VolumeDeleteRequest {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+
+	out := make([]*volume_server_pb.VolumeDeleteRequest, len(v.deleteRequests))
+	copy(out, v.deleteRequests)
+	return out
+}
+
+// MarkReadonlyCount returns the number of readonly calls.
+func (v *VolumeServer) MarkReadonlyCount() int {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	return v.markReadonlyCalls
+}
+
+// MarkWritableCount returns the number of writable calls.
+func (v *VolumeServer) MarkWritableCount() int {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	return v.markWritableCalls
+}
+
+// ReadFileStatusCount returns the number of ReadVolumeFileStatus calls.
+func (v *VolumeServer) ReadFileStatusCount() int {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	return v.readFileStatusCalls
+}
+
+// Shutdown stops the volume server.
+func (v *VolumeServer) Shutdown() {
+	if v.server != nil {
+		v.server.GracefulStop()
+	}
+	if v.listener != nil {
+		_ = v.listener.Close()
+	}
+}
+
+func (v *VolumeServer) filePath(volumeID uint32, ext string) string {
+	return filepath.Join(v.baseDir, fmt.Sprintf("%d%s", volumeID, ext))
+}
+
+func (v *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream volume_server_pb.VolumeServer_CopyFileServer) error {
+	if req == nil {
+		return fmt.Errorf("copy file request is nil")
+	}
+	path := v.filePath(req.VolumeId, req.Ext)
+	file, err := os.Open(path)
+	if err != nil {
+		if req.IgnoreSourceFileNotFound {
+			return nil
+		}
+		return err
+	}
+	defer file.Close()
+
+	buf := make([]byte, 64*1024)
+	remaining := int64(req.GetStopOffset())
+	for {
+		if remaining == 0 {
+			break
+		}
+
+		readBuf := buf
+		if remaining > 0 && remaining < int64(len(buf)) {
+			readBuf = buf[:remaining]
+		}
+
+		n, readErr := file.Read(readBuf)
+		if n > 0 {
+			if err := stream.Send(&volume_server_pb.CopyFileResponse{FileContent: readBuf[:n]}); err != nil {
+				return err
+			}
+			if remaining > 0 {
+				remaining -= int64(n)
+			}
+		}
+		if readErr == io.EOF {
+			break
+		}
+		if readErr != nil {
+			return readErr
+		}
+	}
+	return nil
+}
+
+func (v *VolumeServer) ReceiveFile(stream volume_server_pb.VolumeServer_ReceiveFileServer) error {
+	var (
+		info         *volume_server_pb.ReceiveFileInfo
+		file         *os.File
+		bytesWritten uint64
+		filePath     string
+	)
+	defer func() {
+		if file != nil {
+			_ = file.Close()
+		}
+	}()
+
+	for {
+		req, err := stream.Recv()
+		if err == io.EOF {
+			if info == nil {
+				return stream.SendAndClose(&volume_server_pb.ReceiveFileResponse{Error: "missing file info"})
+			}
+			v.mu.Lock()
+			v.receivedFiles[filePath] = bytesWritten
+			v.mu.Unlock()
+			return stream.SendAndClose(&volume_server_pb.ReceiveFileResponse{BytesWritten: bytesWritten})
+		}
+		if err != nil {
+			return err
+		}
+
+		if reqInfo := req.GetInfo(); reqInfo != nil {
+			info = reqInfo
+			filePath = v.filePath(info.VolumeId, info.Ext)
+			if err := os.MkdirAll(filepath.Dir(filePath), 0755); err != nil {
+				return err
+			}
+			file, err = os.Create(filePath)
+			if err != nil {
+				return err
+			}
+			continue
+		}
+
+		chunk := req.GetFileContent()
+		if len(chunk) == 0 {
+			continue
+		}
+		if file == nil {
+			return fmt.Errorf("file info not received")
+		}
+		n, writeErr := file.Write(chunk)
+		if writeErr != nil {
+			return writeErr
+		}
+		bytesWritten += uint64(n)
+	}
+}
+
+func (v *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_server_pb.VolumeEcShardsMountRequest) (*volume_server_pb.VolumeEcShardsMountResponse, error) {
+	v.mu.Lock()
+	v.mountRequests = append(v.mountRequests, req)
+	v.mu.Unlock()
+	return &volume_server_pb.VolumeEcShardsMountResponse{}, nil
+}
+
+// VolumeEcShardsUnmount is a no-op stub: the worker's pre-distribute
+// cleanup calls it against every destination, and the fake server has no
+// mounted state to clear.
+func (v *VolumeServer) VolumeEcShardsUnmount(ctx context.Context, req *volume_server_pb.VolumeEcShardsUnmountRequest) (*volume_server_pb.VolumeEcShardsUnmountResponse, error) {
+	return &volume_server_pb.VolumeEcShardsUnmountResponse{}, nil
+}
+
+// VolumeEcShardsDelete is a no-op stub paired with VolumeEcShardsUnmount
+// above; the fake server doesn't persist shard files beyond what
+// ReceiveFile wrote, so there's nothing to remove. It still echoes the
+// full_teardown acknowledgement so the worker doesn't treat the fake as a
+// pre-upgrade server that silently skipped the teardown.
+func (v *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_server_pb.VolumeEcShardsDeleteRequest) (*volume_server_pb.VolumeEcShardsDeleteResponse, error) {
+	return &volume_server_pb.VolumeEcShardsDeleteResponse{FullTeardownDone: req.FullTeardown}, nil
+}
+
+func (v *VolumeServer) VolumeEcShardsInfo(ctx context.Context, req *volume_server_pb.VolumeEcShardsInfoRequest) (*volume_server_pb.VolumeEcShardsInfoResponse, error) {
+	if req == nil {
+		return nil, fmt.Errorf("VolumeEcShardsInfo request is nil")
+	}
+	v.mu.Lock()
+	defer v.mu.Unlock()
+
+	// Report whichever shards exist on disk: seeded or mounted. Collection
+	// comes from the matching mount request when one exists.
+	collectionByShard := make(map[uint32]string)
+	for _, mr := range v.mountRequests {
+		if mr == nil || mr.VolumeId != req.VolumeId {
+			continue
+		}
+		for _, shardId := range mr.ShardIds {
+			if _, ok := collectionByShard[shardId]; !ok {
+				collectionByShard[shardId] = mr.Collection
+			}
+		}
+	}
+
+	// Answer with the layout out of the .vif that was actually delivered here,
+	// the way a real holder answers from the context it mounted the shards
+	// with. A coordinator uses this to tell a server that understands the
+	// shard block layout from one that never knew the field, so a fake that
+	// always reported "unset" would look like a pre-upgrade server.
+	resp := &volume_server_pb.VolumeEcShardsInfoResponse{
+		EcShardConfig: v.ecShardConfigFromVif(req.VolumeId),
+	}
+	prefix := fmt.Sprintf("%d.ec", req.VolumeId)
+	entries, _ := os.ReadDir(v.baseDir)
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+		name := entry.Name()
+		if !strings.HasPrefix(name, prefix) {
+			continue
+		}
+		suffix := strings.TrimPrefix(name, prefix)
+		if len(suffix) < 2 {
+			continue
+		}
+		var shardId uint32
+		if _, err := fmt.Sscanf(suffix[:2], "%d", &shardId); err != nil {
+			continue
+		}
+		var size int64
+		if info, err := entry.Info(); err == nil {
+			size = info.Size()
+		}
+		resp.EcShardInfos = append(resp.EcShardInfos, &volume_server_pb.EcShardInfo{
+			ShardId:    shardId,
+			Size:       size,
+			Collection: collectionByShard[shardId],
+			VolumeId:   req.VolumeId,
+		})
+	}
+	return resp, nil
+}
+
+// ecShardConfigFromVif reads the EC layout out of the .vif this server was
+// given, which distribution ships to every holder alongside its shards.
+func (v *VolumeServer) ecShardConfigFromVif(volumeID uint32) *volume_server_pb.EcShardConfig {
+	vi, _, found, err := volume_info.MaybeLoadVolumeInfo(v.filePath(volumeID, ".vif"))
+	if err != nil || !found {
+		return nil
+	}
+	return vi.GetEcShardConfig()
+}
+
+func (v *VolumeServer) VolumeDelete(ctx context.Context, req *volume_server_pb.VolumeDeleteRequest) (*volume_server_pb.VolumeDeleteResponse, error) {
+	v.mu.Lock()
+	v.deleteRequests = append(v.deleteRequests, req)
+	v.mu.Unlock()
+
+	if req != nil {
+		_ = os.Remove(v.filePath(req.VolumeId, ".dat"))
+		_ = os.Remove(v.filePath(req.VolumeId, ".idx"))
+	}
+
+	return &volume_server_pb.VolumeDeleteResponse{}, nil
+}
+
+func (v *VolumeServer) VolumeMarkReadonly(ctx context.Context, req *volume_server_pb.VolumeMarkReadonlyRequest) (*volume_server_pb.VolumeMarkReadonlyResponse, error) {
+	v.mu.Lock()
+	v.markReadonlyCalls++
+	if req != nil {
+		v.readonlyVolumes[req.VolumeId] = true
+	}
+	v.mu.Unlock()
+	return &volume_server_pb.VolumeMarkReadonlyResponse{}, nil
+}
+
+func (v *VolumeServer) VolumeMarkWritable(ctx context.Context, req *volume_server_pb.VolumeMarkWritableRequest) (*volume_server_pb.VolumeMarkWritableResponse, error) {
+	v.mu.Lock()
+	v.markWritableCalls++
+	if req != nil {
+		v.readonlyVolumes[req.VolumeId] = false
+	}
+	v.mu.Unlock()
+	return &volume_server_pb.VolumeMarkWritableResponse{}, nil
+}
+
+func (v *VolumeServer) VolumeStatus(ctx context.Context, req *volume_server_pb.VolumeStatusRequest) (*volume_server_pb.VolumeStatusResponse, error) {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+	return &volume_server_pb.VolumeStatusResponse{IsReadOnly: v.readonlyVolumes[req.GetVolumeId()]}, nil
+}
+
+func (v *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_server_pb.ReadVolumeFileStatusRequest) (*volume_server_pb.ReadVolumeFileStatusResponse, error) {
+	v.mu.Lock()
+	v.readFileStatusCalls++
+	v.mu.Unlock()
+
+	datInfo, err := os.Stat(v.filePath(req.VolumeId, ".dat"))
+	if err != nil {
+		return nil, err
+	}
+
+	idxInfo, err := os.Stat(v.filePath(req.VolumeId, ".idx"))
+	if err != nil {
+		return nil, err
+	}
+
+	return &volume_server_pb.ReadVolumeFileStatusResponse{
+		VolumeId:    req.VolumeId,
+		DatFileSize: uint64(datInfo.Size()),
+		IdxFileSize: uint64(idxInfo.Size()),
+		FileCount:   1,
+	}, nil
+}
+
+func (v *VolumeServer) VacuumVolumeCheck(ctx context.Context, req *volume_server_pb.VacuumVolumeCheckRequest) (*volume_server_pb.VacuumVolumeCheckResponse, error) {
+	v.mu.Lock()
+	v.vacuumCheckCalls++
+	ratio := v.vacuumGarbageRatio
+	v.mu.Unlock()
+	return &volume_server_pb.VacuumVolumeCheckResponse{GarbageRatio: ratio}, nil
+}
+
+func (v *VolumeServer) VacuumVolumeCompact(req *volume_server_pb.VacuumVolumeCompactRequest, stream volume_server_pb.VolumeServer_VacuumVolumeCompactServer) error {
+	v.mu.Lock()
+	v.vacuumCompactCalls++
+	v.mu.Unlock()
+	return stream.Send(&volume_server_pb.VacuumVolumeCompactResponse{ProcessedBytes: 1024})
+}
+
+func (v *VolumeServer) VacuumVolumeCommit(ctx context.Context, req *volume_server_pb.VacuumVolumeCommitRequest) (*volume_server_pb.VacuumVolumeCommitResponse, error) {
+	v.mu.Lock()
+	v.vacuumCommitCalls++
+	readOnly := v.vacuumCommitReadOnly
+	v.mu.Unlock()
+	return &volume_server_pb.VacuumVolumeCommitResponse{IsReadOnly: readOnly}, nil
+}
+
+func (v *VolumeServer) VacuumVolumeCleanup(ctx context.Context, req *volume_server_pb.VacuumVolumeCleanupRequest) (*volume_server_pb.VacuumVolumeCleanupResponse, error) {
+	v.mu.Lock()
+	v.vacuumCleanupCalls++
+	v.mu.Unlock()
+	return &volume_server_pb.VacuumVolumeCleanupResponse{}, nil
+}
+
+func (v *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, stream volume_server_pb.VolumeServer_VolumeCopyServer) error {
+	v.mu.Lock()
+	v.volumeCopyCalls++
+	v.mu.Unlock()
+
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	var statusResp *volume_server_pb.ReadVolumeFileStatusResponse
+	if err := operation.WithVolumeServerClient(false, pb.ServerAddress(req.SourceDataNode), dialOption,
+		func(client volume_server_pb.VolumeServerClient) error {
+			var readErr error
+			statusResp, readErr = client.ReadVolumeFileStatus(stream.Context(), &volume_server_pb.ReadVolumeFileStatusRequest{
+				VolumeId: req.VolumeId,
+			})
+			return readErr
+		}); err != nil {
+		return err
+	}
+
+	if err := v.copyRemoteFile(stream.Context(), req.SourceDataNode, req.VolumeId, ".dat", statusResp.DatFileSize, dialOption); err != nil {
+		return err
+	}
+	if err := v.copyRemoteFile(stream.Context(), req.SourceDataNode, req.VolumeId, ".idx", statusResp.IdxFileSize, dialOption); err != nil {
+		return err
+	}
+
+	if err := stream.Send(&volume_server_pb.VolumeCopyResponse{ProcessedBytes: int64(statusResp.DatFileSize + statusResp.IdxFileSize)}); err != nil {
+		return err
+	}
+	return stream.Send(&volume_server_pb.VolumeCopyResponse{LastAppendAtNs: uint64(time.Now().UnixNano())})
+}
+
+func (v *VolumeServer) VolumeMount(ctx context.Context, req *volume_server_pb.VolumeMountRequest) (*volume_server_pb.VolumeMountResponse, error) {
+	v.mu.Lock()
+	v.volumeMountCalls++
+	v.mu.Unlock()
+	return &volume_server_pb.VolumeMountResponse{}, nil
+}
+
+func (v *VolumeServer) VolumeTailReceiver(ctx context.Context, req *volume_server_pb.VolumeTailReceiverRequest) (*volume_server_pb.VolumeTailReceiverResponse, error) {
+	v.mu.Lock()
+	v.tailReceiverCalls++
+	v.mu.Unlock()
+	return &volume_server_pb.VolumeTailReceiverResponse{}, nil
+}
+
+func (v *VolumeServer) copyRemoteFile(ctx context.Context, sourceDataNode string, volumeID uint32, ext string, fileSize uint64, dialOption grpc.DialOption) error {
+	path := v.filePath(volumeID, ext)
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return err
+	}
+
+	file, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	return operation.WithVolumeServerClient(true, pb.ServerAddress(sourceDataNode), dialOption,
+		func(client volume_server_pb.VolumeServerClient) error {
+			stream, err := client.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+				VolumeId:   volumeID,
+				Ext:        ext,
+				StopOffset: fileSize,
+			})
+			if err != nil {
+				return err
+			}
+
+			for {
+				resp, recvErr := stream.Recv()
+				if recvErr == io.EOF {
+					return nil
+				}
+				if recvErr != nil {
+					return recvErr
+				}
+				if len(resp.FileContent) == 0 {
+					continue
+				}
+				if _, err := file.Write(resp.FileContent); err != nil {
+					return err
+				}
+			}
+		})
+}

test/plugin_workers/framework.go

@@ -0,0 +1,171 @@
+package pluginworkers
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// HarnessConfig configures the shared plugin worker test harness.
+type HarnessConfig struct {
+	PluginOptions plugin.Options
+	WorkerOptions pluginworker.WorkerOptions
+	Handlers      []pluginworker.JobHandler
+}
+
+// Harness manages an in-process plugin admin server and worker.
+type Harness struct {
+	t *testing.T
+
+	pluginSvc *plugin.Plugin
+
+	adminServer   *grpc.Server
+	adminListener net.Listener
+	adminGrpcAddr string
+
+	worker       *pluginworker.Worker
+	workerCtx    context.Context
+	workerCancel context.CancelFunc
+	workerDone   chan struct{}
+}
+
+// NewHarness starts a plugin admin gRPC server and a worker connected to it.
+func NewHarness(t *testing.T, cfg HarnessConfig) *Harness {
+	t.Helper()
+
+	pluginOpts := cfg.PluginOptions
+	if pluginOpts.DataDir == "" {
+		pluginOpts.DataDir = t.TempDir()
+	}
+
+	pluginSvc, err := plugin.New(pluginOpts)
+	require.NoError(t, err)
+
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+
+	adminServer := pb.NewGrpcServer()
+	plugin_pb.RegisterPluginControlServiceServer(adminServer, pluginSvc)
+	go func() {
+		_ = adminServer.Serve(listener)
+	}()
+
+	adminGrpcAddr := listener.Addr().String()
+	adminPort := listener.Addr().(*net.TCPAddr).Port
+	adminAddr := fmt.Sprintf("127.0.0.1:0.%d", adminPort)
+
+	workerOpts := cfg.WorkerOptions
+	if workerOpts.AdminServer == "" {
+		workerOpts.AdminServer = adminAddr
+	}
+	if workerOpts.GrpcDialOption == nil {
+		workerOpts.GrpcDialOption = grpc.WithTransportCredentials(insecure.NewCredentials())
+	}
+	if workerOpts.WorkerID == "" {
+		workerOpts.WorkerID = "plugin-worker-test"
+	}
+	if workerOpts.WorkerVersion == "" {
+		workerOpts.WorkerVersion = "test"
+	}
+	if workerOpts.WorkerAddress == "" {
+		workerOpts.WorkerAddress = "127.0.0.1"
+	}
+	if len(cfg.Handlers) > 0 {
+		workerOpts.Handlers = cfg.Handlers
+	}
+
+	worker, err := pluginworker.NewWorker(workerOpts)
+	require.NoError(t, err)
+
+	workerCtx, workerCancel := context.WithCancel(context.Background())
+	workerDone := make(chan struct{})
+	go func() {
+		defer close(workerDone)
+		_ = worker.Run(workerCtx)
+	}()
+
+	harness := &Harness{
+		t:             t,
+		pluginSvc:     pluginSvc,
+		adminServer:   adminServer,
+		adminListener: listener,
+		adminGrpcAddr: adminGrpcAddr,
+		worker:        worker,
+		workerCtx:     workerCtx,
+		workerCancel:  workerCancel,
+		workerDone:    workerDone,
+	}
+
+	require.Eventually(t, func() bool {
+		return len(pluginSvc.ListWorkers()) > 0
+	}, 5*time.Second, 50*time.Millisecond)
+
+	t.Cleanup(func() {
+		harness.Shutdown()
+	})
+
+	return harness
+}
+
+// Plugin exposes the underlying admin plugin service.
+func (h *Harness) Plugin() *plugin.Plugin {
+	return h.pluginSvc
+}
+
+// AdminGrpcAddress returns the gRPC address for the admin server.
+func (h *Harness) AdminGrpcAddress() string {
+	return h.adminGrpcAddr
+}
+
+// WaitForJobType waits until a worker with the given capability is registered.
+func (h *Harness) WaitForJobType(jobType string) {
+	h.t.Helper()
+	require.Eventually(h.t, func() bool {
+		workers := h.pluginSvc.ListWorkers()
+		for _, worker := range workers {
+			if worker == nil || worker.Capabilities == nil {
+				continue
+			}
+			if _, ok := worker.Capabilities[jobType]; ok {
+				return true
+			}
+		}
+		return false
+	}, 5*time.Second, 50*time.Millisecond)
+}
+
+// Shutdown stops the worker and admin server.
+func (h *Harness) Shutdown() {
+	if h.workerCancel != nil {
+		h.workerCancel()
+	}
+
+	if h.workerDone != nil {
+		select {
+		case <-h.workerDone:
+		case <-time.After(2 * time.Second):
+		}
+	}
+
+	if h.adminServer != nil {
+		h.adminServer.GracefulStop()
+	}
+
+	if h.adminListener != nil {
+		_ = h.adminListener.Close()
+	}
+
+	if h.pluginSvc != nil {
+		h.pluginSvc.Shutdown()
+	}
+}

test/plugin_workers/vacuum/detection_test.go

@@ -0,0 +1,105 @@
+package vacuum_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	pluginworkers "github.com/seaweedfs/seaweedfs/test/plugin_workers"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/protobuf/proto"
+)
+
+func TestVacuumDetectionIntegration(t *testing.T) {
+	volumeID := uint32(101)
+	source := pluginworkers.NewVolumeServer(t, "")
+
+	response := buildVacuumVolumeListResponse(t, source.Address(), volumeID, 0.6)
+	master := pluginworkers.NewMasterServer(t, response)
+
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	handler := vacuum.NewVacuumHandler(dialOption, 1)
+	harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
+		WorkerOptions: pluginworker.WorkerOptions{
+			GrpcDialOption: dialOption,
+		},
+		Handlers: []pluginworker.JobHandler{handler},
+	})
+	harness.WaitForJobType("vacuum")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	proposals, err := harness.Plugin().RunDetection(ctx, "vacuum", &plugin_pb.ClusterContext{
+		MasterGrpcAddresses: []string{master.Address()},
+	}, 10)
+	require.NoError(t, err)
+	require.NotEmpty(t, proposals)
+
+	proposal := proposals[0]
+	require.Equal(t, "vacuum", proposal.JobType)
+	paramsValue := proposal.Parameters["task_params_pb"]
+	require.NotNil(t, paramsValue)
+
+	params := &worker_pb.TaskParams{}
+	require.NoError(t, proto.Unmarshal(paramsValue.GetBytesValue(), params))
+	require.NotEmpty(t, params.Sources)
+	require.NotNil(t, params.GetVacuumParams())
+}
+
+func buildVacuumVolumeListResponse(t *testing.T, serverAddress string, volumeID uint32, garbageRatio float64) *master_pb.VolumeListResponse {
+	t.Helper()
+
+	volumeSizeLimitMB := uint64(100)
+	volumeSize := uint64(90) * 1024 * 1024
+	deletedBytes := uint64(float64(volumeSize) * garbageRatio)
+	volumeModifiedAt := time.Now().Add(-48 * time.Hour).Unix()
+
+	diskInfo := &master_pb.DiskInfo{
+		DiskId:         0,
+		MaxVolumeCount: 100,
+		VolumeCount:    1,
+		VolumeInfos: []*master_pb.VolumeInformationMessage{
+			{
+				Id:               volumeID,
+				Collection:       "vac-test",
+				DiskId:           0,
+				Size:             volumeSize,
+				DeletedByteCount: deletedBytes,
+				ModifiedAtSecond: volumeModifiedAt,
+				ReplicaPlacement: 1,
+				ReadOnly:         false,
+			},
+		},
+	}
+
+	node := &master_pb.DataNodeInfo{
+		Id:        serverAddress,
+		Address:   serverAddress,
+		DiskInfos: map[string]*master_pb.DiskInfo{"hdd": diskInfo},
+	}
+
+	return &master_pb.VolumeListResponse{
+		VolumeSizeLimitMb: volumeSizeLimitMB,
+		TopologyInfo: &master_pb.TopologyInfo{
+			DataCenterInfos: []*master_pb.DataCenterInfo{
+				{
+					Id: "dc-1",
+					RackInfos: []*master_pb.RackInfo{
+						{
+							Id:            "rack-1",
+							DataNodeInfos: []*master_pb.DataNodeInfo{node},
+						},
+					},
+				},
+			},
+		},
+	}
+}

test/plugin_workers/vacuum/execution_test.go

@@ -0,0 +1,108 @@
+package vacuum_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	pluginworkers "github.com/seaweedfs/seaweedfs/test/plugin_workers"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+func vacuumJobSpec(volumeID uint32, server string) *plugin_pb.JobSpec {
+	return &plugin_pb.JobSpec{
+		JobId:   fmt.Sprintf("vacuum-job-%d", volumeID),
+		JobType: "vacuum",
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"volume_id": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(volumeID)},
+			},
+			"server": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: server},
+			},
+			"collection": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "vac-test"},
+			},
+		},
+	}
+}
+
+func TestVacuumExecutionIntegration(t *testing.T) {
+	volumeID := uint32(202)
+
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	handler := vacuum.NewVacuumHandler(dialOption, 1)
+	harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
+		WorkerOptions: pluginworker.WorkerOptions{
+			GrpcDialOption: dialOption,
+		},
+		Handlers: []pluginworker.JobHandler{handler},
+	})
+	harness.WaitForJobType("vacuum")
+
+	source := pluginworkers.NewVolumeServer(t, "")
+	source.SetVacuumGarbageRatio(0.6)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	result, err := harness.Plugin().ExecuteJob(ctx, vacuumJobSpec(volumeID, source.Address()), nil, 1)
+	require.NoError(t, err)
+	require.NotNil(t, result)
+	require.True(t, result.Success)
+
+	checkCalls, compactCalls, commitCalls, cleanupCalls := source.VacuumStats()
+	require.GreaterOrEqual(t, checkCalls, 2)
+	require.GreaterOrEqual(t, compactCalls, 1)
+	require.GreaterOrEqual(t, commitCalls, 1)
+	// Cleanup is only invoked when Phase 1 (Compact) fails to roll back
+	// the .cpd/.cpx/.cpldb temp files; on the success path Commit
+	// consumes them (rename .cpd → .dat, .cpx → .idx, .cpldb → .ldb via
+	// the leveldb needle map) so no Cleanup call is needed. Matches
+	// topology.vacuumOneVolumeId which only calls batchVacuumVolumeCleanup
+	// on the Compact-failure branch.
+	require.Equal(t, 0, cleanupCalls)
+	// Phase 3 marks each replica writable so master returns it to the
+	// writables layout. See upstream seaweedfs#9685.
+	require.GreaterOrEqual(t, source.MarkWritableCount(), 1)
+}
+
+// A replica that commits still read-only (operator-set, EIO-quarantined,
+// disk-space-low) must not be force-marked writable: master built-in vacuum
+// skips it via SetVolumeAvailable, and it recovers on its own ReadOnly=false
+// heartbeat.
+func TestVacuumExecutionSkipsMarkWritableWhenReadOnly(t *testing.T) {
+	volumeID := uint32(203)
+
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	handler := vacuum.NewVacuumHandler(dialOption, 1)
+	harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
+		WorkerOptions: pluginworker.WorkerOptions{
+			GrpcDialOption: dialOption,
+		},
+		Handlers: []pluginworker.JobHandler{handler},
+	})
+	harness.WaitForJobType("vacuum")
+
+	source := pluginworkers.NewVolumeServer(t, "")
+	source.SetVacuumGarbageRatio(0.6)
+	source.SetVacuumCommitReadOnly(true)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	result, err := harness.Plugin().ExecuteJob(ctx, vacuumJobSpec(volumeID, source.Address()), nil, 1)
+	require.NoError(t, err)
+	require.NotNil(t, result)
+	require.True(t, result.Success)
+
+	_, _, commitCalls, _ := source.VacuumStats()
+	require.GreaterOrEqual(t, commitCalls, 1)
+	require.Equal(t, 0, source.MarkWritableCount())
+}

test/plugin_workers/volume_balance/detection_test.go

@@ -0,0 +1,139 @@
+package volume_balance_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	pluginworkers "github.com/seaweedfs/seaweedfs/test/plugin_workers"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/protobuf/proto"
+)
+
+func TestVolumeBalanceDetectionIntegration(t *testing.T) {
+	response := buildBalanceVolumeListResponse(t)
+	master := pluginworkers.NewMasterServer(t, response)
+
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	handler := balance.NewVolumeBalanceHandler(dialOption)
+	harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
+		WorkerOptions: pluginworker.WorkerOptions{
+			GrpcDialOption: dialOption,
+		},
+		Handlers: []pluginworker.JobHandler{handler},
+	})
+	harness.WaitForJobType("volume_balance")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	proposals, err := harness.Plugin().RunDetection(ctx, "volume_balance", &plugin_pb.ClusterContext{
+		MasterGrpcAddresses: []string{master.Address()},
+	}, 10)
+	require.NoError(t, err)
+	// With default batch_size=20 and 10 overloaded volumes vs 1 underloaded,
+	// all moves are grouped into a single batch proposal.
+	require.Len(t, proposals, 1, "expected exactly one batch proposal")
+
+	proposal := proposals[0]
+	require.Equal(t, "volume_balance", proposal.JobType)
+	paramsValue := proposal.Parameters["task_params_pb"]
+	require.NotNil(t, paramsValue)
+
+	params := &worker_pb.TaskParams{}
+	require.NoError(t, proto.Unmarshal(paramsValue.GetBytesValue(), params))
+
+	bp := params.GetBalanceParams()
+	require.NotNil(t, bp, "expected BalanceParams in batch proposal")
+	require.Greater(t, len(bp.Moves), 1, "batch proposal should contain multiple moves")
+	for _, move := range bp.Moves {
+		require.NotZero(t, move.VolumeId)
+		require.NotEmpty(t, move.SourceNode)
+		require.NotEmpty(t, move.TargetNode)
+	}
+}
+
+func buildBalanceVolumeListResponse(t *testing.T) *master_pb.VolumeListResponse {
+	t.Helper()
+
+	volumeSizeLimitMB := uint64(100)
+	volumeModifiedAt := time.Now().Add(-2 * time.Hour).Unix()
+
+	overloadedVolumes := make([]*master_pb.VolumeInformationMessage, 0, 10)
+	for i := 0; i < 10; i++ {
+		volumeID := uint32(1000 + i)
+		overloadedVolumes = append(overloadedVolumes, &master_pb.VolumeInformationMessage{
+			Id:               volumeID,
+			Collection:       "balance",
+			DiskId:           0,
+			Size:             20 * 1024 * 1024,
+			DeletedByteCount: 0,
+			ModifiedAtSecond: volumeModifiedAt,
+			ReplicaPlacement: 1,
+			ReadOnly:         false,
+		})
+	}
+
+	underloadedVolumes := []*master_pb.VolumeInformationMessage{
+		{
+			Id:               2000,
+			Collection:       "balance",
+			DiskId:           0,
+			Size:             20 * 1024 * 1024,
+			DeletedByteCount: 0,
+			ModifiedAtSecond: volumeModifiedAt,
+			ReplicaPlacement: 1,
+			ReadOnly:         false,
+		},
+	}
+
+	overloadedDisk := &master_pb.DiskInfo{
+		DiskId:         0,
+		MaxVolumeCount: 100,
+		VolumeCount:    int64(len(overloadedVolumes)),
+		VolumeInfos:    overloadedVolumes,
+	}
+
+	underloadedDisk := &master_pb.DiskInfo{
+		DiskId:         0,
+		MaxVolumeCount: 100,
+		VolumeCount:    int64(len(underloadedVolumes)),
+		VolumeInfos:    underloadedVolumes,
+	}
+
+	overloadedNode := &master_pb.DataNodeInfo{
+		Id:        "10.0.0.1:8080",
+		Address:   "10.0.0.1:8080",
+		DiskInfos: map[string]*master_pb.DiskInfo{"hdd": overloadedDisk},
+	}
+
+	underloadedNode := &master_pb.DataNodeInfo{
+		Id:        "10.0.0.2:8080",
+		Address:   "10.0.0.2:8080",
+		DiskInfos: map[string]*master_pb.DiskInfo{"hdd": underloadedDisk},
+	}
+
+	rack := &master_pb.RackInfo{
+		Id:            "rack-1",
+		DataNodeInfos: []*master_pb.DataNodeInfo{overloadedNode, underloadedNode},
+	}
+
+	return &master_pb.VolumeListResponse{
+		VolumeSizeLimitMb: volumeSizeLimitMB,
+		TopologyInfo: &master_pb.TopologyInfo{
+			DataCenterInfos: []*master_pb.DataCenterInfo{
+				{
+					Id:        "dc-1",
+					RackInfos: []*master_pb.RackInfo{rack},
+				},
+			},
+		},
+	}
+}

test/plugin_workers/volume_balance/execution_test.go

@@ -0,0 +1,162 @@
+package volume_balance_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	pluginworkers "github.com/seaweedfs/seaweedfs/test/plugin_workers"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/protobuf/proto"
+)
+
+const testVolumeDatSize = 1 * 1024 * 1024
+
+func TestVolumeBalanceExecutionIntegration(t *testing.T) {
+	volumeID := uint32(303)
+
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	handler := balance.NewVolumeBalanceHandler(dialOption)
+	harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
+		WorkerOptions: pluginworker.WorkerOptions{
+			GrpcDialOption: dialOption,
+		},
+		Handlers: []pluginworker.JobHandler{handler},
+	})
+	harness.WaitForJobType("volume_balance")
+
+	source := pluginworkers.NewVolumeServer(t, "")
+	target := pluginworkers.NewVolumeServer(t, "")
+	pluginworkers.WriteTestVolumeFiles(t, source.BaseDir(), volumeID, testVolumeDatSize)
+
+	job := &plugin_pb.JobSpec{
+		JobId:   fmt.Sprintf("balance-job-%d", volumeID),
+		JobType: "volume_balance",
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"volume_id": {
+				Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(volumeID)},
+			},
+			"collection": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "balance"},
+			},
+			"source_server": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: source.Address()},
+			},
+			"target_server": {
+				Kind: &plugin_pb.ConfigValue_StringValue{StringValue: target.Address()},
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	result, err := harness.Plugin().ExecuteJob(ctx, job, nil, 1)
+	require.NoError(t, err)
+	require.NotNil(t, result)
+	require.True(t, result.Success)
+
+	require.GreaterOrEqual(t, source.MarkReadonlyCount(), 1)
+	require.GreaterOrEqual(t, len(source.DeleteRequests()), 1)
+
+	copyCalls, _, tailCalls := target.BalanceStats()
+	require.GreaterOrEqual(t, copyCalls, 1)
+	require.GreaterOrEqual(t, tailCalls, 1)
+}
+
+func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
+	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
+	handler := balance.NewVolumeBalanceHandler(dialOption)
+	harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
+		WorkerOptions: pluginworker.WorkerOptions{
+			GrpcDialOption: dialOption,
+		},
+		Handlers: []pluginworker.JobHandler{handler},
+	})
+	harness.WaitForJobType("volume_balance")
+
+	// Create one source and one target fake volume server.
+	source := pluginworkers.NewVolumeServer(t, "")
+	target := pluginworkers.NewVolumeServer(t, "")
+
+	// Build a batch job with 3 volume moves from source → target.
+	volumeIDs := []uint32{401, 402, 403}
+	for _, vid := range volumeIDs {
+		pluginworkers.WriteTestVolumeFiles(t, source.BaseDir(), vid, testVolumeDatSize)
+	}
+	moves := make([]*worker_pb.BalanceMoveSpec, len(volumeIDs))
+	for i, vid := range volumeIDs {
+		moves[i] = &worker_pb.BalanceMoveSpec{
+			VolumeId:   vid,
+			SourceNode: source.Address(),
+			TargetNode: target.Address(),
+			Collection: "batch-test",
+		}
+	}
+
+	params := &worker_pb.TaskParams{
+		TaskId: "batch-balance-test",
+		TaskParams: &worker_pb.TaskParams_BalanceParams{
+			BalanceParams: &worker_pb.BalanceTaskParams{
+				MaxConcurrentMoves: 2,
+				Moves:              moves,
+			},
+		},
+	}
+	paramBytes, err := proto.Marshal(params)
+	require.NoError(t, err)
+
+	job := &plugin_pb.JobSpec{
+		JobId:   "batch-balance-test",
+		JobType: "volume_balance",
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"task_params_pb": {
+				Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: paramBytes},
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	result, err := harness.Plugin().ExecuteJob(ctx, job, nil, 1)
+	require.NoError(t, err)
+	require.NotNil(t, result)
+	require.True(t, result.Success, "batch balance job should succeed; result: %+v", result)
+
+	// Each of the 3 moves should have marked the source readonly and deleted.
+	require.Equal(t, len(volumeIDs), source.MarkReadonlyCount(),
+		"each move should mark source volume readonly")
+	require.Equal(t, len(volumeIDs), len(source.DeleteRequests()),
+		"each move should delete the source volume")
+
+	// Verify delete requests reference the expected volume IDs.
+	deletedVols := make(map[uint32]bool)
+	for _, req := range source.DeleteRequests() {
+		deletedVols[req.VolumeId] = true
+	}
+	for _, vid := range volumeIDs {
+		require.True(t, deletedVols[vid], "volume %d should have been deleted from source", vid)
+	}
+
+	// Each move reads source status once inside the target's fake VolumeCopy
+	// implementation and once for the pre-delete verification, and reads
+	// target status once probing for a pre-existing copy and once for the
+	// verification.
+	require.Equal(t, len(volumeIDs)*2, source.ReadFileStatusCount(),
+		"each move should read source volume status during target copy and at verification")
+	require.Equal(t, len(volumeIDs)*2, target.ReadFileStatusCount(),
+		"each move should probe the target and read its status at verification")
+
+	// Target should have received copy and tail calls for all 3 volumes.
+	copyCalls, _, tailCalls := target.BalanceStats()
+	require.Equal(t, len(volumeIDs), copyCalls, "target should receive one copy per volume")
+	require.Equal(t, len(volumeIDs), tailCalls, "target should receive one tail per volume")
+}

test/plugin_workers/volume_fixtures.go

@@ -0,0 +1,55 @@
+package pluginworkers
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// WriteTestVolumeFiles creates a minimal .dat/.idx pair for the given volume.
+func WriteTestVolumeFiles(t *testing.T, baseDir string, volumeID uint32, datSize int) (string, string) {
+	t.Helper()
+
+	if err := os.MkdirAll(baseDir, 0755); err != nil {
+		t.Fatalf("create volume dir: %v", err)
+	}
+
+	datPath := filepath.Join(baseDir, volumeFilename(volumeID, ".dat"))
+	idxPath := filepath.Join(baseDir, volumeFilename(volumeID, ".idx"))
+
+	// The idx entry records the needle's data size. The actual on-disk size
+	// includes header + checksum + timestamp (GetActualSize). The .dat must
+	// be large enough to hold the full needle.
+	needleDataSize := types.Size(datSize)
+	actualSize := needle.GetActualSize(needleDataSize, needle.Version3)
+	data := make([]byte, actualSize)
+	rng := rand.New(rand.NewSource(99))
+	_, _ = rng.Read(data)
+	if err := os.WriteFile(datPath, data, 0644); err != nil {
+		t.Fatalf("write dat file: %v", err)
+	}
+
+	entry := make([]byte, types.NeedleMapEntrySize)
+	idEnd := types.NeedleIdSize
+	offsetEnd := idEnd + types.OffsetSize
+	sizeEnd := offsetEnd + types.SizeSize
+
+	types.NeedleIdToBytes(entry[:idEnd], types.NeedleId(1))
+	types.OffsetToBytes(entry[idEnd:offsetEnd], types.ToOffset(0))
+	types.SizeToBytes(entry[offsetEnd:sizeEnd], needleDataSize)
+
+	if err := os.WriteFile(idxPath, entry, 0644); err != nil {
+		t.Fatalf("write idx file: %v", err)
+	}
+
+	return datPath, idxPath
+}
+
+func volumeFilename(volumeID uint32, ext string) string {
+	return fmt.Sprintf("%d%s", volumeID, ext)
+}