snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/ec/ec_balance_migrate_test.go

@@ -0,0 +1,63 @@
+package ec
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// TestEcBalanceMigratesCrossDiskTypeShards: a cross-tier encode generates its
+// shards beside the source .dat — in the SOURCE disk-type bucket — while the
+// encode's balance targets -diskType. The balance must still see and spread
+// those shards when the volume is named as migrating; without that, the
+// planner finds nothing in the target bucket, plans no moves, and the encode's
+// spread guard aborts a perfectly good encode.
+func TestEcBalanceMigratesCrossDiskTypeShards(t *testing.T) {
+	allShards := []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}
+
+	build := func() []*EcNode {
+		// Volume 1's fresh shards sit in the default (hdd, "") bucket of dn1;
+		// the balance runs with -diskType=ssd.
+		return []*EcNode{
+			newEcNode("dc1", "rack1", "dn1", 100).addEcVolumeAndShardsForTest(1, "c1", allShards),
+			newEcNode("dc1", "rack2", "dn2", 100),
+			newEcNode("dc1", "rack3", "dn3", 100),
+		}
+	}
+
+	// Without the migrating hint the target-bucket filter hides the shards:
+	// nothing moves. This is the standalone ec.balance semantic — shards of
+	// other tiers stay where they are.
+	ecb := &ecBalancer{
+		ecNodes:        build(),
+		applyBalancing: false,
+		diskType:       types.SsdType,
+	}
+	if err := ecb.balance([]string{"c1"}); err != nil {
+		t.Fatalf("balance without migrating hint: %v", err)
+	}
+	if got := ecb.ecNodes[0].LocalShardIdCount(1); got != len(allShards) {
+		t.Fatalf("without migrating hint, cross-type shards must stay put; dn1 has %d", got)
+	}
+
+	// With the volume named as migrating, the balance ingests the shards from
+	// the source bucket and spreads them.
+	ecb = &ecBalancer{
+		ecNodes:            build(),
+		applyBalancing:     false,
+		diskType:           types.SsdType,
+		migratingVolumeIds: map[uint32]bool{1: true},
+	}
+	if err := ecb.balance([]string{"c1"}); err != nil {
+		t.Fatalf("balance with migrating hint: %v", err)
+	}
+	onDn1 := ecb.ecNodes[0].LocalShardIdCount(1)
+	spread := ecb.ecNodes[1].LocalShardIdCount(1) + ecb.ecNodes[2].LocalShardIdCount(1)
+	if onDn1 == len(allShards) || spread == 0 {
+		t.Fatalf("migrating volume's shards did not spread: dn1=%d others=%d", onDn1, spread)
+	}
+	if onDn1+spread != len(allShards) {
+		t.Fatalf("shards lost or duplicated during dry-run spread: dn1=%d others=%d", onDn1, spread)
+	}
+}

weed/ec/ec_check_replication.go

@@ -0,0 +1,211 @@
+package ec
+
+import (
+	"fmt"
+	"io"
+	"slices"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+// CheckEcVolumeReplication reports EC volumes with under- or over-replicated
+// shards among the given data nodes. volumeIDs, when non-empty, narrows the
+// check to those volume ids.
+func CheckEcVolumeReplication(writer io.Writer, dataNodes []*master_pb.DataNodeInfo, volumeIDs map[uint32]bool, showDetails bool) error {
+	runner := &ecCheckReplicationRunner{
+		writer:      writer,
+		dataNodes:   dataNodes,
+		volumeIDMap: volumeIDs,
+	}
+	return runner.checkEcVolumes(showDetails)
+}
+
+type ecCheckReplicationRunner struct {
+	writer      io.Writer
+	dataNodes   []*master_pb.DataNodeInfo
+	volumeIDMap map[uint32]bool
+}
+
+func (r *ecCheckReplicationRunner) write(format string, a ...any) {
+	fmt.Fprintf(r.writer, format, a...)
+}
+
+func (r *ecCheckReplicationRunner) isVolumeIDValid(vid uint32) bool {
+	if len(r.volumeIDMap) == 0 {
+		return true
+	}
+	return r.volumeIDMap[vid]
+}
+
+// ecVolumeShardReplication aggregates the observed shards for a single EC volume,
+// together with the data+parity ratio the volume was encoded with. The ratio is
+// taken per volume (via erasure_coding.EcShardsVolume*Shards) so custom EC
+// ratios are checked against their own expected shard count.
+type ecVolumeShardReplication struct {
+	dataShards   int
+	parityShards int
+	// shardAddresses maps a shard id to the sorted node addresses hosting it.
+	shardAddresses map[erasure_coding.ShardId][]string
+}
+
+func (h *ecVolumeShardReplication) totalShards() int {
+	return h.dataShards + h.parityShards
+}
+
+// replicaCount is the total number of shard copies observed, counting each
+// over-replicated shard once per hosting node.
+func (h *ecVolumeShardReplication) replicaCount() int {
+	n := 0
+	for _, addrs := range h.shardAddresses {
+		n += len(addrs)
+	}
+	return n
+}
+
+// unexpectedShardIds returns, in ascending order, any observed shard ids that
+// fall outside the volume's expected data+parity range. These are anomalies
+// (e.g. a stray shard) and are treated as over-replication.
+func (h *ecVolumeShardReplication) unexpectedShardIds() []erasure_coding.ShardId {
+	var ids []erasure_coding.ShardId
+	for sid := range h.shardAddresses {
+		if int(sid) >= h.totalShards() {
+			ids = append(ids, sid)
+		}
+	}
+	slices.Sort(ids)
+	return ids
+}
+
+// TODO: check shard sizes?
+func (r *ecCheckReplicationRunner) checkEcVolumes(showDetails bool) error {
+	// collect EC shard placement, keyed by volume id
+	volumes := map[uint32]*ecVolumeShardReplication{}
+	for _, dni := range r.dataNodes {
+		nodeAddress := dni.GetAddress()
+		for _, di := range dni.GetDiskInfos() {
+			for _, eci := range di.GetEcShardInfos() {
+				vid := eci.GetId()
+				if !r.isVolumeIDValid(vid) {
+					continue
+				}
+
+				h, ok := volumes[vid]
+				if !ok {
+					// all shards of a volume share one ratio; take it from the
+					// first shard message seen for the volume.
+					h = &ecVolumeShardReplication{
+						dataShards:     erasure_coding.EcShardsVolumeDataShards(eci),
+						parityShards:   erasure_coding.EcShardsVolumeParityShards(eci),
+						shardAddresses: map[erasure_coding.ShardId][]string{},
+					}
+					volumes[vid] = h
+				}
+
+				sinfo := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci)
+				for _, sid := range sinfo.Ids() {
+					h.shardAddresses[sid] = append(h.shardAddresses[sid], nodeAddress)
+				}
+			}
+		}
+	}
+
+	if len(volumes) == 0 {
+		// Asking about specific volume IDs that turn out not to be EC volumes is
+		// an error; an unfiltered run over a cluster with no EC volumes is a
+		// legitimate, healthy state rather than a failure.
+		if len(r.volumeIDMap) > 0 {
+			return fmt.Errorf("no EC volumes found")
+		}
+		r.write("No EC volumes found.\n")
+		return nil
+	}
+
+	// keep the shard address lists nicely sorted, for the sake of readability.
+	for _, h := range volumes {
+		for _, addrs := range h.shardAddresses {
+			slices.Sort(addrs)
+		}
+	}
+
+	// classify each volume against its own expected shard count
+	underreplicatedVolumeIDs := []uint32{}
+	overreplicatedVolumeIDs := []uint32{}
+	for vid, h := range volumes {
+		under := false
+		over := false
+		for sid := 0; sid < h.totalShards(); sid++ {
+			switch len(h.shardAddresses[erasure_coding.ShardId(sid)]) {
+			case 0:
+				under = true
+			case 1:
+			default:
+				over = true
+			}
+		}
+		// shard ids beyond the expected data+parity range are unexpected extras,
+		// i.e. redundant data the ratio doesn't call for.
+		if len(h.unexpectedShardIds()) > 0 {
+			over = true
+		}
+
+		// under- and over-replication are independent problems (missing shards risk
+		// data loss, redundant shards waste space), so a volume exhibiting both is
+		// reported in both lists.
+		if under {
+			underreplicatedVolumeIDs = append(underreplicatedVolumeIDs, vid)
+		}
+		if over {
+			overreplicatedVolumeIDs = append(overreplicatedVolumeIDs, vid)
+		}
+	}
+	slices.Sort(underreplicatedVolumeIDs)
+	slices.Sort(overreplicatedVolumeIDs)
+
+	// ...and display results
+	if len(underreplicatedVolumeIDs) == 0 && len(overreplicatedVolumeIDs) == 0 {
+		r.write("EC volumes are healthy.\n")
+		return nil
+	}
+
+	if len(underreplicatedVolumeIDs) != 0 {
+		r.write("Found %d/%d under-replicated EC volumes: %v\n", len(underreplicatedVolumeIDs), len(volumes), underreplicatedVolumeIDs)
+	}
+	if len(overreplicatedVolumeIDs) != 0 {
+		r.write("Found %d/%d over-replicated EC volumes: %v\n", len(overreplicatedVolumeIDs), len(volumes), overreplicatedVolumeIDs)
+	}
+
+	if showDetails {
+		if len(underreplicatedVolumeIDs) != 0 {
+			r.write("\n")
+			r.writeShardMaps("under-replicated", underreplicatedVolumeIDs, volumes)
+		}
+		if len(overreplicatedVolumeIDs) != 0 {
+			r.write("\n")
+			r.writeShardMaps("over-replicated", overreplicatedVolumeIDs, volumes)
+		}
+	}
+
+	return nil
+}
+
+func (r *ecCheckReplicationRunner) writeShardMaps(kind string, volumeIDs []uint32, volumes map[uint32]*ecVolumeShardReplication) {
+	for _, vid := range volumeIDs {
+		h := volumes[vid]
+		r.write("Shards map for %s EC volume %v (%d/%d shards):\n", kind, vid, h.replicaCount(), h.totalShards())
+		for sid := 0; sid < h.totalShards(); sid++ {
+			shardTypeDesc := ""
+			if sid >= h.dataShards {
+				shardTypeDesc = " (parity)"
+			}
+			if addrs, ok := h.shardAddresses[erasure_coding.ShardId(sid)]; ok {
+				r.write("\t%02d%s => %v\n", sid, shardTypeDesc, addrs)
+			} else {
+				r.write("\t%02d%s is missing\n", sid, shardTypeDesc)
+			}
+		}
+		for _, sid := range h.unexpectedShardIds() {
+			r.write("\t%02d (unexpected) => %v\n", int(sid), h.shardAddresses[sid])
+		}
+	}
+}

weed/ec/ec_check_replication_test.go

@@ -0,0 +1,289 @@
+package ec
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+// TestEcCheckReplication exercises the ec.check.replication classification and reporting over a
+// fake topology. EC volume 1 is fully replicated, EC volume 2 is
+// under-replicated, and EC volume 3 is over-replicated.
+func TestEcCheckReplication(t *testing.T) {
+	nodes := []*master_pb.DataNodeInfo{
+		{
+			Address: "localhost:1111",
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"d1": {
+					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+						{
+							Id:          1,
+							EcIndexBits: 0b0000000011111,
+						},
+						{
+							Id:          2,
+							EcIndexBits: 0b0000000011111,
+						},
+						{
+							Id:          3,
+							EcIndexBits: 0b11111111111111,
+						},
+					},
+				},
+			},
+		},
+		{
+			Address: "localhost:2222",
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"d2": {
+					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+						{
+							Id:          1,
+							EcIndexBits: 0b11111111100000,
+						},
+						{
+							Id:          2,
+							EcIndexBits: 0b11111000000000,
+						},
+						{
+							Id:          3,
+							EcIndexBits: 0b00000000001111,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	testCases := []struct {
+		name        string
+		showDetails bool
+		volumeIDMap map[uint32]bool
+		want        string
+		wantErr     error
+	}{
+		{
+			name:        "default",
+			showDetails: false,
+			want: `Found 1/3 under-replicated EC volumes: [2]
+Found 1/3 over-replicated EC volumes: [3]
+`,
+		},
+		{
+			name:        "single healthy volume",
+			showDetails: false,
+			volumeIDMap: map[uint32]bool{1: true},
+			want: `EC volumes are healthy.
+`,
+		},
+		{
+			name:        "filtered by volume ID",
+			showDetails: false,
+			volumeIDMap: map[uint32]bool{1: true, 2: true, 5: true},
+			want: `Found 1/2 under-replicated EC volumes: [2]
+`,
+		},
+		{
+			name:        "no valid volume IDs",
+			showDetails: false,
+			volumeIDMap: map[uint32]bool{5: true, 6: true},
+			want:        "",
+			wantErr:     fmt.Errorf("no EC volumes found"),
+		},
+		{
+			name:        "with details",
+			showDetails: true,
+			want: `Found 1/3 under-replicated EC volumes: [2]
+Found 1/3 over-replicated EC volumes: [3]
+
+Shards map for under-replicated EC volume 2 (10/14 shards):
+	00 => [localhost:1111]
+	01 => [localhost:1111]
+	02 => [localhost:1111]
+	03 => [localhost:1111]
+	04 => [localhost:1111]
+	05 is missing
+	06 is missing
+	07 is missing
+	08 is missing
+	09 => [localhost:2222]
+	10 (parity) => [localhost:2222]
+	11 (parity) => [localhost:2222]
+	12 (parity) => [localhost:2222]
+	13 (parity) => [localhost:2222]
+
+Shards map for over-replicated EC volume 3 (18/14 shards):
+	00 => [localhost:1111 localhost:2222]
+	01 => [localhost:1111 localhost:2222]
+	02 => [localhost:1111 localhost:2222]
+	03 => [localhost:1111 localhost:2222]
+	04 => [localhost:1111]
+	05 => [localhost:1111]
+	06 => [localhost:1111]
+	07 => [localhost:1111]
+	08 => [localhost:1111]
+	09 => [localhost:1111]
+	10 (parity) => [localhost:1111]
+	11 (parity) => [localhost:1111]
+	12 (parity) => [localhost:1111]
+	13 (parity) => [localhost:1111]
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		var buf bytes.Buffer
+
+		runner := &ecCheckReplicationRunner{
+			writer:      &buf,
+			dataNodes:   nodes,
+			volumeIDMap: tc.volumeIDMap,
+		}
+		gotErr := runner.checkEcVolumes(tc.showDetails)
+		got := buf.String()
+
+		if got != tc.want {
+			t.Errorf("%s: got %v, want %v", tc.name, got, tc.want)
+		}
+		if !reflect.DeepEqual(gotErr, tc.wantErr) {
+			t.Errorf("%s: got error %v, want %v", tc.name, gotErr, tc.wantErr)
+		}
+	}
+}
+
+// TestEcCheckReplicationNoVolumes checks the empty-topology behavior: an unfiltered
+// run over a cluster with no EC volumes is healthy (informational message, no
+// error), while a run filtered to volume IDs that are not EC volumes errors.
+func TestEcCheckReplicationNoVolumes(t *testing.T) {
+	// a data node reporting a normal volume but no EC shards
+	nodes := []*master_pb.DataNodeInfo{
+		{
+			Address:   "localhost:1111",
+			DiskInfos: map[string]*master_pb.DiskInfo{"d1": {}},
+		},
+	}
+
+	t.Run("unfiltered", func(t *testing.T) {
+		var buf bytes.Buffer
+		runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes}
+		if err := runner.checkEcVolumes(false); err != nil {
+			t.Errorf("got error %v, want nil", err)
+		}
+		if got, want := buf.String(), "No EC volumes found.\n"; got != want {
+			t.Errorf("got %q, want %q", got, want)
+		}
+	})
+
+	t.Run("filtered no match", func(t *testing.T) {
+		var buf bytes.Buffer
+		runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes, volumeIDMap: map[uint32]bool{5: true}}
+		if err := runner.checkEcVolumes(false); err == nil {
+			t.Errorf("got nil error, want %q", "no EC volumes found")
+		}
+		if got := buf.String(); got != "" {
+			t.Errorf("got output %q, want none", got)
+		}
+	})
+}
+
+// TestEcCheckReplicationUnexpectedShard verifies that a shard id beyond a volume's
+// expected data+parity range is flagged as over-replication and surfaced in the
+// details map, rather than being silently ignored. Volume 20 is a healthy 10+4
+// (shards 0..13 on nodeA) with a stray shard 14 on nodeB.
+func TestEcCheckReplicationUnexpectedShard(t *testing.T) {
+	nodes := []*master_pb.DataNodeInfo{
+		{
+			Address: "nodeA",
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"d1": {
+					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+						{Id: 20, EcIndexBits: 0b11111111111111}, // shards 0..13
+					},
+				},
+			},
+		},
+		{
+			Address: "nodeB",
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"d2": {
+					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+						{Id: 20, EcIndexBits: 0b100000000000000}, // stray shard 14
+					},
+				},
+			},
+		},
+	}
+
+	want := `Found 1/1 over-replicated EC volumes: [20]
+
+Shards map for over-replicated EC volume 20 (15/14 shards):
+	00 => [nodeA]
+	01 => [nodeA]
+	02 => [nodeA]
+	03 => [nodeA]
+	04 => [nodeA]
+	05 => [nodeA]
+	06 => [nodeA]
+	07 => [nodeA]
+	08 => [nodeA]
+	09 => [nodeA]
+	10 (parity) => [nodeA]
+	11 (parity) => [nodeA]
+	12 (parity) => [nodeA]
+	13 (parity) => [nodeA]
+	14 (unexpected) => [nodeB]
+`
+
+	var buf bytes.Buffer
+	runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes}
+	if err := runner.checkEcVolumes(true); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if got := buf.String(); got != want {
+		t.Errorf("unexpected shard:\n got: %q\nwant: %q", got, want)
+	}
+}
+
+// TestEcCheckReplicationMixedAnomalies verifies that a volume with both a missing
+// shard and a duplicated shard is reported in both lists. Volume 30 has shard 5
+// missing everywhere (under-replicated) and shard 0 on two nodes (over-replicated).
+func TestEcCheckReplicationMixedAnomalies(t *testing.T) {
+	nodes := []*master_pb.DataNodeInfo{
+		{
+			Address: "nodeA",
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"d1": {
+					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+						{Id: 30, EcIndexBits: 0b11111111011111}, // shards 0..13 except 5
+					},
+				},
+			},
+		},
+		{
+			Address: "nodeB",
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"d2": {
+					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+						{Id: 30, EcIndexBits: 0b00000000000001}, // duplicate of shard 0
+					},
+				},
+			},
+		},
+	}
+
+	want := `Found 1/1 under-replicated EC volumes: [30]
+Found 1/1 over-replicated EC volumes: [30]
+`
+
+	var buf bytes.Buffer
+	runner := &ecCheckReplicationRunner{writer: &buf, dataNodes: nodes}
+	if err := runner.checkEcVolumes(false); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if got := buf.String(); got != want {
+		t.Errorf("mixed anomalies:\n got: %q\nwant: %q", got, want)
+	}
+}

weed/ec/ec_common_test.go

@@ -0,0 +1,519 @@
+package ec
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// TestPickBestDiskOnNodeSelectsPhysicalDiskZero verifies physical disk 0 can be
+// chosen as the preferred shard target. Gating on bestDiskId != 0 discarded a
+// best-scoring disk 0 (0 is also the zero value) and returned the fallback.
+func TestPickBestDiskOnNodeSelectsPhysicalDiskZero(t *testing.T) {
+	ecNode := &EcNode{
+		Disks: map[uint32]*EcDisk{
+			0: {DiskId: 0, DiskType: string(types.SsdType), FreeEcSlots: 10, EcShards: map[needle.VolumeId]*erasure_coding.ShardsInfo{}},
+			5: {DiskId: 5, DiskType: string(types.HardDriveType), FreeEcSlots: 10, EcShards: map[needle.VolumeId]*erasure_coding.ShardsInfo{}},
+		},
+	}
+
+	// Disk 0 matches the requested type; it must win over the non-matching
+	// fallback on disk 5 instead of being treated as "no match".
+	if got := PickBestDiskOnNode(ecNode, needle.VolumeId(42), types.SsdType, false, 0, 0); got != 0 {
+		t.Fatalf("want physical disk 0 (matching type), got %d", got)
+	}
+}
+
+func TestCommandEcBalanceSmall(t *testing.T) {
+	ecb := &ecBalancer{
+		ecNodes: []*EcNode{
+			newEcNode("dc1", "rack1", "dn1", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}),
+			newEcNode("dc1", "rack2", "dn2", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}),
+		},
+		applyBalancing: false,
+		diskType:       types.HardDriveType,
+	}
+
+	ecb.balance([]string{"c1"})
+}
+
+func TestCommandEcBalanceNothingToMove(t *testing.T) {
+	ecb := &ecBalancer{
+		ecNodes: []*EcNode{
+			newEcNode("dc1", "rack1", "dn1", 100).
+				addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6}).
+				addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{7, 8, 9, 10, 11, 12, 13}),
+			newEcNode("dc1", "rack1", "dn2", 100).
+				addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{7, 8, 9, 10, 11, 12, 13}).
+				addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6}),
+		},
+		applyBalancing: false,
+		diskType:       types.HardDriveType,
+	}
+
+	ecb.balance([]string{"c1"})
+}
+
+func TestCommandEcBalanceAddNewServers(t *testing.T) {
+	ecb := &ecBalancer{
+		ecNodes: []*EcNode{
+			newEcNode("dc1", "rack1", "dn1", 100).
+				addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6}).
+				addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{7, 8, 9, 10, 11, 12, 13}),
+			newEcNode("dc1", "rack1", "dn2", 100).
+				addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{7, 8, 9, 10, 11, 12, 13}).
+				addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6}),
+			newEcNode("dc1", "rack1", "dn3", 100),
+			newEcNode("dc1", "rack1", "dn4", 100),
+		},
+		applyBalancing: false,
+		diskType:       types.HardDriveType,
+	}
+
+	ecb.balance([]string{"c1"})
+}
+
+func TestCommandEcBalanceAddNewRacks(t *testing.T) {
+	ecb := &ecBalancer{
+		ecNodes: []*EcNode{
+			newEcNode("dc1", "rack1", "dn1", 100).
+				addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6}).
+				addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{7, 8, 9, 10, 11, 12, 13}),
+			newEcNode("dc1", "rack1", "dn2", 100).
+				addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{7, 8, 9, 10, 11, 12, 13}).
+				addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6}),
+			newEcNode("dc1", "rack2", "dn3", 100),
+			newEcNode("dc1", "rack2", "dn4", 100),
+		},
+		applyBalancing: false,
+		diskType:       types.HardDriveType,
+	}
+
+	ecb.balance([]string{"c1"})
+}
+
+func TestCommandEcBalanceVolumeEvenButRackUneven(t *testing.T) {
+	ecb := ecBalancer{
+		ecNodes: []*EcNode{
+			newEcNode("dc1", "rack1", "dn_shared", 100).
+				addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0}).
+				addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{0}),
+
+			newEcNode("dc1", "rack1", "dn_a1", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{1}),
+			newEcNode("dc1", "rack1", "dn_a2", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{2}),
+			newEcNode("dc1", "rack1", "dn_a3", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{3}),
+			newEcNode("dc1", "rack1", "dn_a4", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{4}),
+			newEcNode("dc1", "rack1", "dn_a5", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{5}),
+			newEcNode("dc1", "rack1", "dn_a6", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{6}),
+			newEcNode("dc1", "rack1", "dn_a7", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{7}),
+			newEcNode("dc1", "rack1", "dn_a8", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{8}),
+			newEcNode("dc1", "rack1", "dn_a9", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{9}),
+			newEcNode("dc1", "rack1", "dn_a10", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{10}),
+			newEcNode("dc1", "rack1", "dn_a11", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{11}),
+			newEcNode("dc1", "rack1", "dn_a12", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{12}),
+			newEcNode("dc1", "rack1", "dn_a13", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{13}),
+
+			newEcNode("dc1", "rack1", "dn_b1", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{1}),
+			newEcNode("dc1", "rack1", "dn_b2", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{2}),
+			newEcNode("dc1", "rack1", "dn_b3", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{3}),
+			newEcNode("dc1", "rack1", "dn_b4", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{4}),
+			newEcNode("dc1", "rack1", "dn_b5", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{5}),
+			newEcNode("dc1", "rack1", "dn_b6", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{6}),
+			newEcNode("dc1", "rack1", "dn_b7", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{7}),
+			newEcNode("dc1", "rack1", "dn_b8", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{8}),
+			newEcNode("dc1", "rack1", "dn_b9", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{9}),
+			newEcNode("dc1", "rack1", "dn_b10", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{10}),
+			newEcNode("dc1", "rack1", "dn_b11", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{11}),
+			newEcNode("dc1", "rack1", "dn_b12", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{12}),
+			newEcNode("dc1", "rack1", "dn_b13", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{13}),
+
+			newEcNode("dc1", "rack1", "dn3", 100),
+		},
+		applyBalancing: false,
+		diskType:       types.HardDriveType,
+	}
+
+	ecb.balance([]string{"c1"})
+}
+
+func newEcNode(dc string, rack string, dataNodeId string, freeEcSlot int) *EcNode {
+	return &EcNode{
+		Info: &master_pb.DataNodeInfo{
+			Id:        dataNodeId,
+			DiskInfos: make(map[string]*master_pb.DiskInfo),
+		},
+		DC:         DataCenterId(dc),
+		Rack:       RackId(rack),
+		FreeEcSlot: freeEcSlot,
+	}
+}
+
+func (ecNode *EcNode) addEcVolumeAndShardsForTest(vid uint32, collection string, shardIds []erasure_coding.ShardId) *EcNode {
+	return ecNode.AddEcVolumeShards(needle.VolumeId(vid), collection, shardIds, types.HardDriveType)
+}
+
+// TestCommandEcBalanceEvenDataAndParityDistribution verifies that after balancing:
+// 1. Data shards (0-9) are evenly distributed across racks (max 2 per rack for 6 racks)
+// 2. Parity shards (10-13) are evenly distributed across racks (max 1 per rack for 6 racks)
+func TestCommandEcBalanceEvenDataAndParityDistribution(t *testing.T) {
+	// Setup: All 14 shards start on rack1 (simulating fresh EC encode)
+	ecb := &ecBalancer{
+		ecNodes: []*EcNode{
+			// All shards initially on rack1/dn1
+			newEcNode("dc1", "rack1", "dn1", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}),
+			// Empty nodes on other racks
+			newEcNode("dc1", "rack2", "dn2", 100),
+			newEcNode("dc1", "rack3", "dn3", 100),
+			newEcNode("dc1", "rack4", "dn4", 100),
+			newEcNode("dc1", "rack5", "dn5", 100),
+			newEcNode("dc1", "rack6", "dn6", 100),
+		},
+		applyBalancing: false, // Dry-run mode (simulates moves by updating internal state)
+		diskType:       types.HardDriveType,
+	}
+
+	ecb.balance([]string{"c1"})
+
+	// After balancing (dry-run), verify the PLANNED distribution by checking what moves were proposed
+	// The ecb.ecNodes state is updated during dry-run to track planned moves
+	vid := needle.VolumeId(1)
+	dataShardCount := erasure_coding.DataShardsCount     // 10
+	parityShardCount := erasure_coding.ParityShardsCount // 4
+
+	// Count data and parity shards per rack based on current (updated) state
+	dataPerRack, parityPerRack := countDataAndParityShardsPerRack(ecb.ecNodes, vid, dataShardCount)
+
+	// With 6 racks:
+	// - Data shards (10): max 2 per rack (ceil(10/6) = 2)
+	// - Parity shards (4): max 1 per rack (ceil(4/6) = 1)
+	maxDataPerRack := CeilDivide(dataShardCount, 6)     // 2
+	maxParityPerRack := CeilDivide(parityShardCount, 6) // 1
+
+	// Verify no rack has more than max data shards
+	for rackId, count := range dataPerRack {
+		if count > maxDataPerRack {
+			t.Errorf("rack %s has %d data shards, expected max %d", rackId, count, maxDataPerRack)
+		}
+	}
+
+	// Verify no rack has more than max parity shards
+	for rackId, count := range parityPerRack {
+		if count > maxParityPerRack {
+			t.Errorf("rack %s has %d parity shards, expected max %d", rackId, count, maxParityPerRack)
+		}
+	}
+
+	// Verify all shards are distributed (total counts)
+	totalData := 0
+	totalParity := 0
+	for _, count := range dataPerRack {
+		totalData += count
+	}
+	for _, count := range parityPerRack {
+		totalParity += count
+	}
+	if totalData != dataShardCount {
+		t.Errorf("total data shards = %d, expected %d", totalData, dataShardCount)
+	}
+	if totalParity != parityShardCount {
+		t.Errorf("total parity shards = %d, expected %d", totalParity, parityShardCount)
+	}
+
+	// Verify data shards are spread across at least 5 racks (10 shards / 2 max per rack)
+	racksWithData := len(dataPerRack)
+	minRacksForData := dataShardCount / maxDataPerRack // At least 5 racks needed for 10 data shards
+	if racksWithData < minRacksForData {
+		t.Errorf("data shards spread across only %d racks, expected at least %d", racksWithData, minRacksForData)
+	}
+
+	// Verify parity shards are spread across at least 4 racks (4 shards / 1 max per rack)
+	racksWithParity := len(parityPerRack)
+	if racksWithParity < parityShardCount {
+		t.Errorf("parity shards spread across only %d racks, expected at least %d", racksWithParity, parityShardCount)
+	}
+
+	t.Logf("Distribution after balancing:")
+	t.Logf("  Data shards per rack: %v (max allowed: %d)", dataPerRack, maxDataPerRack)
+	t.Logf("  Parity shards per rack: %v (max allowed: %d)", parityPerRack, maxParityPerRack)
+}
+
+// countDataAndParityShardsPerRack counts data and parity shards per rack
+func countDataAndParityShardsPerRack(ecNodes []*EcNode, vid needle.VolumeId, dataShardCount int) (dataPerRack, parityPerRack map[string]int) {
+	dataPerRack = make(map[string]int)
+	parityPerRack = make(map[string]int)
+
+	for _, ecNode := range ecNodes {
+		si := FindEcVolumeShardsInfo(ecNode, vid, types.HardDriveType)
+		for _, shardId := range si.Ids() {
+			rackId := string(ecNode.Rack)
+			if int(shardId) < dataShardCount {
+				dataPerRack[rackId]++
+			} else {
+				parityPerRack[rackId]++
+			}
+		}
+	}
+	return
+}
+
+// TestCommandEcBalanceMultipleVolumesEvenDistribution tests that multiple volumes
+// each get their data and parity shards evenly distributed
+func TestCommandEcBalanceMultipleVolumesEvenDistribution(t *testing.T) {
+	// Setup: Two volumes, each with all 14 shards on different starting racks
+	ecb := &ecBalancer{
+		ecNodes: []*EcNode{
+			// Volume 1: all shards on rack1
+			newEcNode("dc1", "rack1", "dn1", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}),
+			// Volume 2: all shards on rack2
+			newEcNode("dc1", "rack2", "dn2", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}),
+			// Empty nodes on other racks
+			newEcNode("dc1", "rack3", "dn3", 100),
+			newEcNode("dc1", "rack4", "dn4", 100),
+			newEcNode("dc1", "rack5", "dn5", 100),
+			newEcNode("dc1", "rack6", "dn6", 100),
+		},
+		applyBalancing: false, // Dry-run mode
+		diskType:       types.HardDriveType,
+	}
+
+	ecb.balance([]string{"c1"})
+
+	// Check both volumes
+	for _, vid := range []needle.VolumeId{1, 2} {
+		dataPerRack, parityPerRack := countDataAndParityShardsPerRack(ecb.ecNodes, vid, erasure_coding.DataShardsCount)
+
+		maxDataPerRack := CeilDivide(erasure_coding.DataShardsCount, 6)
+		maxParityPerRack := CeilDivide(erasure_coding.ParityShardsCount, 6)
+
+		for rackId, count := range dataPerRack {
+			if count > maxDataPerRack {
+				t.Errorf("volume %d: rack %s has %d data shards, expected max %d", vid, rackId, count, maxDataPerRack)
+			}
+		}
+		for rackId, count := range parityPerRack {
+			if count > maxParityPerRack {
+				t.Errorf("volume %d: rack %s has %d parity shards, expected max %d", vid, rackId, count, maxParityPerRack)
+			}
+		}
+
+		t.Logf("Volume %d - Data: %v, Parity: %v", vid, dataPerRack, parityPerRack)
+	}
+}
+
+// TestCommandEcBalanceAllNodesShareAllVolumes reproduces the scenario from issue #8793:
+// When every node has a shard of every volume, ec.balance was unable to move any shards
+// because it skipped volumes that already existed on the target node at the volume level.
+func TestCommandEcBalanceAllNodesShareAllVolumes(t *testing.T) {
+	// 4 nodes, all in same rack, 2 volumes with 14 shards each.
+	// Distribute shards so every node has shards of both volumes, but unevenly:
+	// dn1: vol1 shards 0-4, vol2 shards 0-4 => 10 shards
+	// dn2: vol1 shards 5-9, vol2 shards 5-9 => 10 shards
+	// dn3: vol1 shards 10-12, vol2 shards 10-12 => 6 shards
+	// dn4: vol1 shard 13, vol2 shard 13 => 2 shards
+	// Total: 28 shards, average = 7 per node
+	ecb := &ecBalancer{
+		ecNodes: []*EcNode{
+			newEcNode("dc1", "rack1", "dn1", 100).
+				addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4}).
+				addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4}),
+			newEcNode("dc1", "rack1", "dn2", 100).
+				addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{5, 6, 7, 8, 9}).
+				addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{5, 6, 7, 8, 9}),
+			newEcNode("dc1", "rack1", "dn3", 100).
+				addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{10, 11, 12}).
+				addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{10, 11, 12}),
+			newEcNode("dc1", "rack1", "dn4", 100).
+				addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{13}).
+				addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{13}),
+		},
+		applyBalancing: false,
+		diskType:       types.HardDriveType,
+	}
+
+	ecb.balance([]string{"c1"})
+
+	// Count total shards per node after balancing
+	for _, node := range ecb.ecNodes {
+		count := 0
+		if diskInfo, found := node.Info.DiskInfos[string(types.HardDriveType)]; found {
+			for _, ecsi := range diskInfo.EcShardInfos {
+				count += erasure_coding.GetShardCount(ecsi)
+			}
+		}
+		// Average is 7, so all nodes should be at 7 (ceil(28/4) = 7)
+		if count > 7 {
+			t.Errorf("node %s has %d shards after balancing, expected at most 7", node.Info.Id, count)
+		}
+		t.Logf("node %s: %d shards", node.Info.Id, count)
+	}
+}
+
+// TestCommandEcBalanceIssue8793Topology simulates the real cluster from issue #8793:
+// 14 nodes (9 with max=80, 5 with max=33), all in one rack, with mixed capacities.
+// Each EC volume has 1 shard per node. Nodes have uneven totals (some have extra volumes).
+func TestCommandEcBalanceIssue8793Topology(t *testing.T) {
+	// Simulate 22 EC volumes across 14 nodes (each volume has 14 shards, 1 per node).
+	// Give nodes 0-3 an extra volume each (vol 23-26, all 14 shards) to create imbalance.
+	// Before balancing: nodes 0-3 have 22+14=36 shards each, nodes 4-13 have 22 shards each.
+	// Total = 4*36 + 10*22 = 144+220 = 364. Capacities are heterogeneous (max 80 vs
+	// 33), so the utilization-based global phase balances by fullness, not count:
+	// 364 shards over 9*80+5*33=885 slots is ~41% full, so large nodes settle near
+	// 33 shards and small nodes near 14 (all ~41% full) rather than ~26 each, which
+	// would drive the small nodes to ~79%.
+
+	type nodeSpec struct {
+		id      string
+		maxSlot int
+	}
+	nodes := []nodeSpec{
+		{"192.168.0.12:8332", 80}, {"192.168.0.12:8333", 80}, {"192.168.0.12:8334", 80},
+		{"192.168.0.12:8335", 80}, {"192.168.0.12:8336", 80}, {"192.168.0.12:8337", 80},
+		{"192.168.0.12:8338", 80}, {"192.168.0.12:8339", 80}, {"192.168.0.12:8340", 80},
+		{"192.168.0.12:8341", 33}, {"192.168.0.12:8342", 33}, {"192.168.0.12:8343", 33},
+		{"192.168.0.25:8350", 33}, {"192.168.0.25:8351", 33},
+	}
+
+	ecNodes := make([]*EcNode, len(nodes))
+	for i, ns := range nodes {
+		ecNodes[i] = newEcNode("home", "center", ns.id, ns.maxSlot)
+	}
+
+	// 22 shared volumes: each node gets exactly 1 shard (shard i for node i)
+	for vid := uint32(1); vid <= 22; vid++ {
+		for i := range ecNodes {
+			ecNodes[i].addEcVolumeAndShardsForTest(vid, "cldata", []erasure_coding.ShardId{erasure_coding.ShardId(i)})
+		}
+	}
+
+	// 4 extra volumes only on first 4 nodes (all 14 shards each) to create imbalance
+	for extra := uint32(0); extra < 4; extra++ {
+		vid := 23 + extra
+		nodeIdx := int(extra)
+		allShards := make([]erasure_coding.ShardId, 14)
+		for s := 0; s < 14; s++ {
+			allShards[s] = erasure_coding.ShardId(s)
+		}
+		ecNodes[nodeIdx].addEcVolumeAndShardsForTest(vid, "cldata", allShards)
+	}
+
+	ecb := &ecBalancer{
+		ecNodes:        ecNodes,
+		applyBalancing: false,
+		diskType:       types.HardDriveType,
+	}
+
+	// Log initial state
+	for _, node := range ecb.ecNodes {
+		count := 0
+		if diskInfo, found := node.Info.DiskInfos[string(types.HardDriveType)]; found {
+			for _, ecsi := range diskInfo.EcShardInfos {
+				count += erasure_coding.GetShardCount(ecsi)
+			}
+		}
+		t.Logf("BEFORE node %s (max %d): %d shards", node.Info.Id, node.FreeEcSlot+count, count)
+	}
+
+	ecb.balance([]string{"cldata"})
+
+	// Verify even FULLNESS (shards/capacity), not even count: with heterogeneous
+	// capacities the utilization-based global phase fills nodes proportionally, so
+	// large nodes hold more shards than small ones while every node ends near the
+	// overall fullness. (An even-count result would over-fill the small nodes.)
+	capacityByID := make(map[string]int, len(nodes))
+	for _, ns := range nodes {
+		capacityByID[ns.id] = ns.maxSlot
+	}
+
+	totalShards := 0
+	totalCapacity := 0
+	shardCounts := make(map[string]int)
+	for _, node := range ecb.ecNodes {
+		count := 0
+		if diskInfo, found := node.Info.DiskInfos[string(types.HardDriveType)]; found {
+			for _, ecsi := range diskInfo.EcShardInfos {
+				count += erasure_coding.GetShardCount(ecsi)
+			}
+		}
+		shardCounts[node.Info.Id] = count
+		totalShards += count
+		totalCapacity += capacityByID[node.Info.Id]
+	}
+	overallFullness := float64(totalShards) / float64(totalCapacity)
+
+	// Tolerance well below the gap a count-even result would show (small nodes
+	// would sit ~38 points above overall), but above integer-rounding skew.
+	const tolerance = 0.05
+	for _, node := range ecb.ecNodes {
+		count := shardCounts[node.Info.Id]
+		capacity := capacityByID[node.Info.Id]
+		fullness := float64(count) / float64(capacity)
+		t.Logf("AFTER  node %s: %d/%d shards (%.0f%% full, overall %.0f%%)",
+			node.Info.Id, count, capacity, fullness*100, overallFullness*100)
+		if diff := fullness - overallFullness; diff > tolerance || diff < -tolerance {
+			t.Errorf("node %s fullness %.1f%% deviates from overall %.1f%% by more than %.0f points",
+				node.Info.Id, fullness*100, overallFullness*100, tolerance*100)
+		}
+	}
+}
+
+// TestCommandEcBalanceVolumeIdsFilter checks that -volumeIds keeps every phase,
+// dedup included, off the volumes that were not asked for.
+func TestCommandEcBalanceVolumeIdsFilter(t *testing.T) {
+	ecb := &ecBalancer{
+		ecNodes: []*EcNode{
+			// Volume 1: all shards on one node, so balancing has plenty to move.
+			newEcNode("dc1", "rack1", "dn1", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}),
+			// Volume 2 is equally lopsided, and shard 0 is duplicated on dn3.
+			newEcNode("dc1", "rack2", "dn2", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}),
+			newEcNode("dc1", "rack3", "dn3", 100).addEcVolumeAndShardsForTest(2, "c1", []erasure_coding.ShardId{0}),
+			newEcNode("dc1", "rack4", "dn4", 100),
+			newEcNode("dc1", "rack5", "dn5", 100),
+			newEcNode("dc1", "rack6", "dn6", 100),
+		},
+		applyBalancing: false,
+		diskType:       types.HardDriveType,
+		volumeIds:      map[uint32]bool{1: true},
+	}
+
+	if err := ecb.balance([]string{"c1"}); err != nil {
+		t.Fatalf("balance: %v", err)
+	}
+
+	// Volume 1 spreads out.
+	if count := ecb.ecNodes[0].LocalShardIdCount(1); count == 14 {
+		t.Errorf("volume 1 was not balanced: dn1 still holds all %d shards", count)
+	}
+
+	// Volume 2 keeps every shard where it was, duplicate included.
+	if count := ecb.ecNodes[1].LocalShardIdCount(2); count != 14 {
+		t.Errorf("dn2 holds %d shards of the unselected volume 2, want 14", count)
+	}
+	if count := ecb.ecNodes[2].LocalShardIdCount(2); count != 1 {
+		t.Errorf("dn3 holds %d shards of the unselected volume 2, want the duplicate to survive", count)
+	}
+}
+
+func TestCommandEcBalanceVolumeIdsNotFound(t *testing.T) {
+	ecb := &ecBalancer{
+		ecNodes: []*EcNode{
+			newEcNode("dc1", "rack1", "dn1", 100).addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}),
+			newEcNode("dc1", "rack2", "dn2", 100),
+		},
+		applyBalancing: false,
+		diskType:       types.HardDriveType,
+		volumeIds:      map[uint32]bool{1: true, 99: true},
+	}
+
+	err := ecb.balance([]string{"c1"})
+	if err == nil || !strings.Contains(err.Error(), "[99]") {
+		t.Fatalf("want an error naming volume 99, got %v", err)
+	}
+	// The valid id must not be balanced either: the plan is all-or-nothing.
+	if count := ecb.ecNodes[0].LocalShardIdCount(1); count != 14 {
+		t.Errorf("dn1 holds %d shards, want the rejected plan to leave volume 1 untouched", count)
+	}
+}

weed/ec/ec_decode.go

@@ -0,0 +1,463 @@
+package ec
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func DoEcDecode(env *Env, topoInfo *master_pb.TopologyInfo, collection string, vid needle.VolumeId, diskType types.DiskType, checkMinFreeSpace bool, diskUsageState *DecodeDiskUsageState) (err error) {
+
+	if !env.isLocked() {
+		return fmt.Errorf("lock is lost")
+	}
+
+	// find volume location
+	nodeToEcShardsInfo, dataShards := collectEcNodeShardsInfo(topoInfo, vid)
+
+	fmt.Printf("ec volume %d shard locations: %+v\n", vid, nodeToEcShardsInfo)
+
+	if len(nodeToEcShardsInfo) == 0 {
+		return fmt.Errorf("no EC shards found for volume %d", vid)
+	}
+
+	// A decode interrupted while deleting the shards leaves the regenerated
+	// volume in place with the set half gone, so a re-run finds both and fails
+	// re-collecting the first shard the interrupted run removed.
+	//
+	// Finding a volume beside the shards is not enough to act on: an encode
+	// interrupted before it deleted the original leaves the same shape, as does
+	// a decode killed while generating, whose volume may be half written. Both
+	// of those leave the shard set COMPLETE. Only the deletion phase can remove
+	// a data shard, so require one to be gone -- that is also exactly the state
+	// no decode can recover from, which makes finishing the cleanup the only
+	// move left rather than a choice between two.
+	if _, found := missingDataShard(nodeToEcShardsInfo, dataShards); found {
+		holder, hasVolume := regularVolumeHolder(topoInfo, vid)
+		if !hasVolume {
+			return fmt.Errorf("volume %d cannot be decoded: data shards are missing and no decoded volume exists to finish", vid)
+		}
+		if err := verifyDecodedVolumeBeforeDelete(env.GrpcDialOption, holder, vid); err != nil {
+			return fmt.Errorf("volume %d is already decoded on %s but did not verify, keeping its ec shards: %v", vid, holder, err)
+		}
+		fmt.Printf("volume %d is already decoded on %s; deleting the ec shards the interrupted run left behind\n", vid, holder)
+		return unmountAndDeleteEcShardsWithPrefix("deleteDecodedEcShards", env.GrpcDialOption, collection, nodeToEcShardsInfo, vid)
+	}
+
+	var originalShardCounts map[pb.ServerAddress]int
+	if diskUsageState != nil {
+		originalShardCounts = make(map[pb.ServerAddress]int, len(nodeToEcShardsInfo))
+		for location, si := range nodeToEcShardsInfo {
+			originalShardCounts[location] = si.Count()
+		}
+	}
+
+	var eligibleTargets map[pb.ServerAddress]struct{}
+	if checkMinFreeSpace {
+		if diskUsageState == nil {
+			return fmt.Errorf("min free space checking requires disk usage state")
+		}
+		eligibleTargets = make(map[pb.ServerAddress]struct{})
+		for location := range nodeToEcShardsInfo {
+			if freeCount, found := diskUsageState.freeVolumeCount(location); found && freeCount > 0 {
+				eligibleTargets[location] = struct{}{}
+			}
+		}
+		if len(eligibleTargets) == 0 {
+			return fmt.Errorf("no eligible target datanodes with free volume slots for volume %d (diskType %s); use -checkMinFreeSpace=false to override", vid, diskType.ReadableString())
+		}
+	}
+
+	// collect ec shards to the server with most space
+	targetNodeLocation, err := collectEcShards(env, nodeToEcShardsInfo, collection, vid, eligibleTargets, dataShards)
+	if err != nil {
+		return fmt.Errorf("collectEcShards for volume %d: %v", vid, err)
+	}
+
+	// generate a normal volume
+	err = generateNormalVolume(env.GrpcDialOption, vid, collection, targetNodeLocation)
+	if err != nil {
+		// Special case: if the EC index has no live entries, decoding is a no-op.
+		// Just purge EC shards and return success without generating/mounting an empty volume.
+		if isEcDecodeEmptyVolumeErr(err) {
+			if err := unmountAndDeleteEcShards(env.GrpcDialOption, collection, nodeToEcShardsInfo, vid); err != nil {
+				return err
+			}
+			if diskUsageState != nil {
+				diskUsageState.applyDecode(targetNodeLocation, originalShardCounts, false)
+			}
+			return nil
+		}
+		return fmt.Errorf("generate normal volume %d on %s: %v", vid, targetNodeLocation, err)
+	}
+
+	// mount the decoded volume after server-side offline compaction succeeded
+	err = mountDecodedVolume(env.GrpcDialOption, targetNodeLocation, vid)
+	if err != nil {
+		return fmt.Errorf("mount decoded volume %d on %s: %v", vid, targetNodeLocation, err)
+	}
+
+	// Confirm the regenerated .dat is present and non-empty before destroying
+	// the shards. Without this gate, a silent failure in generate/mount could
+	// leave the cluster with neither shards nor volume.
+	if err := verifyDecodedVolumeBeforeDelete(env.GrpcDialOption, targetNodeLocation, vid); err != nil {
+		return fmt.Errorf("verify decoded volume %d on %s before deleting shards: %w", vid, targetNodeLocation, err)
+	}
+
+	// delete the previous ec shards
+	err = unmountAndDeleteEcShardsWithPrefix("deleteDecodedEcShards", env.GrpcDialOption, collection, nodeToEcShardsInfo, vid)
+	if err != nil {
+		return fmt.Errorf("delete ec shards for volume %d: %v", vid, err)
+	}
+	if diskUsageState != nil {
+		diskUsageState.applyDecode(targetNodeLocation, originalShardCounts, true)
+	}
+
+	return nil
+}
+
+func isEcDecodeEmptyVolumeErr(err error) bool {
+	st, ok := status.FromError(err)
+	if !ok {
+		return false
+	}
+	if st.Code() != codes.FailedPrecondition {
+		return false
+	}
+	// Keep this robust against wording tweaks while still being specific.
+	return strings.Contains(st.Message(), erasure_coding.EcNoLiveEntriesSubstring)
+}
+
+func unmountAndDeleteEcShards(grpcDialOption grpc.DialOption, collection string, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, vid needle.VolumeId) error {
+	return unmountAndDeleteEcShardsWithPrefix("unmountAndDeleteEcShards", grpcDialOption, collection, nodeToShardsInfo, vid)
+}
+
+func unmountAndDeleteEcShardsWithPrefix(prefix string, grpcDialOption grpc.DialOption, collection string, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, vid needle.VolumeId) error {
+	ewg := util.NewErrorWaitGroup(len(nodeToShardsInfo))
+
+	// unmount and delete ec shards in parallel (one goroutine per location)
+	for location, si := range nodeToShardsInfo {
+		location, si := location, si // capture loop variables for goroutine
+		ewg.Add(func() error {
+			fmt.Printf("unmount ec volume %d on %s has shards: %+v\n", vid, location, si.Ids())
+			if err := UnmountEcShards(grpcDialOption, vid, location, si.Ids()); err != nil {
+				return fmt.Errorf("%s unmount ec volume %d on %s: %w", prefix, vid, location, err)
+			}
+
+			fmt.Printf("delete ec volume %d on %s has shards: %+v\n", vid, location, si.Ids())
+			if err := SourceServerDeleteEcShards(grpcDialOption, collection, vid, location, si.Ids()); err != nil {
+				return fmt.Errorf("%s delete ec volume %d on %s: %w", prefix, vid, location, err)
+			}
+			return nil
+		})
+	}
+	return ewg.Wait()
+}
+
+// missingDataShard reports the first data shard absent from every holder.
+// Parity shards are not enough to answer this: the decode rebuilds the volume
+// from the data shards, so one of those going missing is what makes a re-run
+// impossible.
+func missingDataShard(nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, dataShards int) (erasure_coding.ShardId, bool) {
+	var present erasure_coding.ShardBits
+	for _, si := range nodeToShardsInfo {
+		for _, id := range si.Ids() {
+			present = present.Set(id)
+		}
+	}
+	for id := 0; id < dataShards; id++ {
+		if !present.Has(erasure_coding.ShardId(id)) {
+			return erasure_coding.ShardId(id), true
+		}
+	}
+	return 0, false
+}
+
+// regularVolumeHolder returns a server already serving vid as a regular
+// volume. A decode only finds one when an earlier run was interrupted between
+// regenerating the volume and deleting the shards it came from.
+func regularVolumeHolder(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) (pb.ServerAddress, bool) {
+	var holder pb.ServerAddress
+	found := false
+	EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+		if found {
+			return
+		}
+		for _, diskInfo := range dn.DiskInfos {
+			for _, vi := range diskInfo.VolumeInfos {
+				if needle.VolumeId(vi.Id) == vid {
+					holder = pb.NewServerAddressFromDataNode(dn)
+					found = true
+					return
+				}
+			}
+		}
+	})
+	return holder, found
+}
+
+func verifyDecodedVolumeBeforeDelete(grpcDialOption grpc.DialOption, target pb.ServerAddress, vid needle.VolumeId) error {
+	var resp *volume_server_pb.ReadVolumeFileStatusResponse
+	if err := operation.WithVolumeServerClient(false, target, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+		r, e := client.ReadVolumeFileStatus(context.Background(), &volume_server_pb.ReadVolumeFileStatusRequest{
+			VolumeId: uint32(vid),
+		})
+		if e != nil {
+			return e
+		}
+		resp = r
+		return nil
+	}); err != nil {
+		return fmt.Errorf("read volume file status: %w", err)
+	}
+	if resp.DatFileSize == 0 {
+		return fmt.Errorf("decoded .dat is 0 bytes")
+	}
+	if resp.IdxFileSize == 0 {
+		return fmt.Errorf("decoded .idx is 0 bytes")
+	}
+	glog.V(0).Infof("ec decode verification ok for volume %d on %s: dat=%d idx=%d", vid, target, resp.DatFileSize, resp.IdxFileSize)
+	return nil
+}
+
+func mountDecodedVolume(grpcDialOption grpc.DialOption, targetNodeLocation pb.ServerAddress, vid needle.VolumeId) error {
+	return operation.WithVolumeServerClient(false, targetNodeLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+		_, mountErr := volumeServerClient.VolumeMount(context.Background(), &volume_server_pb.VolumeMountRequest{
+			VolumeId: uint32(vid),
+		})
+		return mountErr
+	})
+}
+
+func generateNormalVolume(grpcDialOption grpc.DialOption, vid needle.VolumeId, collection string, sourceVolumeServer pb.ServerAddress) error {
+	fmt.Printf("generateNormalVolume from ec volume %d on %s\n", vid, sourceVolumeServer)
+
+	err := operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+		_, genErr := volumeServerClient.VolumeEcShardsToVolume(context.Background(), &volume_server_pb.VolumeEcShardsToVolumeRequest{
+			VolumeId:   uint32(vid),
+			Collection: collection,
+		})
+		return genErr
+	})
+
+	return err
+
+}
+
+func collectEcShards(env *Env, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, collection string, vid needle.VolumeId, eligibleTargets map[pb.ServerAddress]struct{}, dataShards int) (targetNodeLocation pb.ServerAddress, err error) {
+
+	maxShardCount := -1
+	existingShardsInfo := erasure_coding.NewShardsInfo()
+	for loc, si := range nodeToShardsInfo {
+		if eligibleTargets != nil {
+			if _, ok := eligibleTargets[loc]; !ok {
+				continue
+			}
+		}
+		toBeCopiedShardCount := si.MinusParityShards(dataShards).Count()
+		if toBeCopiedShardCount > maxShardCount {
+			maxShardCount = toBeCopiedShardCount
+			targetNodeLocation = loc
+			existingShardsInfo = si
+		}
+	}
+	if targetNodeLocation == "" {
+		return "", fmt.Errorf("no eligible target datanodes available to decode volume %d", vid)
+	}
+
+	// Trust only shards the target can actually serve: an interrupted earlier
+	// decode or balance can leave the master believing the target holds a
+	// shard whose file never landed. A phantom entry here would exclude the
+	// shard from the copy set and the decode would then fail with "missing
+	// shard"; probing the target's live inventory makes the re-run re-copy it.
+	if present, probeErr := erasure_coding.CollectShardsOnServer(context.Background(), collection, uint32(vid), string(targetNodeLocation), env.GrpcDialOption); probeErr == nil {
+		confirmed := erasure_coding.NewShardsInfo()
+		for _, sid := range existingShardsInfo.Ids() {
+			if present.Has(sid) {
+				confirmed.Set(erasure_coding.NewShardInfo(sid, 0))
+			}
+		}
+		existingShardsInfo = confirmed
+	} else {
+		fmt.Printf("collectEcShards: probe %s inventory for volume %d: %v (keeping the topology's view)\n", targetNodeLocation, vid, probeErr)
+	}
+
+	fmt.Printf("collectEcShards: ec volume %d collect shards to %s from: %+v\n", vid, targetNodeLocation, nodeToShardsInfo)
+
+	copiedShardsInfo := erasure_coding.NewShardsInfo()
+	for loc, si := range nodeToShardsInfo {
+		if loc == targetNodeLocation {
+			continue
+		}
+
+		needToCopyShardsInfo := si.Minus(existingShardsInfo).MinusParityShards(dataShards)
+
+		err = operation.WithVolumeServerClient(false, targetNodeLocation, env.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+
+			// Always collect .ecj from every shard location. Each server's .ecj
+			// only contains deletions for needles whose data resides in shards
+			// held by that server. Without merging all .ecj files, deletions
+			// recorded on other servers would be lost during decode.
+			if needToCopyShardsInfo.Count() > 0 {
+				fmt.Printf("copy %d.%v %s => %s\n", vid, needToCopyShardsInfo.Ids(), loc, targetNodeLocation)
+			} else {
+				fmt.Printf("collect ecj %d %s => %s\n", vid, loc, targetNodeLocation)
+			}
+
+			_, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{
+				VolumeId:       uint32(vid),
+				Collection:     collection,
+				ShardIds:       needToCopyShardsInfo.IdsUint32(),
+				CopyEcxFile:    false,
+				CopyEcjFile:    true,
+				CopyVifFile:    needToCopyShardsInfo.Count() > 0,
+				SourceDataNode: string(loc),
+			})
+			if copyErr != nil {
+				return fmt.Errorf("copy %d.%v %s => %s: %v", vid, needToCopyShardsInfo.Ids(), loc, targetNodeLocation, copyErr)
+			}
+
+			if needToCopyShardsInfo.Count() > 0 {
+				fmt.Printf("mount %d.%v on %s\n", vid, needToCopyShardsInfo.Ids(), targetNodeLocation)
+				_, mountErr := volumeServerClient.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
+					VolumeId:   uint32(vid),
+					Collection: collection,
+					ShardIds:   needToCopyShardsInfo.IdsUint32(),
+				})
+				if mountErr != nil {
+					return fmt.Errorf("mount %d.%v on %s: %v", vid, needToCopyShardsInfo.Ids(), targetNodeLocation, mountErr)
+				}
+			}
+
+			return nil
+		})
+
+		if err != nil {
+			break
+		}
+
+		copiedShardsInfo.Add(needToCopyShardsInfo)
+	}
+
+	nodeToShardsInfo[targetNodeLocation] = existingShardsInfo.Plus(copiedShardsInfo)
+
+	return targetNodeLocation, err
+}
+func CollectEcShardIds(topoInfo *master_pb.TopologyInfo, collectionMatcher *wildcard.CollectionMatcher, diskType types.DiskType) (vids []needle.VolumeId) {
+	vidMap := make(map[uint32]bool)
+	EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+		if diskInfo, found := dn.DiskInfos[string(diskType)]; found {
+			for _, v := range diskInfo.EcShardInfos {
+				if collectionMatcher.Matches(v.Collection) {
+					vidMap[v.Id] = true
+				}
+			}
+		}
+	})
+
+	for vid := range vidMap {
+		vids = append(vids, needle.VolumeId(vid))
+	}
+
+	return
+}
+
+func collectEcNodeShardsInfo(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) (map[pb.ServerAddress]*erasure_coding.ShardsInfo, int) {
+	res := make(map[pb.ServerAddress]*erasure_coding.ShardsInfo)
+	EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+		// Union across ALL disk-type buckets and, within a node, across its
+		// physical disks. Shards sit wherever encode generation and balance
+		// left them — a cross-tier encode leaves them in the source bucket, a
+		// partial migration straddles buckets — and a decode that only looks
+		// at one bucket reports a decodable volume as having no shards at all.
+		// (Same rationale as the encode's shard verification.)
+		for _, diskInfo := range dn.DiskInfos {
+			if diskInfo == nil {
+				continue
+			}
+			for _, v := range diskInfo.EcShardInfos {
+				if v.Id == uint32(vid) {
+					addr := pb.NewServerAddressFromDataNode(dn)
+					si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(v)
+					if existing, ok := res[addr]; ok {
+						existing.Add(si)
+					} else {
+						res[addr] = si
+					}
+				}
+			}
+		}
+	})
+
+	// OSS is always 10+4; the per-volume ratio override lives in the enterprise build.
+	return res, erasure_coding.DataShardsCount
+}
+
+type DecodeDiskUsageState struct {
+	byNode map[pb.ServerAddress]*decodeDiskUsageCounts
+}
+
+type decodeDiskUsageCounts struct {
+	maxVolumeCount    int64
+	volumeCount       int64
+	remoteVolumeCount int64
+	ecShardCount      int64
+}
+
+func NewDecodeDiskUsageState(topoInfo *master_pb.TopologyInfo, diskType types.DiskType) *DecodeDiskUsageState {
+	state := &DecodeDiskUsageState{byNode: make(map[pb.ServerAddress]*decodeDiskUsageCounts)}
+	EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+		if diskInfo, found := dn.DiskInfos[string(diskType)]; found {
+			state.byNode[pb.NewServerAddressFromDataNode(dn)] = &decodeDiskUsageCounts{
+				maxVolumeCount:    diskInfo.MaxVolumeCount,
+				volumeCount:       diskInfo.VolumeCount,
+				remoteVolumeCount: diskInfo.RemoteVolumeCount,
+				ecShardCount:      int64(CountShards(diskInfo.EcShardInfos)),
+			}
+		}
+	})
+	return state
+}
+
+func (state *DecodeDiskUsageState) freeVolumeCount(location pb.ServerAddress) (int64, bool) {
+	if state == nil {
+		return 0, false
+	}
+	usage, found := state.byNode[location]
+	if !found {
+		return 0, false
+	}
+	free := usage.maxVolumeCount - (usage.volumeCount - usage.remoteVolumeCount)
+	free -= (usage.ecShardCount + int64(erasure_coding.DataShardsCount) - 1) / int64(erasure_coding.DataShardsCount)
+	return free, true
+}
+
+func (state *DecodeDiskUsageState) applyDecode(targetNodeLocation pb.ServerAddress, shardCounts map[pb.ServerAddress]int, createdVolume bool) {
+	if state == nil {
+		return
+	}
+	for location, shardCount := range shardCounts {
+		if usage, found := state.byNode[location]; found {
+			usage.ecShardCount -= int64(shardCount)
+		}
+	}
+	if createdVolume {
+		if usage, found := state.byNode[targetNodeLocation]; found {
+			usage.volumeCount++
+		}
+	}
+}

weed/ec/ec_decode_test.go

@@ -0,0 +1,91 @@
+package ec
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+// shardsOn builds one holder's inventory.
+func shardsOn(ids ...int) *erasure_coding.ShardsInfo {
+	si := erasure_coding.NewShardsInfo()
+	for _, id := range ids {
+		si.Set(erasure_coding.NewShardInfo(erasure_coding.ShardId(id), 1024))
+	}
+	return si
+}
+
+// missingDataShard decides whether a decode may delete the shards of a volume
+// that already exists elsewhere. Only the deletion phase of a decode removes a
+// data shard, so the answer separates "an earlier decode was interrupted while
+// cleaning up" from the two states that look the same from the outside: an
+// encode interrupted before it deleted the original, and a decode killed while
+// generating. Both of those leave every shard in place, and the volume beside
+// them may be the untouched original or a half-written one -- neither is safe
+// to trade for the shards.
+func TestMissingDataShard(t *testing.T) {
+	const dataShards = 10
+
+	tests := []struct {
+		name       string
+		holders    map[pb.ServerAddress]*erasure_coding.ShardsInfo
+		wantID     erasure_coding.ShardId
+		wantMissed bool
+	}{
+		{
+			name: "complete set on one holder decodes, so nothing is finished",
+			holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": shardsOn(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13),
+			},
+		},
+		{
+			name: "complete set spread across holders is still complete",
+			holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": shardsOn(0, 1, 2, 3, 4),
+				"server2:8080": shardsOn(5, 6, 7, 8, 9),
+				"server3:8080": shardsOn(10, 11, 12, 13),
+			},
+		},
+		{
+			name: "parity gone is not the deletion phase: the volume still decodes",
+			holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": shardsOn(0, 1, 2, 3, 4, 5, 6, 7, 8, 9),
+			},
+		},
+		{
+			name: "a data shard gone is a decode that was interrupted mid-cleanup",
+			holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": shardsOn(0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13),
+			},
+			wantID:     6,
+			wantMissed: true,
+		},
+		{
+			name: "the first gap is reported, not the last",
+			holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": shardsOn(0, 3, 4, 5, 6, 7, 8, 9),
+			},
+			wantID:     1,
+			wantMissed: true,
+		},
+		{
+			name:       "no holders at all leaves every data shard missing",
+			holders:    map[pb.ServerAddress]*erasure_coding.ShardsInfo{},
+			wantID:     0,
+			wantMissed: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			id, missed := missingDataShard(tt.holders, dataShards)
+			if missed != tt.wantMissed {
+				t.Fatalf("missingDataShard = %v, want %v", missed, tt.wantMissed)
+			}
+			if missed && id != tt.wantID {
+				t.Errorf("missing shard id = %d, want %d", id, tt.wantID)
+			}
+		})
+	}
+}

weed/ec/ec_destructive_guards_test.go

@@ -0,0 +1,231 @@
+package ec
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// topoWith builds a one-node topology snapshot where regularVids appear as
+// regular volumes (.dat) and ecVids appear as EC shards.
+func topoWith(regularVids, ecVids []uint32) *master_pb.TopologyInfo {
+	di := &master_pb.DiskInfo{Type: "hdd"}
+	for _, v := range regularVids {
+		di.VolumeInfos = append(di.VolumeInfos, &master_pb.VolumeInformationMessage{Id: v})
+	}
+	for _, v := range ecVids {
+		di.EcShardInfos = append(di.EcShardInfos, &master_pb.VolumeEcShardInformationMessage{Id: v})
+	}
+	return &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{{
+			RackInfos: []*master_pb.RackInfo{{
+				DataNodeInfos: []*master_pb.DataNodeInfo{{
+					Id:        "n1",
+					DiskInfos: map[string]*master_pb.DiskInfo{"hdd": di},
+				}},
+			}},
+		}},
+	}
+}
+
+// TestAssertEncodableRegularVolumes: ec.encode must refuse an already-EC volume
+// (no .dat — re-encoding would destroy its shards) and an unknown id, while
+// allowing a regular volume and a failed-encode retry (regular .dat + orphans).
+func TestAssertEncodableRegularVolumes(t *testing.T) {
+	cases := []struct {
+		name          string
+		regular, ec   []uint32
+		vids          []needle.VolumeId
+		wantErrSubstr string
+	}{
+		{"regular volume", []uint32{1}, nil, []needle.VolumeId{1}, ""},
+		{"already EC", nil, []uint32{1}, []needle.VolumeId{1}, "already EC"},
+		{"regular plus stale orphan shards", []uint32{1}, []uint32{1}, []needle.VolumeId{1}, ""},
+		{"unknown id", nil, nil, []needle.VolumeId{1}, "not found"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := AssertEncodableRegularVolumes(topoWith(tc.regular, tc.ec), tc.vids)
+			if tc.wantErrSubstr == "" {
+				if err != nil {
+					t.Fatalf("want nil error, got %v", err)
+				}
+				return
+			}
+			if err == nil || !strings.Contains(err.Error(), tc.wantErrSubstr) {
+				t.Fatalf("want error containing %q, got %v", tc.wantErrSubstr, err)
+			}
+		})
+	}
+}
+
+func newDryRunRebuilder(log *bytes.Buffer, nodes ...*EcNode) *ecRebuilder {
+	return &ecRebuilder{
+		env:          &Env{},
+		ecNodes:      nodes,
+		writer:       log,
+		applyChanges: false,
+	}
+}
+
+// TestPrepareDataToRecover_DryRunRecoverableNoSideEffects: a dry-run on a
+// recoverable degraded volume the rebuilder holds none of must still succeed
+// and report its plan, while recording zero shards to delete (so the deferred
+// cleanup issues no VolumeEcShardsDelete RPC).
+func TestPrepareDataToRecover_DryRunRecoverableNoSideEffects(t *testing.T) {
+	var log bytes.Buffer
+	rebuilder := newEcNode("dc1", "rack1", "rebuilder", 100)
+	remote := newEcNode("dc1", "rack1", "remote", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11})
+	erb := newDryRunRebuilder(&log, rebuilder, remote)
+
+	locations := make(EcShardLocations, erasure_coding.MaxShardCount)
+	for i := 0; i < 12; i++ {
+		locations[i] = []*EcNode{remote}
+	}
+
+	copied, local, err := erb.prepareDataToRecover(rebuilder, "c1", needle.VolumeId(1), locations)
+	if err != nil {
+		t.Fatalf("dry-run on a recoverable volume must not error: %v", err)
+	}
+	if len(copied) != 0 {
+		t.Errorf("dry-run must record no copied shards (deletion set), got %v", copied)
+	}
+	if len(local) != 0 {
+		t.Errorf("rebuilder holds no shards locally, got %v", local)
+	}
+	if !strings.Contains(log.String(), "would copy") {
+		t.Errorf("dry-run should report the would-copy plan, got:\n%s", log.String())
+	}
+}
+
+// TestPrepareDataToRecover_UnionsLocalShardsAcrossDisks: shards the rebuilder
+// holds on more than one disk must all count as local (not copied), so the
+// per-disk overwrite that made one disk's shards look remote (then self-copied
+// with O_TRUNC and node-wide deleted) cannot happen.
+func TestPrepareDataToRecover_UnionsLocalShardsAcrossDisks(t *testing.T) {
+	var log bytes.Buffer
+	rebuilder := newEcNode("dc1", "rack1", "rebuilder", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1}).
+		AddEcVolumeShards(needle.VolumeId(1), "c1", []erasure_coding.ShardId{5}, types.SsdType)
+	remote := newEcNode("dc1", "rack1", "remote", 100)
+	erb := newDryRunRebuilder(&log, rebuilder)
+
+	locations := make(EcShardLocations, erasure_coding.MaxShardCount)
+	for i := 0; i < 12; i++ {
+		locations[i] = []*EcNode{remote}
+	}
+	locations[0] = []*EcNode{rebuilder}
+	locations[1] = []*EcNode{rebuilder}
+	locations[5] = []*EcNode{rebuilder}
+
+	copied, local, err := erb.prepareDataToRecover(rebuilder, "c1", needle.VolumeId(1), locations)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	missing := map[erasure_coding.ShardId]bool{0: true, 1: true, 5: true}
+	for _, s := range local {
+		delete(missing, s)
+	}
+	if len(missing) != 0 {
+		t.Errorf("shards %v should be local (union across disks); local=%v", missing, local)
+	}
+	for _, s := range copied {
+		if s == 0 || s == 1 || s == 5 {
+			t.Errorf("local shard %d must never be in the copy/delete set", s)
+		}
+	}
+}
+
+// TestCountLocalShards_UnionsAcrossDisks: slot accounting must union shards
+// across the node's disks like prepareDataToRecover does, or slotsNeeded is
+// overstated and selectAndReserveRebuilder can reject a valid node.
+func TestCountLocalShards_UnionsAcrossDisks(t *testing.T) {
+	var log bytes.Buffer
+	rebuilder := newEcNode("dc1", "rack1", "rebuilder", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1}).
+		AddEcVolumeShards(needle.VolumeId(1), "c1", []erasure_coding.ShardId{5}, types.SsdType)
+	erb := newDryRunRebuilder(&log, rebuilder)
+
+	if got := erb.countLocalShards(rebuilder, "c1", needle.VolumeId(1)); got != 3 {
+		t.Errorf("countLocalShards must union across disks: want 3, got %d", got)
+	}
+}
+
+// TestPrepareDataToRecover_SelfSourceNotCopied: if the rebuilder is the only
+// listed holder of a shard it does not report locally, it must be treated as
+// local rather than copied onto itself and then deleted.
+func TestPrepareDataToRecover_SelfSourceNotCopied(t *testing.T) {
+	var log bytes.Buffer
+	rebuilder := newEcNode("dc1", "rack1", "rebuilder", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 6, 7, 8, 9, 10})
+	erb := newDryRunRebuilder(&log, rebuilder)
+
+	locations := make(EcShardLocations, erasure_coding.MaxShardCount)
+	for _, s := range []int{0, 1, 2, 3, 4, 6, 7, 8, 9, 10} {
+		locations[s] = []*EcNode{rebuilder}
+	}
+	locations[5] = []*EcNode{rebuilder} // sole holder is the rebuilder, not in its reported shards
+
+	copied, local, err := erb.prepareDataToRecover(rebuilder, "c1", needle.VolumeId(1), locations)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	for _, s := range copied {
+		if s == 5 {
+			t.Errorf("self-sourced shard 5 must not be copied/deleted")
+		}
+	}
+	found := false
+	for _, s := range local {
+		if s == 5 {
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("self-sourced shard 5 should be classified local, got local=%v", local)
+	}
+}
+
+func TestPrepareDataToRecover_ApplyCopyFailureDoesNotCountAsRecoverable(t *testing.T) {
+	var log bytes.Buffer
+	rebuilder := newEcNode("dc1", "rack1", "127.0.0.1:1", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8})
+	remote := newEcNode("dc1", "rack1", "127.0.0.1:2", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{9})
+	erb := &ecRebuilder{
+		env: &Env{
+			GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+		},
+		ecNodes:      []*EcNode{rebuilder, remote},
+		writer:       &log,
+		applyChanges: true,
+	}
+
+	locations := make(EcShardLocations, erasure_coding.MaxShardCount)
+	for i := 0; i < 9; i++ {
+		locations[i] = []*EcNode{rebuilder}
+	}
+	locations[9] = []*EcNode{remote}
+
+	copied, local, err := erb.prepareDataToRecover(rebuilder, "c1", needle.VolumeId(1), locations)
+	if err == nil {
+		t.Fatalf("copy failure must leave only 9 usable shards and fail recoverability gate; copied=%v local=%v log:\n%s", copied, local, log.String())
+	}
+	if !strings.Contains(err.Error(), "9 shards are not enough") {
+		t.Fatalf("expected recoverability error to count only successful/local shards, got %v; log:\n%s", err, log.String())
+	}
+	if len(copied) != 0 {
+		t.Fatalf("failed copy must not be recorded as copied/deletable, got %v", copied)
+	}
+	if !strings.Contains(log.String(), "failed to copy 1.9") {
+		t.Fatalf("expected failed copy to be logged, got:\n%s", log.String())
+	}
+}

weed/ec/ec_encode_shard_sizes_test.go

@@ -0,0 +1,214 @@
+package ec
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// sizedShards builds one holder's inventory as {shard id: size}.
+func sizedShards(idToSize map[int]int64) *erasure_coding.ShardsInfo {
+	si := erasure_coding.NewShardsInfo()
+	for id, size := range idToSize {
+		si.Set(erasure_coding.NewShardInfo(erasure_coding.ShardId(id), erasure_coding.ShardSize(size)))
+	}
+	return si
+}
+
+// An encode deletes the volume the shards were made from, on the strength of
+// having counted them. Every shard takes one piece of each block row, so they
+// are written to the same length: one that disagrees was truncated or half
+// copied, and counting cannot see it.
+func TestRequireUniformShardSizes(t *testing.T) {
+	tests := []struct {
+		name        string
+		byNode      map[pb.ServerAddress]*erasure_coding.ShardsInfo
+		wantErr     bool
+		errContains []string
+	}{
+		{
+			name: "one holder, all shards the same length",
+			byNode: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": sizedShards(map[int]int64{0: 1048576, 1: 1048576, 2: 1048576}),
+			},
+		},
+		{
+			name: "spread across holders, still one length",
+			byNode: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": sizedShards(map[int]int64{0: 1048576, 1: 1048576}),
+				"server2:8080": sizedShards(map[int]int64{2: 1048576, 3: 1048576}),
+			},
+		},
+		{
+			name: "a truncated shard is named with its holder",
+			byNode: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": sizedShards(map[int]int64{0: 1048576, 1: 1048576}),
+				"server2:8080": sizedShards(map[int]int64{2: 4096}),
+			},
+			wantErr:     true,
+			errContains: []string{"disagree on size", "server2:8080.2", "4096"},
+		},
+		{
+			name: "two copies of one shard that disagree are still a disagreement",
+			byNode: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": sizedShards(map[int]int64{5: 1048576}),
+				"server2:8080": sizedShards(map[int]int64{5: 1048570}),
+			},
+			wantErr:     true,
+			errContains: []string{"disagree on size"},
+		},
+
+		// Sizes the cluster does not report must never block an encode: an
+		// older volume server, or one that has not heartbeated its sizes yet,
+		// reports zero, and refusing to delete on that would strand every
+		// encode against such a cluster in the hybrid state this check exists
+		// to avoid.
+		{
+			name: "unreported sizes are skipped, not read as a disagreement",
+			byNode: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": sizedShards(map[int]int64{0: 1048576, 1: 0}),
+				"server2:8080": sizedShards(map[int]int64{2: 0}),
+			},
+		},
+		{
+			name: "nothing reported at all verifies vacuously",
+			byNode: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": sizedShards(map[int]int64{0: 0, 1: 0}),
+			},
+		},
+		{
+			name:   "no holders is not a size problem",
+			byNode: map[pb.ServerAddress]*erasure_coding.ShardsInfo{},
+		},
+		{
+			name: "a nil inventory is skipped rather than panicking",
+			byNode: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": nil,
+				"server2:8080": sizedShards(map[int]int64{0: 1048576}),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := requireUniformShardSizes(needle.VolumeId(7), tt.byNode)
+			if !tt.wantErr {
+				if err != nil {
+					t.Fatalf("want the encode to proceed, got %v", err)
+				}
+				return
+			}
+			if err == nil {
+				t.Fatal("want the deletion held back, got no error")
+			}
+			for _, want := range tt.errContains {
+				if !strings.Contains(err.Error(), want) {
+					t.Errorf("error %q does not mention %q", err, want)
+				}
+			}
+		})
+	}
+}
+
+// ecShardReport builds one heartbeat entry: a generation stamp and the shards
+// that generation put on this node, all of one size.
+func ecShardReport(vid uint32, encodeTsNs int64, size int64, ids ...int) *master_pb.VolumeEcShardInformationMessage {
+	si := erasure_coding.NewShardsInfo()
+	for _, id := range ids {
+		si.Set(erasure_coding.NewShardInfo(erasure_coding.ShardId(id), erasure_coding.ShardSize(size)))
+	}
+	return &master_pb.VolumeEcShardInformationMessage{
+		Id:          vid,
+		Collection:  "ectest",
+		EcIndexBits: si.Bitmap(),
+		ShardSizes:  si.SizesInt64(),
+		EncodeTsNs:  encodeTsNs,
+	}
+}
+
+func topologyWith(reports map[string][]*master_pb.VolumeEcShardInformationMessage) *master_pb.TopologyInfo {
+	var nodes []*master_pb.DataNodeInfo
+	for addr, shards := range reports {
+		nodes = append(nodes, &master_pb.DataNodeInfo{
+			Id: addr,
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"hdd": {EcShardInfos: shards},
+			},
+		})
+	}
+	return &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{{
+			Id:        "dc1",
+			RackInfos: []*master_pb.RackInfo{{Id: "rack1", DataNodeInfos: nodes}},
+		}},
+	}
+}
+
+// A re-encode can change the ratio, so an older generation's shards are a
+// different length by nature. Recoverability already judges only the newest
+// generation; if the size check did not, an orphaned older encode -- one the
+// pre-encode sweep could not reach, but the master still hears about -- would
+// disagree on size at every retry and strand the volume beside its shards for
+// good.
+func TestShardSizeCheckIgnoresOlderEncodeGenerations(t *testing.T) {
+	const vid = 9
+
+	topo := topologyWith(map[string][]*master_pb.VolumeEcShardInformationMessage{
+		// the current encode: one length, spread over two nodes
+		"server1:8080": {ecShardReport(vid, 2000, 1048576, 0, 1, 2)},
+		"server2:8080": {ecShardReport(vid, 2000, 1048576, 3, 4, 5)},
+		// an orphan from an earlier encode, at a different length
+		"server3:8080": {ecShardReport(vid, 1000, 524288, 0, 1)},
+	})
+
+	byNode := collectNewestGenerationShardsInfo(topo, needle.VolumeId(vid))
+	if _, stale := byNode[pb.ServerAddress("server3:8080")]; stale {
+		t.Error("the older generation's holder must not be collected")
+	}
+	if err := requireUniformShardSizes(needle.VolumeId(vid), byNode); err != nil {
+		t.Fatalf("a healthy current generation must not be vetoed by an orphan: %v", err)
+	}
+}
+
+// Fencing to the newest generation must not blind the check: a short shard
+// inside that generation is exactly what it exists to catch.
+func TestShardSizeCheckStillCatchesShortShardInNewestGeneration(t *testing.T) {
+	const vid = 9
+
+	topo := topologyWith(map[string][]*master_pb.VolumeEcShardInformationMessage{
+		"server1:8080": {ecShardReport(vid, 2000, 1048576, 0, 1)},
+		"server2:8080": {ecShardReport(vid, 2000, 4096, 2)},
+		"server3:8080": {ecShardReport(vid, 1000, 524288, 0)},
+	})
+
+	err := requireUniformShardSizes(needle.VolumeId(vid), collectNewestGenerationShardsInfo(topo, needle.VolumeId(vid)))
+	if err == nil {
+		t.Fatal("a truncated shard in the newest generation must hold the deletion back")
+	}
+	if !strings.Contains(err.Error(), "server2:8080.2") {
+		t.Errorf("error %q does not name the short shard", err)
+	}
+}
+
+// Volumes encoded before generation stamping report zero, which is a single
+// legacy generation rather than an orphan to drop.
+func TestShardSizeCheckHandlesUnstampedGenerations(t *testing.T) {
+	const vid = 9
+
+	topo := topologyWith(map[string][]*master_pb.VolumeEcShardInformationMessage{
+		"server1:8080": {ecShardReport(vid, 0, 1048576, 0, 1)},
+		"server2:8080": {ecShardReport(vid, 0, 4096, 2)},
+	})
+
+	byNode := collectNewestGenerationShardsInfo(topo, needle.VolumeId(vid))
+	if len(byNode) != 2 {
+		t.Fatalf("unstamped shards form one generation, got %d holder(s)", len(byNode))
+	}
+	if err := requireUniformShardSizes(needle.VolumeId(vid), byNode); err == nil {
+		t.Error("a truncated legacy shard must still hold the deletion back")
+	}
+}

weed/ec/ec_encode_test.go

@@ -0,0 +1,531 @@
+package ec
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSelectVolumeIdsFromTopology(t *testing.T) {
+	// Topology volumeSizeLimit:1000 MB hdd(volume:28/30 active:28 free:2 remote:0)
+	//   DataCenter DefaultDataCenter hdd(volume:28/30 active:28 free:2 remote:0)
+	//     Rack DefaultRack hdd(volume:28/30 active:28 free:2 remote:0)
+	//       DataNode seaweedfs-volume-0.seaweedfs-volume.sea:8080 hdd(volume:14/15 active:14 free:1 remote:0)
+	//         Disk hdd(volume:14/15 active:14 free:1 remote:0) id:0
+	//           volume Id:1, Size:23520, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:3, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366011
+	//           volume Id:2, Size:28536, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:1, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770365291
+	//           volume Id:3, Size:100752, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364931
+	//           volume Id:4, Size:3112, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:4, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300850
+	//           volume Id:5, Size:145208, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770365652
+	//           volume Id:6, Size:146456, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:4, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364812
+	//           volume Id:7, Size:8, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:0, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300824
+	//           volume Id:8, Size:808, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300796
+	//           volume Id:9, Size:8, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:0, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300814
+	//           volume Id:10, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364737
+	//           volume Id:11, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364796
+	//           volume Id:12, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364829
+	//           volume Id:13, Size:570428616, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:68, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366062
+	//           volume Id:14, Size:478153400, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:57, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366020
+	//         Disk hdd {Size:4194776448 FileCount:518 DeletedFileCount:0 DeletedBytes:0}
+	//       DataNode seaweedfs-volume-0.seaweedfs-volume.sea:8080 {Size:4194776448 FileCount:518 DeletedFileCount:0 DeletedBytes:0}
+	//   DataCenter DefaultDataCenter hdd(volume:28/30 active:28 free:2 remote:0)
+	//     Rack DefaultRack hdd(volume:28/30 active:28 free:2 remote:0)
+	//       DataNode seaweedfs-volume-1.seaweedfs-volume.sea:8080 hdd(volume:14/15 active:14 free:1 remote:0)
+	//         Disk hdd(volume:14/15 active:14 free:1 remote:0) id:0
+	//           volume Id:1, Size:23520, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:3, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366012
+	//           volume Id:2, Size:28536, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:1, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770365292
+	//           volume Id:3, Size:100752, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364931
+	//           volume Id:4, Size:3112, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:4, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300850
+	//           volume Id:5, Size:145208, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770365653
+	//           volume Id:6, Size:146456, ReplicaPlacement:001, Collection:, Version:3, Ttl:, FileCount:4, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364812
+	//           volume Id:7, Size:8, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:0, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300824
+	//           volume Id:8, Size:808, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:2, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300796
+	//           volume Id:9, Size:8, ReplicaPlacement:001, Collection:encrypt-data, Version:3, Ttl:, FileCount:0, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770300814
+	//           volume Id:10, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364737
+	//           volume Id:11, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364796
+	//           volume Id:12, Size:1048582008, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:125, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770364828
+	//           volume Id:13, Size:570428616, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:68, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366062
+	//           volume Id:14, Size:478153400, ReplicaPlacement:001, Collection:test, Version:3, Ttl:, FileCount:57, DeleteCount:0, DeletedByteCount:0, ReadOnly:false, ModifiedAtSecond:1770366020
+	//         Disk hdd {Size:4194776448 FileCount:518 DeletedFileCount:0 DeletedBytes:0}
+	//       DataNode seaweedfs-volume-1.seaweedfs-volume.sea:8080 {Size:4194776448 FileCount:518 DeletedFileCount:0 DeletedBytes:0}
+	//     Rack DefaultRack {Size:8389552896 FileCount:1036 DeletedFileCount:0 DeletedBytes:0}
+	//   DataCenter DefaultDataCenter {Size:8389552896 FileCount:1036 DeletedFileCount:0 DeletedBytes:0}
+	// total size:8389552896 file_count:1036
+
+	topologyInfo := &master_pb.TopologyInfo{
+		Id: "DefaultDataCenter",
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				Id: "DefaultDataCenter",
+				RackInfos: []*master_pb.RackInfo{
+					{
+						Id: "DefaultRack",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								Id: "seaweedfs-volume-0.seaweedfs-volume.sea:8080",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									"hdd": {
+										Type:            "hdd",
+										FreeVolumeCount: 1,
+										VolumeInfos: []*master_pb.VolumeInformationMessage{
+											{Id: 1, Size: 23520, Collection: "", ModifiedAtSecond: 1770366011},
+											{Id: 2, Size: 28536, Collection: "", ModifiedAtSecond: 1770365291},
+											{Id: 3, Size: 100752, Collection: "", ModifiedAtSecond: 1770364931},
+											{Id: 4, Size: 3112, Collection: "", ModifiedAtSecond: 1770300850},
+											{Id: 5, Size: 145208, Collection: "", ModifiedAtSecond: 1770365652},
+											{Id: 6, Size: 146456, Collection: "", ModifiedAtSecond: 1770364812},
+											{Id: 7, Size: 8, Collection: "encrypt-data", ModifiedAtSecond: 1770300824},
+											{Id: 8, Size: 808, Collection: "encrypt-data", ModifiedAtSecond: 1770300796},
+											{Id: 9, Size: 8, Collection: "encrypt-data", ModifiedAtSecond: 1770300814},
+											{Id: 10, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364737},
+											{Id: 11, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364796},
+											{Id: 12, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364829},
+											{Id: 13, Size: 570428616, Collection: "test", ModifiedAtSecond: 1770366062},
+											{Id: 14, Size: 478153400, Collection: "test", ModifiedAtSecond: 1770366020},
+										},
+									},
+								},
+							},
+							{
+								Id: "seaweedfs-volume-1.seaweedfs-volume.sea:8080",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									"hdd": {
+										Type:            "hdd",
+										FreeVolumeCount: 1,
+										VolumeInfos: []*master_pb.VolumeInformationMessage{
+											{Id: 1, Size: 23520, Collection: "", ModifiedAtSecond: 1770366012},
+											{Id: 2, Size: 28536, Collection: "", ModifiedAtSecond: 1770365292},
+											{Id: 3, Size: 100752, Collection: "", ModifiedAtSecond: 1770364931},
+											{Id: 4, Size: 3112, Collection: "", ModifiedAtSecond: 1770300850},
+											{Id: 5, Size: 145208, Collection: "", ModifiedAtSecond: 1770365653},
+											{Id: 6, Size: 146456, Collection: "", ModifiedAtSecond: 1770364812},
+											{Id: 7, Size: 8, Collection: "encrypt-data", ModifiedAtSecond: 1770300824},
+											{Id: 8, Size: 808, Collection: "encrypt-data", ModifiedAtSecond: 1770300796},
+											{Id: 9, Size: 8, Collection: "encrypt-data", ModifiedAtSecond: 1770300814},
+											{Id: 10, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364737},
+											{Id: 11, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364796},
+											{Id: 12, Size: 1048582008, Collection: "test", ModifiedAtSecond: 1770364828},
+											{Id: 13, Size: 570428616, Collection: "test", ModifiedAtSecond: 1770366062},
+											{Id: 14, Size: 478153400, Collection: "test", ModifiedAtSecond: 1770366020},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	volumeSizeLimitMb := uint64(1000)
+	collectionMatcher, _ := wildcard.CompileCollectionMatcher(".*")
+
+	// ec.encode -force
+	// force means we ignore the check for 4+ volume servers.
+	// But in this unit test we are testing selectVolumeIdsFromTopology, which selects volumes.
+	// The -force flag is handled in the caller of this function.
+
+	// Default values
+	quietPeriod := time.Hour
+	fullPercentage := 95.0
+	verbose := true
+
+	// Mock time to be slightly after the latest modification time in the test data
+	// The latest ModifiedAtSecond in the test data is 1770366062 (Volume 13)
+	// Let's set now to be 1770366062 + quietPeriod.Seconds() + 100
+	nowUnixSeconds := int64(1770366062) + int64(quietPeriod.Seconds()) + 100
+	quietSeconds := int64(quietPeriod.Seconds())
+
+	// Test case 1: Select all volumes with sufficient size
+	// Note: The logic requires (FreeVolumeCount >= 2) unless the volume is already selected?
+	// Wait, distinct volume IDs.
+	// Logic:
+	// if diskInfo.FreeVolumeCount < 2 { skip }
+	// In the test data, FreeVolumeCount is 1. So it should skip all volumes basically?
+	// Let's check the code:
+	/*
+		// check free disk space
+		if good, found := vidMap[v.Id]; found {
+			if good {
+				if diskInfo.FreeVolumeCount < 2 {
+					// ...
+					vidMap[v.Id] = false
+					noFreeDisk++
+				}
+			}
+		} else {
+			if diskInfo.FreeVolumeCount < 2 {
+				// ...
+				vidMap[v.Id] = false
+				noFreeDisk++
+			} else {
+				// ...
+				vidMap[v.Id] = true
+			}
+		}
+	*/
+	// Yes, if FreeVolumeCount < 2, it marks the volume as bad (false in vidMap).
+	// In the provided topology, FreeVolumeCount is 1 ("free:1"), so it is < 2.
+	// So ALL volumes should be skipped due to insufficient free disk space.
+
+	vids, _ := SelectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionMatcher, nil, quietSeconds, nowUnixSeconds, fullPercentage, verbose)
+
+	assert.Equal(t, 0, len(vids), "Should select 0 volumes because FreeVolumeCount is 1 (less than 2)")
+
+	// Test case 2: If we had enough free space
+	topologyInfo.DataCenterInfos[0].RackInfos[0].DataNodeInfos[0].DiskInfos["hdd"].FreeVolumeCount = 2
+	topologyInfo.DataCenterInfos[0].RackInfos[0].DataNodeInfos[1].DiskInfos["hdd"].FreeVolumeCount = 2
+
+	// Update expected selection based on size and other filters
+	// Size threshold: 95% of 1000MB = 950MB = 996147200 bytes
+	// Volumes >= 950MB:
+	// Vol 10: 1048582008 > 950MB
+	// Vol 11: 1048582008 > 950MB
+	// Vol 12: 1048582008 > 950MB
+	// Vol 13: 570428616 < 950MB
+	// Vol 14: 478153400 < 950MB
+	// ... others are small
+
+	// So expected volumes: 10, 11, 12.
+
+	vids, _ = SelectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionMatcher, nil, quietSeconds, nowUnixSeconds, fullPercentage, verbose)
+
+	expectedVids := []needle.VolumeId{10, 11, 12}
+	assert.Equal(t, len(expectedVids), len(vids), "Should select 3 volumes")
+
+	// Check content
+	vidMap := make(map[needle.VolumeId]bool)
+	for _, vid := range vids {
+		vidMap[vid] = true
+	}
+	for _, vid := range expectedVids {
+		assert.True(t, vidMap[vid], "Volume %d should be selected", vid)
+	}
+}
+
+func TestEcEncodeNodeCountCheck(t *testing.T) {
+	topologyInfo := &master_pb.TopologyInfo{
+		Id: "DefaultDataCenter",
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				Id: "DefaultDataCenter",
+				RackInfos: []*master_pb.RackInfo{
+					{
+						Id: "DefaultRack",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{Id: "node1"},
+							{Id: "node2"},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	nodeCount := 0
+	EachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+		nodeCount++
+	})
+
+	// Default parity shards count is 4
+	minNodeCount := 4
+
+	// Case 1: Without -force
+	forceChanges := false
+	willProceed := forceChanges || nodeCount >= minNodeCount
+	assert.False(t, willProceed, "Should NOT proceed with %d nodes (min %d) without force", nodeCount, minNodeCount)
+
+	// Case 2: With -force
+	forceChanges = true
+	willProceed = forceChanges || nodeCount >= minNodeCount
+	assert.True(t, willProceed, "Should proceed with -force even with %d nodes", nodeCount)
+}
+
+func TestChunkVolumeIds(t *testing.T) {
+	vids := []needle.VolumeId{101, 102, 103, 104, 105}
+
+	assert.Equal(t, [][]needle.VolumeId{
+		{101, 102},
+		{103, 104},
+		{105},
+	}, ChunkVolumeIds(vids, 2))
+
+	assert.Equal(t, [][]needle.VolumeId{vids}, ChunkVolumeIds(vids, 0))
+}
+
+func ecShardVisibilityTestTopology(nodes ...*master_pb.DataNodeInfo) *master_pb.TopologyInfo {
+	return &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{{
+			Id: "dc1",
+			RackInfos: []*master_pb.RackInfo{{
+				Id:            "rack1",
+				DataNodeInfos: nodes,
+			}},
+		}},
+	}
+}
+
+func TestCollectEcShardBitsByNode(t *testing.T) {
+	allBits := uint32(1)<<erasure_coding.TotalShardsCount - 1
+
+	// One node reports the volume's shards split across two disk types; the
+	// other node reports nothing for it.
+	node1 := &master_pb.DataNodeInfo{
+		Id: "node1:8080",
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"": {
+				MaxVolumeCount: 10,
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{Id: 1, EcIndexBits: allBits & 0x00ff, DiskId: 0},
+				},
+			},
+			"ssd": {
+				Type:           "ssd",
+				MaxVolumeCount: 10,
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{Id: 1, EcIndexBits: allBits &^ 0x00ff, DiskId: 1},
+					{Id: 2, EcIndexBits: allBits, DiskId: 1},
+				},
+			},
+		},
+	}
+	node2 := &master_pb.DataNodeInfo{
+		Id:        "node2:8080",
+		DiskInfos: map[string]*master_pb.DiskInfo{"": {MaxVolumeCount: 10}},
+	}
+	topo := ecShardVisibilityTestTopology(node1, node2)
+
+	byNode := CollectEcShardBitsByNode(topo, needle.VolumeId(1))
+	assert.Len(t, byNode, 1)
+	assert.Equal(t, erasure_coding.ShardBits(allBits), byNode[pb.NewServerAddressFromDataNode(node1)])
+
+	assert.Empty(t, CollectEcShardBitsByNode(topo, needle.VolumeId(3)))
+}
+
+func TestCollectEcShardBitsByNode_MixedGenerations(t *testing.T) {
+	allBits := uint32(1)<<erasure_coding.TotalShardsCount - 1
+
+	nodeWithGeneration := func(id string, ts int64) *master_pb.DataNodeInfo {
+		return &master_pb.DataNodeInfo{
+			Id: id,
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"": {
+					MaxVolumeCount: 10,
+					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+						{Id: 1, EcIndexBits: allBits, DiskId: 0, EncodeTsNs: ts},
+					},
+				},
+			},
+		}
+	}
+
+	// A full orphaned older generation on node2 must not count: neither toward
+	// the registration union nor as a second holder.
+	fresh := nodeWithGeneration("node1:8080", 200)
+	orphan := nodeWithGeneration("node2:8080", 100)
+	byNode := CollectEcShardBitsByNode(ecShardVisibilityTestTopology(fresh, orphan), needle.VolumeId(1))
+	assert.Len(t, byNode, 1)
+	assert.Equal(t, erasure_coding.ShardBits(allBits), byNode[pb.NewServerAddressFromDataNode(fresh)])
+
+	// Un-stamped entries are the legacy generation zero: dropped when a stamped
+	// generation exists, all counted when nothing is stamped.
+	legacy := nodeWithGeneration("node2:8080", 0)
+	byNode = CollectEcShardBitsByNode(ecShardVisibilityTestTopology(fresh, legacy), needle.VolumeId(1))
+	assert.Len(t, byNode, 1)
+	assert.Equal(t, erasure_coding.ShardBits(allBits), byNode[pb.NewServerAddressFromDataNode(fresh)])
+
+	byNode = CollectEcShardBitsByNode(
+		ecShardVisibilityTestTopology(nodeWithGeneration("node1:8080", 0), nodeWithGeneration("node2:8080", 0)),
+		needle.VolumeId(1))
+	assert.Len(t, byNode, 2)
+}
+
+func TestEcShardsClumpedOnOneNode(t *testing.T) {
+	allBits := uint32(1)<<erasure_coding.TotalShardsCount - 1
+
+	holderNode := func() *master_pb.DataNodeInfo {
+		return &master_pb.DataNodeInfo{
+			Id: "node1:8080",
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"": {
+					MaxVolumeCount: 10,
+					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+						{Id: 1, EcIndexBits: allBits, DiskId: 0},
+					},
+				},
+			},
+		}
+	}
+	emptyNode := func(id string, maxVolumeCount, volumeCount int64) *master_pb.DataNodeInfo {
+		return &master_pb.DataNodeInfo{
+			Id: id,
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"": {MaxVolumeCount: maxVolumeCount, VolumeCount: volumeCount},
+			},
+		}
+	}
+
+	// All shards on one node while another node has free slots: clumped.
+	node1 := holderNode()
+	topo := ecShardVisibilityTestTopology(node1, emptyNode("node2:8080", 10, 0))
+	holder, clumped := ecShardsClumpedOnOneNode(topo, needle.VolumeId(1), types.HardDriveType)
+	assert.True(t, clumped)
+	assert.Equal(t, pb.NewServerAddressFromDataNode(node1), holder)
+
+	// The only other node has no free slots: the balance could not have spread
+	// the shards, so deletion may proceed.
+	topo = ecShardVisibilityTestTopology(holderNode(), emptyNode("node2:8080", 1, 1))
+	_, clumped = ecShardsClumpedOnOneNode(topo, needle.VolumeId(1), types.HardDriveType)
+	assert.False(t, clumped)
+
+	// Single-node cluster: not a clump.
+	topo = ecShardVisibilityTestTopology(holderNode())
+	_, clumped = ecShardsClumpedOnOneNode(topo, needle.VolumeId(1), types.HardDriveType)
+	assert.False(t, clumped)
+
+	// Shards spread across two nodes: not a clump.
+	spread1 := holderNode()
+	spread1.DiskInfos[""].EcShardInfos[0].EcIndexBits = allBits & 0x007f
+	spread2 := emptyNode("node2:8080", 10, 0)
+	spread2.DiskInfos[""].EcShardInfos = []*master_pb.VolumeEcShardInformationMessage{
+		{Id: 1, EcIndexBits: allBits &^ 0x007f, DiskId: 0},
+	}
+	topo = ecShardVisibilityTestTopology(spread1, spread2)
+	_, clumped = ecShardsClumpedOnOneNode(topo, needle.VolumeId(1), types.HardDriveType)
+	assert.False(t, clumped)
+
+	// No shards visible at all: the recoverability check owns that case.
+	topo = ecShardVisibilityTestTopology(emptyNode("node1:8080", 10, 0), emptyNode("node2:8080", 10, 0))
+	_, clumped = ecShardsClumpedOnOneNode(topo, needle.VolumeId(1), types.HardDriveType)
+	assert.False(t, clumped)
+
+	// A full orphaned older generation on another node must not pose as a
+	// second holder and defeat the clump detection.
+	fresh := holderNode()
+	fresh.DiskInfos[""].EcShardInfos[0].EncodeTsNs = 200
+	orphan := emptyNode("node2:8080", 10, 0)
+	orphan.DiskInfos[""].EcShardInfos = []*master_pb.VolumeEcShardInformationMessage{
+		{Id: 1, EcIndexBits: allBits, DiskId: 0, EncodeTsNs: 100},
+	}
+	topo = ecShardVisibilityTestTopology(fresh, orphan, emptyNode("node3:8080", 10, 0))
+	holder, clumped = ecShardsClumpedOnOneNode(topo, needle.VolumeId(1), types.HardDriveType)
+	assert.True(t, clumped)
+	assert.Equal(t, pb.NewServerAddressFromDataNode(fresh), holder)
+}
+
+// A volume that lived on ssd gets its shards written beside it, on ssd, while
+// -diskType still defaults to hdd. The pre-delete check must count them anyway:
+// scoping the count to the hdd bucket saw a complete set as zero shards and
+// aborted the encode, leaving the volume as both a .dat and a full shard set.
+func TestEcShardCountIgnoresDiskTypeOfTheShards(t *testing.T) {
+	allBits := uint32(1)<<erasure_coding.TotalShardsCount - 1
+	node := &master_pb.DataNodeInfo{
+		Id: "node1:8080",
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"":    {MaxVolumeCount: 10},
+			"ssd": {Type: "ssd", MaxVolumeCount: 10, EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{{Id: 1, EcIndexBits: allBits, DiskId: 1}}},
+		},
+	}
+	topo := ecShardVisibilityTestTopology(node)
+
+	var union erasure_coding.ShardBits
+	for _, bits := range CollectEcShardBitsByNode(topo, needle.VolumeId(1)) {
+		union |= bits
+	}
+	assert.Equal(t, erasure_coding.TotalShardsCount, union.Count(),
+		"shards on a non-default medium must still be counted")
+
+	degraded, err := erasure_coding.RequireRecoverableShardSet(
+		1, union, erasure_coding.DataShardsCount, erasure_coding.TotalShardsCount)
+	assert.NoError(t, err, "a complete shard set must not read as unrecoverable")
+	assert.False(t, degraded)
+
+	// The decode's node view unions across buckets for the same reason: a
+	// decode scoped to the hdd bucket would report this decodable volume as
+	// having no shards at all.
+	byNode, _ := collectEcNodeShardsInfo(topo, needle.VolumeId(1))
+	assert.Len(t, byNode, 1, "decode discovery must see shards on a non-default medium")
+	for _, si := range byNode {
+		assert.Equal(t, erasure_coding.TotalShardsCount, si.Count(),
+			"decode discovery must include every shard on the non-default medium")
+	}
+}
+
+// The message an aborted deletion leaves behind is all the operator has to go
+// on, so it has to name which shards were found and not only how many: a set
+// holding 0-9 and one holding 4-13 are both "10 shards", and only the ids say
+// whether what survived can rebuild the volume.
+func TestEcShardSummaryNamesTheShardIds(t *testing.T) {
+	byNode := map[pb.ServerAddress]erasure_coding.ShardBits{
+		"node2:8080": erasure_coding.ShardBits(0).Set(1).Set(2),
+		"node1:8080": erasure_coding.ShardBits(0).Set(0).Set(12),
+	}
+
+	assert.Equal(t, []string{
+		"node1:8080=2 shards [0 12]",
+		"node2:8080=2 shards [1 2]",
+	}, ecShardSummaryByNode(byNode))
+}
+
+// A retried ec.encode clears the leftover shards of an interrupted run before
+// the source health check, but the topology snapshot predates that sweep: the
+// leftovers still depress FreeVolumeCount on the source disk. The counts must
+// refund those slots, or the retry refuses the very volume it just cleaned up
+// after. Numbers mirror the interruption-matrix failure: 8 slots, 4 volumes,
+// 8 shards of other EC volumes, plus 14 leftover shards of volume 3, so the
+// master charges ceil(22/10)=3 slots and reports free=1.
+func TestCollectSourceFreeVolumeCountsRefundsClearedShards(t *testing.T) {
+	newTopo := func() *master_pb.TopologyInfo {
+		return &master_pb.TopologyInfo{
+			DataCenterInfos: []*master_pb.DataCenterInfo{{
+				Id: "dc1",
+				RackInfos: []*master_pb.RackInfo{{
+					Id: "rack0",
+					DataNodeInfos: []*master_pb.DataNodeInfo{{
+						Id:      "127.0.0.1:8110",
+						Address: "127.0.0.1:8110",
+						DiskInfos: map[string]*master_pb.DiskInfo{
+							"hdd": {
+								Type:            "hdd",
+								MaxVolumeCount:  8,
+								VolumeCount:     4,
+								FreeVolumeCount: 1,
+								VolumeInfos: []*master_pb.VolumeInformationMessage{
+									{Id: 3}, {Id: 6},
+								},
+								EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+									{Id: 1, EcIndexBits: 0b00000010001, DiskId: 0}, // shards 0,4
+									{Id: 1, EcIndexBits: 0b10000000100, DiskId: 1}, // shards 2,10
+									{Id: 2, EcIndexBits: 0b00000010001, DiskId: 0},
+									{Id: 2, EcIndexBits: 0b10000000100, DiskId: 1},
+									{Id: 3, EcIndexBits: 0b11111111111111, DiskId: 0}, // interrupted run's leftovers
+								},
+							},
+						},
+					}},
+				}},
+			}},
+		}
+	}
+
+	counts := collectSourceFreeVolumeCounts(newTopo(), []needle.VolumeId{3}, nil)
+	assert.Equal(t, 3, counts["3-127.0.0.1:8110"],
+		"the cleared leftovers' ceil(22/10)-ceil(8/10)=2 slots must be refunded")
+
+	counts = collectSourceFreeVolumeCounts(newTopo(), []needle.VolumeId{6}, nil)
+	assert.Equal(t, 1, counts["6-127.0.0.1:8110"],
+		"no refund when the encoded volume has no leftover shards")
+
+	skipped := map[pb.ServerAddress]struct{}{"127.0.0.1:8110": {}}
+	counts = collectSourceFreeVolumeCounts(newTopo(), []needle.VolumeId{3}, skipped)
+	assert.Equal(t, 1, counts["3-127.0.0.1:8110"],
+		"no refund on a node the sweep skipped as unreachable")
+}

weed/ec/ec_flags_test.go

@@ -0,0 +1,51 @@
+package ec
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/stretchr/testify/assert"
+)
+
+// countFreeShardSlots must report zero for a physically near-full disk even
+// though its slot math (MaxVolumeCount - VolumeCount) says it has room, so EC
+// placement never piles shards onto a full disk. Mirrors the volume #10160 gate.
+func TestCountFreeShardSlotsPhysicalDiskGate(t *testing.T) {
+	const gb = uint64(1) << 30
+
+	mk := func(total, free uint64) *master_pb.DataNodeInfo {
+		return &master_pb.DataNodeInfo{
+			Id: "n1",
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"": {MaxVolumeCount: 100, VolumeCount: 0, DiskTotalBytes: total, DiskFreeBytes: free},
+			},
+		}
+	}
+
+	// Physically empty: slot math applies, positive free slots.
+	if got := CountFreeShardSlots(mk(1000*gb, 900*gb), types.HardDriveType); got <= 0 {
+		t.Errorf("physically empty disk free shard slots = %d, want > 0", got)
+	}
+	// Physically 96% full: gated to zero regardless of slot room.
+	if got := CountFreeShardSlots(mk(1000*gb, 40*gb), types.HardDriveType); got != 0 {
+		t.Errorf("physically full disk free shard slots = %d, want 0", got)
+	}
+	// No byte report (older server): fall back to slot math, positive.
+	if got := CountFreeShardSlots(mk(0, 0), types.HardDriveType); got <= 0 {
+		t.Errorf("unreported-bytes disk free shard slots = %d, want > 0 (slot fallback)", got)
+	}
+}
+
+func TestParseVolumeIdsFlag(t *testing.T) {
+	vids, err := ParseVolumeIdsFlag("101, 102,101, 103")
+	assert.NoError(t, err)
+	assert.Equal(t, []needle.VolumeId{101, 102, 103}, vids)
+
+	_, err = ParseVolumeIdsFlag("101,abc")
+	assert.Error(t, err)
+
+	_, err = ParseVolumeIdsFlag(" , ")
+	assert.Error(t, err)
+}

weed/ec/ec_lifecycle_model_test.go

@@ -0,0 +1,520 @@
+package ec
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"testing"
+)
+
+// Bounded-exhaustive model check of the EC volume lifecycle.
+//
+// The randomized chaos harness samples the state space; this test enumerates
+// it. The lifecycle of one volume is modeled as a state machine whose
+// transitions mirror, step by step, the real pipelines in this package, and
+// the checker explores EVERY schedule within the bound: every operation, a
+// crash (kill, no cleanup) at every step boundary, an error return (which
+// runs the rollback, itself crashable at every step) at every step boundary,
+// a volume-server restart (running the startup reconciliation rules) in every
+// quiescent state, and every recovery re-run from every crashed state.
+//
+// Two safety invariants are asserted in every reachable state, including
+// mid-operation and post-crash:
+//
+//	I1 durability: the volume's data is always recoverable — the regular
+//	   replica exists, or some single generation holds at least dataShards
+//	   shard files.
+//	I2 single serving generation: at most one generation is ever mounted.
+//	I3 single generation on disk: the sweep-before-generate discipline keeps
+//	   at most one generation's files on disk through every schedule. (An
+//	   externally planted file breaks this by construction; that path is the
+//	   chaos harness's, not the orchestration's.)
+//
+// And one convergence property: from every quiescent state, the prescribed
+// recovery — re-running the interrupted operation, per the restart-not-resume
+// model — terminates in a clean state (a writable regular volume with no EC
+// leftovers, or exactly one complete mounted generation with no source
+// replica and no surplus copies).
+//
+// The model's fidelity contract: each step below names the code it stands
+// for. When the pipeline order changes, this file must change with it.
+//
+//	encode      = ProcessEcEncodeBatch/doEcEncode: markReadonly
+//	              (markVolumeReplicaWritable false) → sweep
+//	              (clearPreexistingEcShards, removes every prior generation) →
+//	              generate (VolumeEcShardsGenerate, a new generation's files) →
+//	              mount (VolumeEcShardsMount, registers it) → balance
+//	              (EcBalance; a crash mid-move leaves a surplus copy, never a
+//	              loss, because moves copy before they delete) → verify
+//	              (verifyEcShardsBeforeDelete) → deleteSource
+//	              (doDeleteVolumesWithLocations, the commit point).
+//	rollback    = rollbackFailedEcEncode, run only on an error return before
+//	              the commit: rollbackSweep (clearPreexistingEcShards again) →
+//	              restoreWritable. A kill runs nothing.
+//	decode      = DoEcDecode: collect (collectEcShards) → generateVolume
+//	              (VolumeEcShardsToVolume) → mountVolume (VolumeMount) →
+//	              verifyDecoded (verifyDecodedVolumeBeforeDelete) → three
+//	              per-location shard deletions (unmountAndDeleteEcShards).
+//	              Forward-only: no rollback.
+//	reconcile   = the volume server startup rules (weed/storage
+//	              disk_location_ec.go): with the source .dat present, a
+//	              partial generation fails validation and is deleted
+//	              (handleFoundEcxFile/validateEcVolume); surviving shard files
+//	              with their sidecars re-register (the #9212 adoption).
+//	dedup       = the balance's dedup phase removing surplus copies
+//	              (ecbalancer "dedup" moves + verifyEcShardOnKeepNode).
+//
+// Deliberately outside the model: multi-replica sources (the pipeline syncs
+// and reduces to one best replica before encoding), per-node shard placement
+// (I1/I2 do not depend on it), and externally planted files (the chaos
+// harness covers adoption of foreign shards).
+
+const (
+	modelDataShards  = 10
+	modelTotalShards = 14
+)
+
+type modelGen struct {
+	files   int  // shard files on disk, whether or not registered
+	mounted bool // registered and serving
+	surplus int  // duplicate copies left by an interrupted balance move
+}
+
+type modelState struct {
+	replica         bool
+	replicaReadonly bool
+	gens            []modelGen // oldest first; a new encode appends
+}
+
+func (s modelState) clone() modelState {
+	c := s
+	c.gens = append([]modelGen(nil), s.gens...)
+	return c
+}
+
+func (s modelState) key() string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "r%v,ro%v", s.replica, s.replicaReadonly)
+	for _, g := range s.gens {
+		fmt.Fprintf(&b, "|f%d,m%v,s%d", g.files, g.mounted, g.surplus)
+	}
+	return b.String()
+}
+
+// dropEmptyGens removes generations with no files left.
+func (s *modelState) dropEmptyGens() {
+	kept := s.gens[:0]
+	for _, g := range s.gens {
+		if g.files > 0 {
+			kept = append(kept, g)
+		}
+	}
+	s.gens = kept
+}
+
+// ── invariants ──────────────────────────────────────────────────────────────
+
+func checkInvariants(s modelState) error {
+	// I1: durability.
+	recoverable := s.replica
+	for _, g := range s.gens {
+		if g.files >= modelDataShards {
+			recoverable = true
+		}
+	}
+	if !recoverable {
+		return fmt.Errorf("I1 violated: no replica and no generation with >= %d shard files", modelDataShards)
+	}
+	// I2: at most one mounted generation.
+	mounted := 0
+	for _, g := range s.gens {
+		if g.mounted {
+			mounted++
+		}
+	}
+	if mounted > 1 {
+		return fmt.Errorf("I2 violated: %d generations mounted", mounted)
+	}
+	// I3: at most one generation's files on disk.
+	withFiles := 0
+	for _, g := range s.gens {
+		if g.files > 0 {
+			withFiles++
+		}
+	}
+	if withFiles > 1 {
+		return fmt.Errorf("I3 violated: %d generations have files on disk", withFiles)
+	}
+	return nil
+}
+
+func isCleanEc(s modelState) bool {
+	if s.replica || len(s.gens) != 1 {
+		return false
+	}
+	g := s.gens[0]
+	return g.files == modelTotalShards && g.mounted && g.surplus == 0
+}
+
+// ── operations as step lists ────────────────────────────────────────────────
+
+// A step mutates the state; ok=false means the step's precondition failed
+// (the real command would return an error at this point).
+type modelStep struct {
+	name  string
+	apply func(*modelState) (ok bool)
+}
+
+func encodeSteps() []modelStep {
+	return []modelStep{
+		{"markReadonly", func(s *modelState) bool {
+			if !s.replica {
+				return false
+			}
+			s.replicaReadonly = true
+			return true
+		}},
+		// The orphan sweep tears down per (node, volume); a crash can land
+		// between nodes, leaving a partially removed generation.
+		{"sweepLoc0", func(s *modelState) bool { sweepLocation(s, 0); return true }},
+		{"sweepLoc1", func(s *modelState) bool { sweepLocation(s, 1); return true }},
+		{"sweepLoc2", func(s *modelState) bool { sweepLocation(s, 2); return true }},
+		// Generation writes the shard files on the generation host; a crash
+		// mid-write leaves a partial, unregistered set beside the .dat, which
+		// the startup validation removes and the next sweep also covers.
+		{"generateHalf", func(s *modelState) bool {
+			if !s.replica {
+				return false
+			}
+			s.gens = append(s.gens, modelGen{files: modelTotalShards / 2})
+			return true
+		}},
+		{"generateRest", func(s *modelState) bool {
+			s.gens[len(s.gens)-1].files = modelTotalShards
+			return true
+		}},
+		{"mount", func(s *modelState) bool {
+			s.gens[len(s.gens)-1].mounted = true
+			return true
+		}},
+		// balance: a completed run changes placement only. Its crash variant
+		// is modeled by crashing between "balanceCopy" and "balanceDelete":
+		// the copy landed, the source deletion did not.
+		{"balanceCopy", func(s *modelState) bool {
+			s.gens[len(s.gens)-1].surplus++
+			return true
+		}},
+		{"balanceDelete", func(s *modelState) bool {
+			s.gens[len(s.gens)-1].surplus--
+			return true
+		}},
+		{"verify", func(s *modelState) bool {
+			return s.gens[len(s.gens)-1].files >= modelDataShards
+		}},
+		{"deleteSource", func(s *modelState) bool {
+			s.replica = false
+			s.replicaReadonly = false
+			return true
+		}},
+	}
+}
+
+// encodeCommitIndex is the index of deleteSource: an error return at or after
+// it must not roll back (the shards are the only copy once it runs).
+func encodeCommitIndex() int { return len(encodeSteps()) - 1 }
+
+// sweepLocation removes one location's share of every generation's files —
+// the per-(node, volume) teardown of clearPreexistingEcShards. The last
+// location clears the remainder, sidecars included.
+func sweepLocation(s *modelState, loc int) {
+	for gi := range s.gens {
+		g := &s.gens[gi]
+		chunk := modelTotalShards / 3
+		if loc == 2 {
+			g.files, g.mounted, g.surplus = 0, false, 0
+		} else if g.files > chunk {
+			g.files -= chunk
+		} else {
+			g.files = 0
+		}
+	}
+	s.dropEmptyGens()
+}
+
+func rollbackSteps() []modelStep {
+	return []modelStep{
+		{"rollbackSweep", func(s *modelState) bool {
+			s.gens = nil
+			return true
+		}},
+		{"restoreWritable", func(s *modelState) bool {
+			if s.replica {
+				s.replicaReadonly = false
+			}
+			return true
+		}},
+	}
+}
+
+func decodeSteps() []modelStep {
+	steps := []modelStep{
+		{"collect", func(s *modelState) bool {
+			for _, g := range s.gens {
+				if g.mounted && g.files >= modelDataShards {
+					return true
+				}
+			}
+			return false
+		}},
+		{"generateVolume", func(s *modelState) bool {
+			s.replica = true
+			return true
+		}},
+		{"mountVolume", func(s *modelState) bool { return true }},
+		{"verifyDecoded", func(s *modelState) bool { return s.replica }},
+	}
+	// Shard deletion is per holder location; three locations model a spread
+	// volume, so a crash can land between any two of them.
+	for i := 0; i < 3; i++ {
+		i := i
+		steps = append(steps, modelStep{fmt.Sprintf("deleteShardsLoc%d", i), func(s *modelState) bool {
+			for gi := range s.gens {
+				g := &s.gens[gi]
+				if !g.mounted && g.files == 0 {
+					continue
+				}
+				// Each location holds roughly a third of the shards; the last
+				// pass clears the remainder and the generation with it.
+				chunk := modelTotalShards / 3
+				if i == 2 {
+					g.files, g.surplus, g.mounted = 0, 0, false
+				} else if g.files > chunk {
+					g.files -= chunk
+				}
+			}
+			s.dropEmptyGens()
+			return true
+		}})
+	}
+	return steps
+}
+
+// reconcile applies the volume-server startup rules after a crash-restart.
+func reconcile(s *modelState) {
+	for gi := range s.gens {
+		g := &s.gens[gi]
+		if s.replica && g.files < modelTotalShards {
+			// .dat present and the local generation is partial: validation
+			// fails and the files are removed (disk_location_ec.go).
+			g.files, g.mounted, g.surplus = 0, false, 0
+			continue
+		}
+		if g.files > 0 {
+			// Surviving files re-register through their sidecars (#9212).
+			g.mounted = true
+		}
+	}
+	s.dropEmptyGens()
+}
+
+// dedupSurplus is the balance dedup phase.
+func dedupSurplus(s *modelState) {
+	for gi := range s.gens {
+		s.gens[gi].surplus = 0
+	}
+}
+
+// ── recovery convergence ────────────────────────────────────────────────────
+
+// runToCompletion applies steps without interruption; false when a
+// precondition fails (the real re-run would error out).
+func runToCompletion(s *modelState, steps []modelStep) bool {
+	for _, st := range steps {
+		if !st.apply(s) {
+			return false
+		}
+	}
+	return true
+}
+
+// recover applies the prescribed recovery for a quiescent state and reports
+// whether it converges to a clean state. Per the restart-not-resume model:
+// a surviving replica means the encode restarts from scratch; no replica
+// means the encode had committed and only surplus cleanup can remain; a
+// mounted generation beside a replica can also be decoded away.
+func recoverModel(s modelState) (modelState, error) {
+	// A crash-restart happens before any operator action; its reconciliation
+	// must itself keep the invariants.
+	reconcile(&s)
+	if err := checkInvariants(s); err != nil {
+		return s, fmt.Errorf("after reconcile: %w", err)
+	}
+
+	if !s.replica {
+		// The encode committed: shards are the volume. Dedup cleans surplus.
+		dedupSurplus(&s)
+		if !isCleanEc(s) {
+			return s, fmt.Errorf("no replica and not clean EC")
+		}
+		return s, nil
+	}
+
+	// Replica survives: re-run the encode from scratch (sweep + regenerate),
+	// then the volume is EC; this is the restart path every interrupted
+	// encode and every leftover-orphan state takes.
+	if !runToCompletion(&s, encodeSteps()) {
+		return s, fmt.Errorf("recovery encode did not complete")
+	}
+	dedupSurplus(&s)
+	if err := checkInvariants(s); err != nil {
+		return s, fmt.Errorf("after recovery encode: %w", err)
+	}
+	if !isCleanEc(s) {
+		return s, fmt.Errorf("recovery encode did not converge to clean EC")
+	}
+	// And a decode must be able to bring it back to a regular volume.
+	if !runToCompletion(&s, decodeSteps()) {
+		return s, fmt.Errorf("decode after recovery did not complete")
+	}
+	if err := checkInvariants(s); err != nil {
+		return s, fmt.Errorf("after decode: %w", err)
+	}
+	if !s.replica || len(s.gens) != 0 {
+		return s, fmt.Errorf("decode after recovery did not converge to a bare regular volume")
+	}
+	return s, nil
+}
+
+// ── exhaustive exploration ──────────────────────────────────────────────────
+
+type trace []string
+
+// explore runs every schedule of one operation from state s: after every
+// step, the schedule may continue, crash (kill: stop, nothing else runs), or
+// error out (rollback runs, itself crashable at every step). Every resulting
+// quiescent state is handed to onQuiescent together with the trace that
+// produced it.
+func exploreOp(t *testing.T, s modelState, opName string, steps []modelStep, commitIndex int, withRollback bool, tr trace, onQuiescent func(modelState, trace)) {
+	t.Helper()
+	// Crash before the first step is the same as never starting.
+	for prefix := 1; prefix <= len(steps); prefix++ {
+		run := s.clone()
+		ok := true
+		for i := 0; i < prefix; i++ {
+			if !steps[i].apply(&run) {
+				ok = false
+				break
+			}
+			if err := checkInvariants(run); err != nil {
+				t.Fatalf("%s after %s: %v\ntrace: %v", opName, steps[i].name, err, append(tr, opName+"/"+steps[i].name))
+			}
+		}
+		if !ok {
+			continue // precondition stopped the op; same as an early error
+		}
+		stepTrace := append(append(trace{}, tr...), fmt.Sprintf("%s..%s", opName, steps[prefix-1].name))
+
+		// Kill here: nothing else runs.
+		onQuiescent(run.clone(), append(append(trace{}, stepTrace...), "KILL"))
+
+		// Error return here: the rollback runs, unless the op has committed;
+		// the rollback itself can be killed at each of its step boundaries.
+		if withRollback && prefix <= commitIndex {
+			rb := rollbackSteps()
+			for rbPrefix := 0; rbPrefix <= len(rb); rbPrefix++ {
+				rbRun := run.clone()
+				for i := 0; i < rbPrefix; i++ {
+					rb[i].apply(&rbRun)
+					if err := checkInvariants(rbRun); err != nil {
+						t.Fatalf("%s rollback after %s: %v\ntrace: %v", opName, rb[i].name, err, stepTrace)
+					}
+				}
+				suffix := "FAIL+rollback-killed"
+				if rbPrefix == len(rb) {
+					suffix = "FAIL+rollback-complete"
+				}
+				onQuiescent(rbRun.clone(), append(append(trace{}, stepTrace...), fmt.Sprintf("%s@%d", suffix, rbPrefix)))
+			}
+		}
+	}
+}
+
+// TestECLifecycleModelExhaustive enumerates every crash and failure schedule
+// of encode and decode up to two chained interrupted operations, checks the
+// safety invariants in every intermediate state, and requires the prescribed
+// recovery to converge from every quiescent state.
+func TestECLifecycleModelExhaustive(t *testing.T) {
+	start := modelState{replica: true}
+
+	visited := map[string]trace{}
+	quiescent := 0
+	schedules := 0
+	var enqueue func(s modelState, depth int, tr trace)
+
+	checkRecovery := func(s modelState, tr trace) {
+		schedules++
+		if _, seen := visited["Q"+s.key()]; seen {
+			return
+		}
+		visited["Q"+s.key()] = tr
+		quiescent++
+		if _, err := recoverModel(s.clone()); err != nil {
+			t.Fatalf("recovery does not converge: %v\nstate: %s\ntrace: %v", err, s.key(), tr)
+		}
+	}
+
+	enqueue = func(s modelState, depth int, tr trace) {
+		checkRecovery(s, tr)
+		if depth == 0 {
+			return
+		}
+		// A crash-restart may happen in any quiescent state before the next
+		// operation; explore both with and without it.
+		restarted := s.clone()
+		reconcile(&restarted)
+		if err := checkInvariants(restarted); err != nil {
+			t.Fatalf("reconcile violated invariants: %v\nstate: %s\ntrace: %v", err, s.key(), tr)
+		}
+		for _, variant := range []struct {
+			st  modelState
+			tag string
+		}{{s, ""}, {restarted, "+restart"}} {
+			st, tag := variant.st, variant.tag
+			if st.replica {
+				exploreOp(t, st, "encode"+tag, encodeSteps(), encodeCommitIndex(), true, tr, func(q modelState, qtr trace) {
+					enqueue(q, depth-1, qtr)
+				})
+			}
+			canDecode := false
+			for _, g := range st.gens {
+				if g.mounted && g.files >= modelDataShards {
+					canDecode = true
+				}
+			}
+			if canDecode {
+				exploreOp(t, st, "decode"+tag, decodeSteps(), len(decodeSteps()), false, tr, func(q modelState, qtr trace) {
+					enqueue(q, depth-1, qtr)
+				})
+			}
+		}
+	}
+
+	// Depth 2: an interrupted operation followed by another interrupted
+	// operation, recovery checked at every quiescent point in between and
+	// after. Deeper chains only revisit already-explored states (the state
+	// space is finite and memoized), which the visited count makes apparent.
+	enqueue(start, 2, trace{"start"})
+
+	// Not spec constants — collapse guards: if a refactor of the model or the
+	// explorer accidentally prunes schedules, these counts crater and this
+	// catches it. Update deliberately when the model changes fidelity.
+	if quiescent < 15 || schedules < 500 {
+		t.Fatalf("suspiciously small exploration: %d distinct quiescent states from %d schedules", quiescent, schedules)
+	}
+	keys := make([]string, 0, len(visited))
+	for k := range visited {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	t.Logf("explored %d schedules reaching %d distinct quiescent states, all recoverable", schedules, quiescent)
+}

weed/ec/ec_rebalance_slots_test.go

@@ -0,0 +1,294 @@
+package ec
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// TestECRebalanceWithLimitedSlots tests that EC rebalance handles the scenario
+// where there are limited free slots on volume servers.
+//
+// This is a regression test for the error:
+//
+//	"no free ec shard slots. only 0 left"
+//
+// Scenario (from real usage):
+// - 6 volume servers in 6 racks
+// - Each server has max=10 volume slots
+// - 7 volumes were EC encoded (7 × 14 = 98 EC shards)
+// - All 14 shards per volume are on the original server (not yet distributed)
+//
+// Expected behavior:
+// - The rebalance algorithm should distribute shards across servers
+// - Even if perfect distribution isn't possible, it should do best-effort
+// - Currently fails with "no free ec shard slots" because freeSlots calculation
+//
+//	doesn't account for shards being moved (freed slots on source, used on target)
+func TestECRebalanceWithLimitedSlots(t *testing.T) {
+	// Build a topology matching the problematic scenario:
+	// 6 servers, each with 2+ volumes worth of EC shards (all 14 shards per volume on same server)
+	topology := buildLimitedSlotsTopology()
+
+	// Collect EC nodes from the topology
+	ecNodes, totalFreeEcSlots := CollectEcVolumeServersByDc(topology, "", types.HardDriveType)
+
+	t.Logf("Topology summary:")
+	t.Logf("  Number of EC nodes: %d", len(ecNodes))
+	t.Logf("  Total free EC slots: %d", totalFreeEcSlots)
+
+	// Log per-node details
+	for _, node := range ecNodes {
+		shardCount := 0
+		for _, diskInfo := range node.Info.DiskInfos {
+			for _, ecShard := range diskInfo.EcShardInfos {
+				shardCount += erasure_coding.GetShardCount(ecShard)
+			}
+		}
+		t.Logf("  Node %s (rack %s): %d shards, %d free slots",
+			node.Info.Id, node.Rack, shardCount, node.FreeEcSlot)
+	}
+
+	// Calculate total EC shards
+	totalEcShards := 0
+	for _, node := range ecNodes {
+		for _, diskInfo := range node.Info.DiskInfos {
+			for _, ecShard := range diskInfo.EcShardInfos {
+				totalEcShards += erasure_coding.GetShardCount(ecShard)
+			}
+		}
+	}
+	t.Logf("  Total EC shards: %d", totalEcShards)
+
+	// Document the issue:
+	// With 98 EC shards (7 volumes × 14 shards) on 6 servers with max=10 each,
+	// total capacity is 60 slots. But shards already occupy slots on their current servers.
+	//
+	// The current algorithm calculates free slots as:
+	//   freeSlots = maxVolumeCount - volumeCount - ecShardCount
+	//
+	// If all shards are on their original servers:
+	// - Server A has 28 shards (2 volumes × 14) → may have negative free slots
+	// - This causes totalFreeEcSlots to be 0 or negative
+	//
+	// The EXPECTED improvement:
+	// - Rebalance should recognize that moving a shard FREES a slot on the source
+	// - The algorithm should work iteratively, moving shards one at a time
+	// - Even if starting with 0 free slots, moving one shard opens a slot
+
+	if totalFreeEcSlots < 1 {
+		// This is the current (buggy) behavior we're documenting
+		t.Logf("")
+		t.Logf("KNOWN ISSUE: totalFreeEcSlots = %d (< 1)", totalFreeEcSlots)
+		t.Logf("")
+		t.Logf("This triggers the error: 'no free ec shard slots. only %d left'", totalFreeEcSlots)
+		t.Logf("")
+		t.Logf("Analysis:")
+		t.Logf("  - %d EC shards across %d servers", totalEcShards, len(ecNodes))
+		t.Logf("  - Shards are concentrated on original servers (not distributed)")
+		t.Logf("  - Current slot calculation doesn't account for slots freed by moving shards")
+		t.Logf("")
+		t.Logf("Expected fix:")
+		t.Logf("  1. Rebalance should work iteratively, moving one shard at a time")
+		t.Logf("  2. Moving a shard from A to B: frees 1 slot on A, uses 1 slot on B")
+		t.Logf("  3. The 'free slots' check should be per-move, not global")
+		t.Logf("  4. Or: calculate 'redistributable slots' = total capacity - shards that must stay")
+
+		// For now, document this is a known issue - don't fail the test
+		// When the fix is implemented, this test should be updated to verify the fix works
+		return
+	}
+
+	// If we get here, the issue might have been fixed
+	t.Logf("totalFreeEcSlots = %d, rebalance should be possible", totalFreeEcSlots)
+}
+
+// TestECRebalanceZeroFreeSlots tests the specific scenario where
+// the topology appears to have free slots but rebalance fails.
+//
+// This can happen when the VolumeCount in the topology includes the original
+// volumes that were EC-encoded, making the free slot calculation incorrect.
+func TestECRebalanceZeroFreeSlots(t *testing.T) {
+	// Build a topology where volumes were NOT deleted after EC encoding
+	// (VolumeCount still reflects the original volumes)
+	topology := buildZeroFreeSlotTopology()
+
+	ecNodes, totalFreeEcSlots := CollectEcVolumeServersByDc(topology, "", types.HardDriveType)
+
+	t.Logf("Zero free slots scenario:")
+	for _, node := range ecNodes {
+		shardCount := 0
+		for _, diskInfo := range node.Info.DiskInfos {
+			for _, ecShard := range diskInfo.EcShardInfos {
+				shardCount += erasure_coding.GetShardCount(ecShard)
+			}
+		}
+		t.Logf("  Node %s: %d shards, %d free slots, volumeCount=%d, max=%d",
+			node.Info.Id, shardCount, node.FreeEcSlot,
+			node.Info.DiskInfos[string(types.HardDriveType)].VolumeCount,
+			node.Info.DiskInfos[string(types.HardDriveType)].MaxVolumeCount)
+	}
+	t.Logf("  Total free slots: %d", totalFreeEcSlots)
+
+	if totalFreeEcSlots == 0 {
+		t.Logf("")
+		t.Logf("SCENARIO REPRODUCED: totalFreeEcSlots = 0")
+		t.Logf("This would trigger: 'no free ec shard slots. only 0 left'")
+	}
+}
+
+// buildZeroFreeSlotTopology creates a topology where rebalance will fail
+// because servers are at capacity (volumeCount equals maxVolumeCount)
+func buildZeroFreeSlotTopology() *master_pb.TopologyInfo {
+	diskTypeKey := string(types.HardDriveType)
+
+	// Each server has max=10, volumeCount=10 (full capacity)
+	// Free capacity = (10-10) * 10 = 0 per server
+	// This will trigger "no free ec shard slots" error
+	return &master_pb.TopologyInfo{
+		Id: "test_zero_free_slots",
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				Id: "dc1",
+				RackInfos: []*master_pb.RackInfo{
+					{
+						Id: "rack0",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								Id: "127.0.0.1:8080",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									diskTypeKey: {
+										Type:           diskTypeKey,
+										MaxVolumeCount: 10,
+										VolumeCount:    10, // At full capacity
+										EcShardInfos:   buildEcShards([]uint32{3, 4}),
+									},
+								},
+							},
+						},
+					},
+					{
+						Id: "rack1",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								Id: "127.0.0.1:8081",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									diskTypeKey: {
+										Type:           diskTypeKey,
+										MaxVolumeCount: 10,
+										VolumeCount:    10,
+										EcShardInfos:   buildEcShards([]uint32{1, 7}),
+									},
+								},
+							},
+						},
+					},
+					{
+						Id: "rack2",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								Id: "127.0.0.1:8082",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									diskTypeKey: {
+										Type:           diskTypeKey,
+										MaxVolumeCount: 10,
+										VolumeCount:    10,
+										EcShardInfos:   buildEcShards([]uint32{2}),
+									},
+								},
+							},
+						},
+					},
+					{
+						Id: "rack3",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								Id: "127.0.0.1:8083",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									diskTypeKey: {
+										Type:           diskTypeKey,
+										MaxVolumeCount: 10,
+										VolumeCount:    10,
+										EcShardInfos:   buildEcShards([]uint32{5, 6}),
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func buildEcShards(volumeIds []uint32) []*master_pb.VolumeEcShardInformationMessage {
+	var shards []*master_pb.VolumeEcShardInformationMessage
+	for _, vid := range volumeIds {
+		si := erasure_coding.NewShardsInfo()
+		for _, id := range erasure_coding.AllShardIds() {
+			si.Set(erasure_coding.NewShardInfo(id, 1234))
+		}
+		shards = append(shards, &master_pb.VolumeEcShardInformationMessage{
+			Id:          vid,
+			Collection:  "ectest",
+			EcIndexBits: si.Bitmap(),
+			ShardSizes:  si.SizesInt64(),
+		})
+	}
+	return shards
+}
+
+// buildLimitedSlotsTopology creates a topology matching the problematic scenario:
+// - 6 servers in 6 racks
+// - Each server has max=10 volume slots
+// - 7 volumes were EC encoded, shards distributed as follows:
+//   - rack0 (8080): volumes 3,4 → 28 shards
+//   - rack1 (8081): volumes 1,7 → 28 shards
+//   - rack2 (8082): volume 2 → 14 shards
+//   - rack3 (8083): volumes 5,6 → 28 shards
+//   - rack4 (8084): (no volumes originally)
+//   - rack5 (8085): (no volumes originally)
+func buildLimitedSlotsTopology() *master_pb.TopologyInfo {
+	return &master_pb.TopologyInfo{
+		Id: "test_limited_slots",
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				Id: "dc1",
+				RackInfos: []*master_pb.RackInfo{
+					buildRackWithEcShards("rack0", "127.0.0.1:8080", 10, []uint32{3, 4}),
+					buildRackWithEcShards("rack1", "127.0.0.1:8081", 10, []uint32{1, 7}),
+					buildRackWithEcShards("rack2", "127.0.0.1:8082", 10, []uint32{2}),
+					buildRackWithEcShards("rack3", "127.0.0.1:8083", 10, []uint32{5, 6}),
+					buildRackWithEcShards("rack4", "127.0.0.1:8084", 10, []uint32{}),
+					buildRackWithEcShards("rack5", "127.0.0.1:8085", 10, []uint32{}),
+				},
+			},
+		},
+	}
+}
+
+// buildRackWithEcShards creates a rack with one data node containing EC shards
+// for the specified volume IDs (all 14 shards per volume)
+func buildRackWithEcShards(rackId, nodeId string, maxVolumes int64, volumeIds []uint32) *master_pb.RackInfo {
+	// Note: types.HardDriveType is "" (empty string), so we use "" as the key
+	diskTypeKey := string(types.HardDriveType)
+
+	return &master_pb.RackInfo{
+		Id: rackId,
+		DataNodeInfos: []*master_pb.DataNodeInfo{
+			{
+				Id: nodeId,
+				DiskInfos: map[string]*master_pb.DiskInfo{
+					diskTypeKey: {
+						Type:           diskTypeKey,
+						MaxVolumeCount: maxVolumes,
+						VolumeCount:    int64(len(volumeIds)), // Original volumes still counted
+						EcShardInfos:   buildEcShards(volumeIds),
+					},
+				},
+			},
+		},
+	}
+}

weed/ec/ec_rebuild.go

@@ -0,0 +1,435 @@
+package ec
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"slices"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type ecRebuilder struct {
+	env          *Env
+	ecNodes      []*EcNode
+	writer       io.Writer
+	applyChanges bool
+	collections  []string
+	volumeIds    []needle.VolumeId
+	diskType     types.DiskType
+
+	ewg       *util.ErrorWaitGroup
+	ecNodesMu sync.Mutex
+}
+
+// RebuildEcVolumes finds and rebuilds missing EC shards for the given
+// collections. Before rebuilding it asks volume servers to recover any shards
+// left unmounted by a missing .ecx index; such shards are invisible to the
+// master, so recovering them first avoids regenerating data that is actually
+// present. volumeIds, when non-empty, restricts the rebuild to those volumes.
+func RebuildEcVolumes(env *Env, ecNodes []*EcNode, writer io.Writer, collections []string, volumeIds []needle.VolumeId, diskType types.DiskType, maxParallelization int, applyChanges bool) error {
+	erb := &ecRebuilder{
+		env:          env,
+		ecNodes:      ecNodes,
+		writer:       writer,
+		applyChanges: applyChanges,
+		collections:  collections,
+		volumeIds:    volumeIds,
+		diskType:     diskType,
+
+		ewg: util.NewErrorWaitGroup(maxParallelization),
+	}
+
+	// Recover shards left unmounted by a missing .ecx index before planning: such
+	// shards never register with the master, so the rebuild below would treat the
+	// volume as short or unrepairable even though its data is intact (issue #10104).
+	erb.recoverMissingIndexes()
+
+	fmt.Printf("rebuildEcVolumes for %d collection(s)\n", len(collections))
+	for _, c := range collections {
+		erb.rebuildEcVolumes(c)
+	}
+
+	return erb.ewg.Wait()
+}
+
+func (erb *ecRebuilder) write(format string, a ...any) {
+	fmt.Fprintf(erb.writer, format, a...)
+}
+
+func (erb *ecRebuilder) isLocked() bool {
+	return erb.env.isLocked()
+}
+
+// matchesVolumeId verifies whether the rebuilder is targeted at a given volume ID.
+func (erb *ecRebuilder) matchesVolumeId(vid needle.VolumeId) bool {
+	if len(erb.volumeIds) == 0 {
+		return true
+	}
+
+	return slices.Contains(erb.volumeIds, vid)
+}
+
+// countLocalShards returns the number of shards already present locally on the node for the given volume.
+// Unions across all of the node's disks, like prepareDataToRecover, so slot
+// accounting matches what the rebuild will actually treat as local.
+func (erb *ecRebuilder) countLocalShards(node *EcNode, collection string, volumeId needle.VolumeId) int {
+	localShardsInfo := erasure_coding.NewShardsInfo()
+	for _, diskInfo := range node.Info.DiskInfos {
+		for _, ecShardInfo := range diskInfo.EcShardInfos {
+			if ecShardInfo.Collection == collection && needle.VolumeId(ecShardInfo.Id) == volumeId {
+				localShardsInfo.Add(erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo))
+			}
+		}
+	}
+	return localShardsInfo.Count()
+}
+
+// selectAndReserveRebuilder atomically selects a rebuilder node with sufficient free slots
+// and reserves slots only for the non-local shards that need to be copied/generated.
+func (erb *ecRebuilder) selectAndReserveRebuilder(collection string, volumeId needle.VolumeId) (*EcNode, int, error) {
+	erb.ecNodesMu.Lock()
+	defer erb.ecNodesMu.Unlock()
+
+	if len(erb.ecNodes) == 0 {
+		return nil, 0, fmt.Errorf("no ec nodes available")
+	}
+
+	// Find the node with the most free slots, considering local shards
+	var bestNode *EcNode
+	var bestSlotsNeeded int
+	var maxAvailableSlots int
+	var minSlotsNeeded int = erasure_coding.TotalShardsCount // Start with maximum possible
+	for _, node := range erb.ecNodes {
+		localShards := erb.countLocalShards(node, collection, volumeId)
+		slotsNeeded := erasure_coding.TotalShardsCount - localShards
+		if slotsNeeded < 0 {
+			slotsNeeded = 0
+		}
+
+		if node.FreeEcSlot > maxAvailableSlots {
+			maxAvailableSlots = node.FreeEcSlot
+		}
+
+		if slotsNeeded < minSlotsNeeded {
+			minSlotsNeeded = slotsNeeded
+		}
+
+		if node.FreeEcSlot >= slotsNeeded {
+			if bestNode == nil || node.FreeEcSlot > bestNode.FreeEcSlot {
+				bestNode = node
+				bestSlotsNeeded = slotsNeeded
+			}
+		}
+	}
+
+	if bestNode == nil {
+		return nil, 0, fmt.Errorf("no node has sufficient free slots for volume %d (need at least %d slots, max available: %d)",
+			volumeId, minSlotsNeeded, maxAvailableSlots)
+	}
+
+	// Reserve slots only for non-local shards
+	bestNode.FreeEcSlot -= bestSlotsNeeded
+
+	return bestNode, bestSlotsNeeded, nil
+}
+
+// releaseRebuilder releases the reserved slots back to the rebuilder node.
+func (erb *ecRebuilder) releaseRebuilder(node *EcNode, slotsToRelease int) {
+	erb.ecNodesMu.Lock()
+	defer erb.ecNodesMu.Unlock()
+
+	// Release slots by incrementing the free slot count
+	node.FreeEcSlot += slotsToRelease
+}
+
+func (erb *ecRebuilder) rebuildEcVolumes(collection string) {
+	fmt.Printf("rebuildEcVolumes for %q\n", collection)
+
+	// collect vid => each shard locations, similar to ecShardMap in topology.go
+	ecShardMap := make(EcShardMap)
+	erb.ecNodesMu.Lock()
+	for _, ecNode := range erb.ecNodes {
+		ecShardMap.registerEcNode(ecNode, collection)
+	}
+	erb.ecNodesMu.Unlock()
+
+	for vid, locations := range ecShardMap {
+		if !erb.matchesVolumeId(vid) {
+			continue
+		}
+		shardCount := locations.shardCount()
+		if shardCount == erasure_coding.TotalShardsCount {
+			continue
+		}
+		if shardCount < erasure_coding.DataShardsCount {
+			erb.write("ec volume %d is unrepairable with %d shards (need %d), skipping\n", vid, shardCount, erasure_coding.DataShardsCount)
+			continue
+		}
+
+		// Capture variables for closure
+		vid := vid
+		locations := locations
+
+		erb.ewg.Add(func() error {
+			// Select rebuilder and reserve slots atomically per volume
+			rebuilder, slotsToReserve, err := erb.selectAndReserveRebuilder(collection, vid)
+			if err != nil {
+				return fmt.Errorf("failed to select rebuilder for volume %d: %v", vid, err)
+			}
+			defer erb.releaseRebuilder(rebuilder, slotsToReserve)
+
+			return erb.rebuildOneEcVolume(collection, vid, locations, rebuilder)
+		})
+	}
+}
+
+// recoverMissingIndexes asks every ec node to fetch a missing .ecx index from a
+// peer and mount the on-disk shards it could not load on its own. Shards
+// orphaned this way (index only on another server) are absent from the master
+// topology, so without this pass ec.rebuild would regenerate or give up on
+// shards whose data is actually present — and a volume whose every holder lacks
+// the index would not appear in the topology at all. Each node therefore
+// recovers all of its on-disk orphans (volume_id 0); an explicit -volumeIds
+// list narrows that to the requested volumes. On apply it refreshes the topology
+// so the rebuild planning sees the recovered shards (issue #10104).
+func (erb *ecRebuilder) recoverMissingIndexes() {
+	erb.ecNodesMu.Lock()
+	nodes := append([]*EcNode(nil), erb.ecNodes...)
+	erb.ecNodesMu.Unlock()
+	if len(nodes) == 0 {
+		return
+	}
+
+	// volume_id 0 means "recover every orphan on the node"; a -volumeIds list
+	// narrows recovery to those ids (each scanned across collections server-side).
+	vids := erb.volumeIds
+	if len(vids) == 0 {
+		vids = []needle.VolumeId{0}
+	}
+
+	if !erb.applyChanges {
+		erb.write("would ask %d ec node(s) to recover EC shards left unmounted by a missing .ecx index\n", len(nodes))
+		return
+	}
+
+	for _, node := range nodes {
+		for _, vid := range vids {
+			err := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(node.Info), erb.env.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+				_, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
+					VolumeId:            uint32(vid),
+					RecoverMissingIndex: true,
+				})
+				return mountErr
+			})
+			if err != nil {
+				erb.write("%s recover missing index (volume %d): %v\n", node.Info.Id, vid, err)
+			}
+		}
+	}
+
+	// Refresh topology so the rebuild planning sees shards the recovery registered.
+	refreshed, _, err := CollectEcNodes(erb.env, erb.diskType)
+	if err != nil {
+		erb.write("failed to refresh ec nodes after index recovery: %v\n", err)
+		return
+	}
+	erb.ecNodesMu.Lock()
+	erb.ecNodes = refreshed
+	erb.ecNodesMu.Unlock()
+}
+
+func (erb *ecRebuilder) rebuildOneEcVolume(collection string, volumeId needle.VolumeId, locations EcShardLocations, rebuilder *EcNode) error {
+	if !erb.isLocked() {
+		return fmt.Errorf("lock is lost")
+	}
+
+	fmt.Printf("rebuildOneEcVolume %s %d\n", collection, volumeId)
+
+	// collect shard files to rebuilder local disk
+	var generatedShardIds []erasure_coding.ShardId
+	copiedShardIds, _, err := erb.prepareDataToRecover(rebuilder, collection, volumeId, locations)
+	defer func() {
+		// Clean up the working copies this run actually made, even when the
+		// recoverability gate failed after some copies already succeeded:
+		// they are temp files on the rebuilder nothing else reclaims. Dry-run
+		// copies nothing (copiedShardIds is empty), so this issues no delete
+		// RPC. Use a local error so a cleanup failure cannot mask the return.
+		if !erb.applyChanges || len(copiedShardIds) == 0 {
+			return
+		}
+		if derr := SourceServerDeleteEcShards(erb.env.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.Info), copiedShardIds); derr != nil {
+			erb.write("%s delete copied ec shards %s %d.%v: %v\n", rebuilder.Info.Id, collection, volumeId, copiedShardIds, derr)
+		}
+	}()
+	if err != nil {
+		return err
+	}
+
+	if !erb.applyChanges {
+		return nil
+	}
+
+	// generate ec shards, and maybe ecx file
+	generatedShardIds, err = erb.generateMissingShards(collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.Info))
+	if err != nil {
+		return err
+	}
+
+	// mount the generated shards
+	err = MountEcShards(erb.env.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.Info), generatedShardIds)
+	if err != nil {
+		return err
+	}
+
+	// ensure ECNode updates are atomic
+	erb.ecNodesMu.Lock()
+	defer erb.ecNodesMu.Unlock()
+	rebuilder.AddEcVolumeShards(volumeId, collection, generatedShardIds, erb.diskType)
+
+	return nil
+}
+
+func (erb *ecRebuilder) generateMissingShards(collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress) (rebuiltShardIds []erasure_coding.ShardId, err error) {
+
+	err = operation.WithVolumeServerClient(false, sourceLocation, erb.env.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+		resp, rebuildErr := volumeServerClient.VolumeEcShardsRebuild(context.Background(), &volume_server_pb.VolumeEcShardsRebuildRequest{
+			VolumeId:   uint32(volumeId),
+			Collection: collection,
+		})
+		if rebuildErr == nil {
+			rebuiltShardIds = erasure_coding.Uint32ToShardIds(resp.RebuiltShardIds)
+		}
+		return rebuildErr
+	})
+	return
+}
+
+func (erb *ecRebuilder) prepareDataToRecover(rebuilder *EcNode, collection string, volumeId needle.VolumeId, locations EcShardLocations) (copiedShardIds []erasure_coding.ShardId, localShardIds []erasure_coding.ShardId, err error) {
+
+	needEcxFile := true
+	localShardsInfo := erasure_coding.NewShardsInfo()
+	for _, diskInfo := range rebuilder.Info.DiskInfos {
+		for _, ecShardInfo := range diskInfo.EcShardInfos {
+			if ecShardInfo.Collection == collection && needle.VolumeId(ecShardInfo.Id) == volumeId {
+				needEcxFile = false
+				// Union across disks: the rebuilder may hold this volume's
+				// shards on more than one disk. Overwriting per-disk would
+				// make a shard on a non-last disk look remote and get copied
+				// onto itself (O_TRUNC) and then node-wide deleted.
+				localShardsInfo.Add(erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo))
+			}
+		}
+	}
+
+	targetShardCount := erasure_coding.TotalShardsCount
+	for i := erasure_coding.TotalShardsCount; i < len(locations); i++ {
+		if len(locations[i]) > 0 {
+			targetShardCount = i + 1
+		}
+	}
+
+	// recoverableRemoteShards counts remote shards that can contribute to the
+	// rebuild. Dry-run counts the plan; apply mode counts only successful copies.
+	recoverableRemoteShards := 0
+	for i := 0; i < targetShardCount; i++ {
+		ecNodes := locations[i]
+		shardId := erasure_coding.ShardId(i)
+		if len(ecNodes) == 0 {
+			erb.write("missing shard %d.%d\n", volumeId, shardId)
+			continue
+		}
+
+		if localShardsInfo.Has(shardId) {
+			localShardIds = append(localShardIds, shardId)
+			erb.write("use existing shard %d.%d\n", volumeId, shardId)
+			continue
+		}
+
+		// The rebuilder is itself the only listed holder: never copy a shard
+		// onto itself (the in-place O_TRUNC would destroy it) nor schedule it
+		// for the post-rebuild delete. Treat it as already local.
+		if ecNodes[0].Info.Id == rebuilder.Info.Id {
+			localShardIds = append(localShardIds, shardId)
+			erb.write("use existing shard %d.%d (already on rebuilder)\n", volumeId, shardId)
+			continue
+		}
+
+		if !erb.applyChanges {
+			recoverableRemoteShards++
+			erb.write("would copy %d.%d from %s\n", volumeId, shardId, ecNodes[0].Info.Id)
+			continue
+		}
+
+		copyErr := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(rebuilder.Info), erb.env.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+			_, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{
+				VolumeId:       uint32(volumeId),
+				Collection:     collection,
+				ShardIds:       []uint32{uint32(shardId)},
+				CopyEcxFile:    needEcxFile,
+				CopyEcjFile:    true,
+				CopyVifFile:    needEcxFile,
+				SourceDataNode: string(pb.NewServerAddressFromDataNode(ecNodes[0].Info)),
+			})
+			return copyErr
+		})
+		if copyErr != nil {
+			erb.write("%s failed to copy %d.%d from %s: %v\n", rebuilder.Info.Id, volumeId, shardId, ecNodes[0].Info.Id, copyErr)
+			continue
+		}
+		recoverableRemoteShards++
+		if needEcxFile {
+			needEcxFile = false
+		}
+		erb.write("%s copied %d.%d from %s\n", rebuilder.Info.Id, volumeId, shardId, ecNodes[0].Info.Id)
+		// Only shards this run actually copied are temp working files to be
+		// deleted afterward; never a pre-existing local or remote shard.
+		copiedShardIds = append(copiedShardIds, shardId)
+	}
+
+	if len(localShardIds)+recoverableRemoteShards >= erasure_coding.DataShardsCount {
+		return copiedShardIds, localShardIds, nil
+	}
+
+	// Hand back what was copied so the caller deletes these orphaned working
+	// shards: recovery failed, but the temp files are already on the rebuilder.
+	return copiedShardIds, localShardIds, fmt.Errorf("%d shards are not enough to recover volume %d", len(localShardIds)+recoverableRemoteShards, volumeId)
+
+}
+
+type EcShardMap map[needle.VolumeId]EcShardLocations
+type EcShardLocations [][]*EcNode
+
+func (ecShardMap EcShardMap) registerEcNode(ecNode *EcNode, collection string) {
+	for _, diskInfo := range ecNode.Info.DiskInfos {
+		for _, shardInfo := range diskInfo.EcShardInfos {
+			if shardInfo.Collection == collection {
+				existing, found := ecShardMap[needle.VolumeId(shardInfo.Id)]
+				if !found {
+					// Use MaxShardCount (32) to support custom EC ratios
+					existing = make([][]*EcNode, erasure_coding.MaxShardCount)
+					ecShardMap[needle.VolumeId(shardInfo.Id)] = existing
+				}
+				for _, shardId := range erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(shardInfo).Ids() {
+					existing[shardId] = append(existing[shardId], ecNode)
+				}
+			}
+		}
+	}
+}
+
+func (ecShardLocations EcShardLocations) shardCount() (count int) {
+	for _, locations := range ecShardLocations {
+		if len(locations) > 0 {
+			count++
+		}
+	}
+	return
+}

weed/ec/ec_rebuild_test.go

@@ -0,0 +1,324 @@
+package ec
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// TestEcShardMapRegister tests that EC shards are properly registered
+func TestEcShardMapRegister(t *testing.T) {
+	ecShardMap := make(EcShardMap)
+
+	// Create test nodes with EC shards
+	node1 := newEcNode("dc1", "rack1", "node1", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6})
+	node2 := newEcNode("dc1", "rack1", "node2", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{7, 8, 9, 10, 11, 12, 13})
+
+	ecShardMap.registerEcNode(node1, "c1")
+	ecShardMap.registerEcNode(node2, "c1")
+
+	// Verify volume 1 is registered
+	locations, found := ecShardMap[needle.VolumeId(1)]
+	if !found {
+		t.Fatal("Expected volume 1 to be registered")
+	}
+
+	// Check shard count
+	count := locations.shardCount()
+	if count != erasure_coding.TotalShardsCount {
+		t.Errorf("Expected %d shards, got %d", erasure_coding.TotalShardsCount, count)
+	}
+
+	// Verify shard distribution
+	for i := 0; i < 7; i++ {
+		if len(locations[i]) != 1 || locations[i][0].Info.Id != "node1" {
+			t.Errorf("Shard %d should be on node1", i)
+		}
+	}
+	for i := 7; i < erasure_coding.TotalShardsCount; i++ {
+		if len(locations[i]) != 1 || locations[i][0].Info.Id != "node2" {
+			t.Errorf("Shard %d should be on node2", i)
+		}
+	}
+}
+
+// TestEcShardMapShardCount tests shard counting
+func TestEcShardMapShardCount(t *testing.T) {
+	testCases := []struct {
+		name          string
+		shardIds      []erasure_coding.ShardId
+		expectedCount int
+	}{
+		{"all shards", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}, 14},
+		{"data shards only", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, 10},
+		{"parity shards only", []erasure_coding.ShardId{10, 11, 12, 13}, 4},
+		{"missing some shards", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8}, 9},
+		{"single shard", []erasure_coding.ShardId{0}, 1},
+		{"no shards", []erasure_coding.ShardId{}, 0},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			locations := make(EcShardLocations, erasure_coding.MaxShardCount)
+			for _, shardId := range tc.shardIds {
+				locations[shardId] = []*EcNode{
+					newEcNode("dc1", "rack1", "node1", 100),
+				}
+			}
+
+			count := locations.shardCount()
+			if count != tc.expectedCount {
+				t.Errorf("Expected %d shards, got %d", tc.expectedCount, count)
+			}
+		})
+	}
+}
+
+// TestRebuildEcVolumesInsufficientShards tests that unrepairable volumes are skipped
+func TestRebuildEcVolumesInsufficientShards(t *testing.T) {
+	var logBuffer bytes.Buffer
+
+	// Create a volume with insufficient shards (less than DataShardsCount)
+	node1 := newEcNode("dc1", "rack1", "node1", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4}) // Only 5 shards
+
+	erb := &ecRebuilder{
+		env:     &Env{},
+		ewg:     util.NewErrorWaitGroup(10),
+		ecNodes: []*EcNode{node1},
+		writer:  &logBuffer,
+	}
+
+	erb.rebuildEcVolumes("c1")
+	err := erb.ewg.Wait()
+
+	// Unrepairable volumes should be skipped (not cause an error)
+	if err != nil {
+		t.Fatalf("Expected no error for unrepairable volume (should be skipped), got: %v", err)
+	}
+	// Verify the skip message was logged
+	if !strings.Contains(logBuffer.String(), "unrepairable") {
+		t.Errorf("Expected 'unrepairable' in log output, got: %s", logBuffer.String())
+	}
+}
+
+// TestRebuildEcVolumesCompleteVolume tests that complete volumes are skipped
+func TestRebuildEcVolumesCompleteVolume(t *testing.T) {
+	var logBuffer bytes.Buffer
+
+	// Create a volume with all shards
+	node1 := newEcNode("dc1", "rack1", "node1", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13})
+
+	erb := &ecRebuilder{
+		env:          &Env{},
+		ewg:          util.NewErrorWaitGroup(10),
+		ecNodes:      []*EcNode{node1},
+		writer:       &logBuffer,
+		applyChanges: false,
+	}
+
+	erb.rebuildEcVolumes("c1")
+	err := erb.ewg.Wait()
+
+	if err != nil {
+		t.Fatalf("Expected no error for complete volume, got: %v", err)
+	}
+
+	// The function should return quickly without attempting rebuild
+	// since the volume is already complete
+}
+
+// TestRebuildEcVolumesInsufficientSpace tests error handling for insufficient disk space
+func TestRebuildEcVolumesInsufficientSpace(t *testing.T) {
+	var logBuffer bytes.Buffer
+
+	// Create a volume with missing shards but insufficient free slots
+	// Node has 10 local shards, missing 4 shards (10,11,12,13), so needs 4 free slots
+	// Set free slots to 3 (insufficient)
+	node1 := newEcNode("dc1", "rack1", "node1", 3). // Only 3 free slots, need 4
+							addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
+
+	erb := &ecRebuilder{
+		env:          &Env{},
+		ewg:          util.NewErrorWaitGroup(10),
+		ecNodes:      []*EcNode{node1},
+		writer:       &logBuffer,
+		applyChanges: false,
+	}
+
+	erb.rebuildEcVolumes("c1")
+	err := erb.ewg.Wait()
+
+	if err == nil {
+		t.Fatal("Expected error for insufficient disk space, got nil")
+	}
+	if !strings.Contains(err.Error(), "no node has sufficient free slots") {
+		t.Errorf("Expected 'no node has sufficient free slots' in error message, got: %s", err.Error())
+	}
+	// Verify the enhanced error message includes diagnostic information
+	if !strings.Contains(err.Error(), "need") || !strings.Contains(err.Error(), "max available") {
+		t.Errorf("Expected diagnostic information in error message, got: %s", err.Error())
+	}
+}
+
+// TestMultipleNodesWithShards tests rebuild with shards distributed across multiple nodes
+func TestMultipleNodesWithShards(t *testing.T) {
+	ecShardMap := make(EcShardMap)
+
+	// Create 3 nodes with different shards
+	node1 := newEcNode("dc1", "rack1", "node1", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3})
+	node2 := newEcNode("dc1", "rack1", "node2", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{4, 5, 6, 7})
+	node3 := newEcNode("dc1", "rack1", "node3", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{8, 9})
+
+	ecShardMap.registerEcNode(node1, "c1")
+	ecShardMap.registerEcNode(node2, "c1")
+	ecShardMap.registerEcNode(node3, "c1")
+
+	locations := ecShardMap[needle.VolumeId(1)]
+	count := locations.shardCount()
+
+	// We have 10 shards total, which is enough for data shards
+	if count != 10 {
+		t.Errorf("Expected 10 shards, got %d", count)
+	}
+
+	// Verify each shard is on the correct node
+	for i := 0; i < 4; i++ {
+		if len(locations[i]) != 1 || locations[i][0].Info.Id != "node1" {
+			t.Errorf("Shard %d should be on node1", i)
+		}
+	}
+	for i := 4; i < 8; i++ {
+		if len(locations[i]) != 1 || locations[i][0].Info.Id != "node2" {
+			t.Errorf("Shard %d should be on node2", i)
+		}
+	}
+	for i := 8; i < 10; i++ {
+		if len(locations[i]) != 1 || locations[i][0].Info.Id != "node3" {
+			t.Errorf("Shard %d should be on node3", i)
+		}
+	}
+}
+
+// TestDuplicateShards tests handling of duplicate shards on multiple nodes
+func TestDuplicateShards(t *testing.T) {
+	ecShardMap := make(EcShardMap)
+
+	// Create 2 nodes with overlapping shards (both have shard 0)
+	node1 := newEcNode("dc1", "rack1", "node1", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3})
+	node2 := newEcNode("dc1", "rack1", "node2", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 4, 5, 6}) // Duplicate shard 0
+
+	ecShardMap.registerEcNode(node1, "c1")
+	ecShardMap.registerEcNode(node2, "c1")
+
+	locations := ecShardMap[needle.VolumeId(1)]
+
+	// Shard 0 should be on both nodes
+	if len(locations[0]) != 2 {
+		t.Errorf("Expected shard 0 on 2 nodes, got %d", len(locations[0]))
+	}
+
+	// Verify both nodes are registered for shard 0
+	foundNode1 := false
+	foundNode2 := false
+	for _, node := range locations[0] {
+		if node.Info.Id == "node1" {
+			foundNode1 = true
+		}
+		if node.Info.Id == "node2" {
+			foundNode2 = true
+		}
+	}
+	if !foundNode1 || !foundNode2 {
+		t.Error("Both nodes should have shard 0")
+	}
+
+	// Shard count should be 7 (unique shards: 0, 1, 2, 3, 4, 5, 6)
+	count := locations.shardCount()
+	if count != 7 {
+		t.Errorf("Expected 7 unique shards, got %d", count)
+	}
+}
+
+// TestPrepareDataToRecoverTargetShardCount tests that prepareDataToRecover caps the shard reporting
+func TestPrepareDataToRecoverTargetShardCount(t *testing.T) {
+	var logBuffer bytes.Buffer
+
+	// Create a node with 10 shards (0-9)
+	node1 := newEcNode("dc1", "rack1", "node1", 100).
+		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
+
+	erb := &ecRebuilder{
+		env:     &Env{},
+		ecNodes: []*EcNode{node1},
+		writer:  &logBuffer,
+	}
+
+	locations := make(EcShardLocations, erasure_coding.MaxShardCount)
+	for i := 0; i < 10; i++ {
+		locations[i] = []*EcNode{node1}
+	}
+
+	// Shards 10-13 are missing, but 14-31 should NOT be reported
+	_, _, err := erb.prepareDataToRecover(node1, "c1", needle.VolumeId(1), locations)
+	if err != nil {
+		t.Fatalf("prepareDataToRecover failed: %v", err)
+	}
+
+	output := logBuffer.String()
+	lines := strings.Split(strings.TrimSpace(output), "\n")
+
+	// We expect "use existing shard 1.0" through "use existing shard 1.9" (10 lines)
+	// and "missing shard 1.10" through "missing shard 1.13" (4 lines)
+	// Total 14 lines expected. Shards 14-31 should be absent.
+	expectedLineCount := 14
+	if len(lines) != expectedLineCount {
+		t.Errorf("Expected %d lines of output, got %d. Output:\n%s", expectedLineCount, len(lines), output)
+	}
+
+	for i := 14; i < erasure_coding.MaxShardCount; i++ {
+		if strings.Contains(output, "shard 1."+strings.TrimSpace(string(rune('0'+i)))) ||
+			strings.Contains(output, "shard 1."+fmt.Sprintf("%d", i)) {
+			t.Errorf("Shard 1.%d should not be reported in output", i)
+		}
+	}
+}
+
+func TestRebuilderVolumeIdMatches(t *testing.T) {
+	testCases := []struct {
+		name               string
+		rebuilderVolumeIds []needle.VolumeId
+		volumeId           needle.VolumeId
+		want               bool
+	}{
+		{"no target volume IDs (nil)", nil, needle.VolumeId(1234), true},
+		{"no target volume IDs", []needle.VolumeId{}, needle.VolumeId(5678), true},
+		{"volume ID matches", []needle.VolumeId{needle.VolumeId(123), needle.VolumeId(456), needle.VolumeId(789)}, needle.VolumeId(456), true},
+		{"volume ID mismatch", []needle.VolumeId{needle.VolumeId(123), needle.VolumeId(456), needle.VolumeId(789)}, needle.VolumeId(321), false},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ecb := &ecRebuilder{
+				volumeIds: tc.rebuilderVolumeIds,
+			}
+
+			if got, want := ecb.matchesVolumeId(tc.volumeId), tc.want; got != want {
+				t.Errorf("Expected %v, got %v", want, got)
+			}
+		})
+	}
+}

weed/ec/ec_scrub.go

@@ -0,0 +1,81 @@
+package ec
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// ScrubEcVolumes asks each volume server to scrub its EC volumes (optionally
+// narrowed to volumeIds) in the given mode, aggregating and reporting results.
+func ScrubEcVolumes(env *Env, writer io.Writer, volumeServerAddrs []pb.ServerAddress, volumeIds []uint32, mode volume_server_pb.VolumeScrubMode, forceDeletedNeedlesCheck bool, maxParallelization int, showDetails bool) error {
+	var brokenVolumesStr, brokenShardsStr []string
+	var details []string
+	var totalVolumes, brokenVolumes, brokenShards, totalFiles uint64
+	var mu sync.Mutex
+
+	ewg := util.NewErrorWaitGroup(maxParallelization)
+	count := 0
+	for _, addr := range volumeServerAddrs {
+		ewg.Add(func() error {
+			mu.Lock()
+			count++
+			fmt.Fprintf(writer, "Scrubbing %s (%d/%d)...\n", addr.String(), count, len(volumeServerAddrs))
+			mu.Unlock()
+
+			err := operation.WithVolumeServerClient(false, addr, env.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+				res, err := volumeServerClient.ScrubEcVolume(context.Background(), &volume_server_pb.ScrubEcVolumeRequest{
+					Mode:                     mode,
+					VolumeIds:                volumeIds,
+					ForceDeletedNeedlesCheck: forceDeletedNeedlesCheck,
+				})
+				if err != nil {
+					return err
+				}
+
+				mu.Lock()
+				defer mu.Unlock()
+
+				totalVolumes += res.GetTotalVolumes()
+				totalFiles += res.GetTotalFiles()
+				brokenVolumes += uint64(len(res.GetBrokenVolumeIds()))
+				brokenShards += uint64(len(res.GetBrokenShardInfos()))
+				for _, d := range res.GetDetails() {
+					details = append(details, fmt.Sprintf("[%s] %s", addr, d))
+				}
+				for _, vid := range res.GetBrokenVolumeIds() {
+					brokenVolumesStr = append(brokenVolumesStr, fmt.Sprintf("%s:%v", addr, vid))
+				}
+				for _, si := range res.GetBrokenShardInfos() {
+					brokenShardsStr = append(brokenShardsStr, fmt.Sprintf("%s:%v:%v", addr, si.VolumeId, si.ShardId))
+				}
+
+				return nil
+			})
+			return err
+		})
+	}
+	if err := ewg.Wait(); err != nil {
+		return err
+	}
+
+	fmt.Fprintf(writer, "Scrubbed %d EC files and %d volumes on %d nodes\n", totalFiles, totalVolumes, len(volumeServerAddrs))
+	if brokenVolumes != 0 {
+		fmt.Fprintf(writer, "\nGot scrub failures on %d EC volumes and %d EC shards :(\n", brokenVolumes, brokenShards)
+		fmt.Fprintf(writer, "Affected volumes: %s\n", strings.Join(brokenVolumesStr, ", "))
+		if len(brokenShardsStr) != 0 {
+			fmt.Fprintf(writer, "Affected shards:  %s\n", strings.Join(brokenShardsStr, ", "))
+		}
+		if showDetails && len(details) != 0 {
+			fmt.Fprintf(writer, "Details:\n\t%s\n", strings.Join(details, "\n\t"))
+		}
+	}
+	return nil
+}

weed/ec/ec_shard_unmount.go

@@ -0,0 +1,210 @@
+package ec
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+const (
+	ecShardActionTimeout = 1 * time.Minute
+)
+
+// ShardRef names one EC shard copy, optionally pinned to a node address.
+type ShardRef struct {
+	ShardID     uint32
+	Collection  string
+	NodeAddress string
+}
+
+func (s *ShardRef) String() string {
+	if s.NodeAddress == "" {
+		return fmt.Sprintf("%d", s.ShardID)
+	}
+	return fmt.Sprintf("%d@%s", s.ShardID, s.NodeAddress)
+}
+
+// ShardRefsFromString parses a comma-separated list of shard IDs, each either a
+// bare numeric ID or <shard_id>@<node_address> to pick one copy.
+func ShardRefsFromString(shards string) ([]*ShardRef, error) {
+	res := []*ShardRef{}
+
+	for _, s := range strings.Split(shards, ",") {
+		s = strings.TrimSpace(s)
+		if s == "" {
+			return nil, fmt.Errorf("empty shard ID in %q", shards)
+		}
+
+		// optional <shard ID>@<node address> to pick one copy
+		idStr, addr, _ := strings.Cut(s, "@")
+
+		id, err := strconv.Atoi(idStr)
+		if err != nil || id < 0 || id >= erasure_coding.MaxShardCount {
+			return nil, fmt.Errorf("invalid shard ID %q", s)
+		}
+
+		res = append(res, &ShardRef{ShardID: uint32(id), NodeAddress: addr})
+	}
+
+	return res, nil
+}
+
+// ShardUnmountRequest describes one ec.shard.unmount run.
+type ShardUnmountRequest struct {
+	Topology      *master_pb.TopologyInfo
+	VolumeID      uint32
+	Shards        []*ShardRef
+	Delete        bool
+	IgnoreInvalid bool
+	Apply         bool
+}
+
+type ecShardUnmounter struct {
+	env    *Env
+	writer io.Writer
+	req    ShardUnmountRequest
+}
+
+// UnmountShards unmounts, and optionally deletes, the requested EC shard
+// copies, resolving them against the live topology first. Dry-run unless
+// req.Apply is set.
+func UnmountShards(env *Env, writer io.Writer, req ShardUnmountRequest) error {
+	c := &ecShardUnmounter{env: env, writer: writer, req: req}
+	return c.doShardsUnmount()
+}
+
+func (c *ecShardUnmounter) write(format string, a ...any) {
+	fmt.Fprintf(c.writer, format, a...)
+}
+
+func (c *ecShardUnmounter) liveShardsForVolume() []*ShardRef {
+	shards := []*ShardRef{}
+
+	for _, dci := range c.req.Topology.GetDataCenterInfos() {
+		for _, ri := range dci.GetRackInfos() {
+			for _, dni := range ri.GetDataNodeInfos() {
+				nodeAddress := dni.GetAddress()
+				for _, di := range dni.GetDiskInfos() {
+					for _, eci := range di.GetEcShardInfos() {
+						if eci.GetId() == c.req.VolumeID {
+							sinfo := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci)
+							for _, sid := range sinfo.Ids() {
+								shards = append(shards, &ShardRef{
+									ShardID:     uint32(sid),
+									Collection:  eci.GetCollection(),
+									NodeAddress: nodeAddress,
+								})
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	sort.SliceStable(shards, func(i, j int) bool { return shards[i].ShardID < shards[j].ShardID })
+	return shards
+}
+
+func (c *ecShardUnmounter) printShards(ss []*ShardRef) {
+	for _, s := range ss {
+		c.write("\t%v\n", s)
+	}
+	c.write("\n")
+}
+
+func (c *ecShardUnmounter) doShardsUnmount() error {
+	liveShards := c.liveShardsForVolume()
+	c.write("Live shard topology for volume ID %d (%d shards):\n", c.req.VolumeID, len(liveShards))
+	c.printShards(liveShards)
+
+	// resolve target shards against the live topology
+	targetShards := []*ShardRef{}
+	for _, ps := range c.req.Shards {
+		var result *ShardRef
+		for _, ts := range liveShards {
+			if ts.ShardID == ps.ShardID {
+				if ps.NodeAddress == "" || ps.NodeAddress == ts.NodeAddress {
+					if result != nil {
+						return fmt.Errorf("shard %v is ambiguous", ps)
+					}
+					result = ts
+				}
+			}
+		}
+		if result == nil {
+			if !c.req.IgnoreInvalid {
+				return fmt.Errorf("shard %v is invalid", ps)
+			}
+			c.write("!!! ignoring invalid shard %v\n", ps)
+		} else {
+			targetShards = append(targetShards, result)
+		}
+	}
+	if len(targetShards) == 0 {
+		return fmt.Errorf("got no shards to process")
+	}
+
+	mode := "unmount"
+	if c.req.Delete {
+		mode = "unmount + delete"
+	}
+	c.write("Will %s %d shard(s):\n", mode, len(targetShards))
+	c.printShards(targetShards)
+
+	if !c.req.Apply {
+		c.write("Not proceeding in dry-run mode\n")
+		return nil
+	}
+
+	if !c.env.isLocked() {
+		return fmt.Errorf("lock is lost")
+	}
+
+	for _, s := range targetShards {
+		if err := c.unmountShard(s); err != nil {
+			return err
+		}
+	}
+
+	c.write("\nAll done!\n")
+	return nil
+}
+
+func (c *ecShardUnmounter) unmountShard(s *ShardRef) error {
+	ctx, cancel := context.WithTimeout(context.Background(), ecShardActionTimeout)
+	defer cancel()
+
+	return operation.WithVolumeServerClient(false, pb.ServerAddress(s.NodeAddress), c.env.GrpcDialOption, func(vsc volume_server_pb.VolumeServerClient) error {
+		c.write("Unmounting shard %v for volume ID %d...\n", s, c.req.VolumeID)
+		if _, err := vsc.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
+			VolumeId: c.req.VolumeID,
+			ShardIds: []uint32{s.ShardID},
+		}); err != nil {
+			return err
+		}
+
+		if c.req.Delete {
+			c.write("Deleting shard %v for volume ID %d...\n", s, c.req.VolumeID)
+			if _, err := vsc.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+				VolumeId:   c.req.VolumeID,
+				Collection: s.Collection,
+				ShardIds:   []uint32{s.ShardID},
+			}); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}

weed/ec/ec_shard_unmount_test.go

@@ -0,0 +1,102 @@
+package ec
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEcShardsFromString(t *testing.T) {
+	tests := []struct {
+		name    string
+		in      string
+		want    []*ShardRef
+		wantErr bool
+	}{
+		{
+			name: "single id",
+			in:   "2",
+			want: []*ShardRef{{ShardID: 2}},
+		},
+		{
+			name: "list of ids",
+			in:   "0, 3 ,5",
+			want: []*ShardRef{{ShardID: 0}, {ShardID: 3}, {ShardID: 5}},
+		},
+		{
+			name: "qualified keeps host:port",
+			in:   "3@10.200.18.88:9007",
+			want: []*ShardRef{{ShardID: 3, NodeAddress: "10.200.18.88:9007"}},
+		},
+		{
+			name: "mixed bare and qualified",
+			in:   "2,3@10.200.18.88:9007",
+			want: []*ShardRef{{ShardID: 2}, {ShardID: 3, NodeAddress: "10.200.18.88:9007"}},
+		},
+		{
+			name:    "colon without node marker is not a shard ID",
+			in:      "3:10.200.18.88:9007",
+			wantErr: true,
+		},
+		{
+			// shard IDs beyond the default 10+4 total are valid on custom-ratio volumes.
+			name: "shard id within MaxShardCount",
+			in:   "20",
+			want: []*ShardRef{{ShardID: 20}},
+		},
+		{
+			name:    "shard id at MaxShardCount is rejected",
+			in:      "32",
+			wantErr: true,
+		},
+		{
+			name:    "negative shard id",
+			in:      "-1",
+			wantErr: true,
+		},
+		{
+			// must not wrap through uint32 into a valid-looking shard ID (4294967301 -> 5)
+			name:    "out-of-range shard id does not wrap",
+			in:      "4294967301",
+			wantErr: true,
+		},
+		{
+			name:    "non-numeric shard id",
+			in:      "abc",
+			wantErr: true,
+		},
+		{
+			name:    "empty element",
+			in:      "1,,2",
+			wantErr: true,
+		},
+	}
+
+	// guard the assumption the bound test relies on
+	assert.Equal(t, uint32(32), uint32(erasure_coding.MaxShardCount))
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ShardRefsFromString(tt.in)
+			if tt.wantErr {
+				assert.Error(t, err)
+				return
+			}
+			assert.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+// the printed topology form must parse back, so entries can be copied verbatim.
+func TestEcShardStringRoundTrips(t *testing.T) {
+	for _, s := range []*ShardRef{
+		{ShardID: 2},
+		{ShardID: 3, NodeAddress: "10.200.18.88:9007"},
+	} {
+		got, err := ShardRefsFromString(s.String())
+		assert.NoError(t, err)
+		assert.Equal(t, []*ShardRef{s}, got)
+	}
+}

weed/ec/env.go

@@ -0,0 +1,39 @@
+// Package ec holds the EC (erasure coding) cluster orchestration logic shared
+// by the weed shell commands and the maintenance workers: topology analysis of
+// EC shards, the encode/balance pipelines, and the volume-server RPC wrappers
+// they drive. The placement policy itself lives in
+// weed/storage/erasure_coding/ecbalancer; low-level shard mechanics live in
+// weed/storage/erasure_coding.
+package ec
+
+import (
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+	"google.golang.org/grpc"
+)
+
+// Env carries the cluster access hooks EC operations need, decoupled from any
+// particular caller (shell CommandEnv, worker task, admin server).
+type Env struct {
+	GrpcDialOption grpc.DialOption
+	// FetchTopology returns a fresh master topology snapshot (and the master's
+	// volume size limit in MB) after an optional delay.
+	FetchTopology func(delay time.Duration) (*master_pb.TopologyInfo, uint64, error)
+	// GetVolumeLocations returns the current replica locations for a volume id,
+	// or false if the volume is unknown.
+	GetVolumeLocations func(vid uint32) ([]wdclient.Location, bool)
+	// IsLocked reports whether the caller still holds the cluster admin lock.
+	// Callers without a lock concept return true.
+	IsLocked func() bool
+}
+
+// isLocked treats a nil Env or nil hook as locked, matching the shell's
+// nil-receiver behavior so dry-run paths work without a cluster connection.
+func (env *Env) isLocked() bool {
+	if env == nil || env.IsLocked == nil {
+		return true
+	}
+	return env.IsLocked()
+}