snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/storage/erasure_coding/distribution/README.md

@@ -0,0 +1,209 @@
+# EC Distribution Package
+
+This package provides erasure coding (EC) shard distribution algorithms that are:
+
+- **Configurable**: Works with any EC ratio (e.g., 10+4, 8+4, 6+3)
+- **Reusable**: Used by shell commands, worker tasks, and seaweed-enterprise
+- **Topology-aware**: Distributes shards across data centers, racks, and nodes proportionally
+
+## Usage
+
+### Basic Usage with Default 10+4 EC
+
+```go
+import (
+    "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/distribution"
+)
+
+// Parse replication policy
+rep, _ := distribution.NewReplicationConfigFromString("110")
+
+// Use default 10+4 EC configuration
+ec := distribution.DefaultECConfig()
+
+// Calculate distribution plan
+dist := distribution.CalculateDistribution(ec, rep)
+
+fmt.Println(dist.Summary())
+// Output:
+// EC Configuration: 10+4 (total: 14, can lose: 4)
+// Replication: replication=110 (DCs:2, Racks/DC:2, Nodes/Rack:1)
+// Distribution Plan:
+//   Data Centers: 2 (target 7 shards each, max 9)
+//   Racks per DC: 2 (target 4 shards each, max 6)
+//   Nodes per Rack: 1 (target 4 shards each, max 6)
+```
+
+### Custom EC Ratios (seaweed-enterprise)
+
+```go
+// Create custom 8+4 EC configuration
+ec, err := distribution.NewECConfig(8, 4)
+if err != nil {
+    log.Fatal(err)
+}
+
+rep, _ := distribution.NewReplicationConfigFromString("200")
+dist := distribution.CalculateDistribution(ec, rep)
+
+// Check fault tolerance
+fmt.Println(dist.FaultToleranceAnalysis())
+// Output:
+// Fault Tolerance Analysis for 8+4:
+//   DC Failure: SURVIVABLE ✓
+//     - Losing one DC loses ~4 shards
+//     - Remaining: 8 shards (need 8)
+```
+
+### Planning Shard Moves
+
+```go
+// Build topology analysis
+analysis := distribution.NewTopologyAnalysis()
+
+// Add nodes and their shard locations
+for _, node := range nodes {
+    analysis.AddNode(&distribution.TopologyNode{
+        NodeID:     node.ID,
+        DataCenter: node.DC,
+        Rack:       node.Rack,
+        FreeSlots:  node.FreeSlots,
+    })
+    for _, shardID := range node.ShardIDs {
+        analysis.AddShardLocation(distribution.ShardLocation{
+            ShardID:    shardID,
+            NodeID:     node.ID,
+            DataCenter: node.DC,
+            Rack:       node.Rack,
+        })
+    }
+}
+analysis.Finalize()
+
+// Create rebalancer and plan moves
+rebalancer := distribution.NewRebalancer(ec, rep)
+plan, err := rebalancer.PlanRebalance(analysis)
+
+for _, move := range plan.Moves {
+    fmt.Printf("Move shard %d from %s to %s\n", 
+        move.ShardID, move.SourceNode.NodeID, move.DestNode.NodeID)
+}
+```
+
+## Algorithm
+
+### Proportional Distribution
+
+The replication policy `XYZ` is interpreted as a ratio:
+
+| Replication | DCs | Racks/DC | Nodes/Rack | 14 Shards Distribution |
+|-------------|-----|----------|------------|------------------------|
+| `000` | 1 | 1 | 1 | All in one place |
+| `001` | 1 | 1 | 2 | 7 per node |
+| `010` | 1 | 2 | 1 | 7 per rack |
+| `100` | 2 | 1 | 1 | 7 per DC |
+| `110` | 2 | 2 | 1 | 7/DC, 4/rack |
+| `200` | 3 | 1 | 1 | 5 per DC |
+
+### Rebalancing Process
+
+1. **DC-level balancing**: Move shards to achieve target shards per DC
+2. **Rack-level balancing**: Within each DC, balance across racks
+3. **Node-level balancing**: Within each rack, balance across nodes
+
+### Shard Priority: Data First, Parity Moves First
+
+When rebalancing, the algorithm prioritizes keeping data shards spread out:
+
+- **Data shards (0 to DataShards-1)**: Serve read requests directly
+- **Parity shards (DataShards to TotalShards-1)**: Only used for reconstruction
+
+**Rebalancing Strategy**:
+- When moving shards FROM an overloaded node, **parity shards are moved first**
+- This keeps data shards in place on well-distributed nodes
+- Result: Data shards remain spread out for optimal read performance
+
+```go
+// Check shard type
+if ec.IsDataShard(shardID) {
+    // Shard serves read requests
+}
+if ec.IsParityShard(shardID) {
+    // Shard only used for reconstruction
+}
+
+// Sort shards for placement (data first for initial distribution)
+sorted := ec.SortShardsDataFirst(shards)
+
+// Sort shards for rebalancing (parity first to move them away)
+sorted := ec.SortShardsParityFirst(shards)
+```
+
+### Fault Tolerance
+
+The package provides fault tolerance analysis:
+
+- **DC Failure**: Can the data survive complete DC loss?
+- **Rack Failure**: Can the data survive complete rack loss?
+- **Node Failure**: Can the data survive single node loss?
+
+For example, with 10+4 EC (can lose 4 shards):
+- Need 4+ DCs for DC-level fault tolerance
+- Need 4+ racks for rack-level fault tolerance
+- Usually survivable at node level
+
+## API Reference
+
+### Types
+
+- `ECConfig`: EC configuration (data shards, parity shards)
+- `ReplicationConfig`: Parsed replication policy
+- `ECDistribution`: Calculated distribution plan
+- `TopologyAnalysis`: Current shard distribution analysis
+- `Rebalancer`: Plans shard moves
+- `RebalancePlan`: List of planned moves
+- `ShardMove`: Single shard move operation
+
+### Key Functions
+
+- `NewECConfig(data, parity int)`: Create EC configuration
+- `DefaultECConfig()`: Returns 10+4 configuration
+- `CalculateDistribution(ec, rep)`: Calculate distribution plan
+- `NewRebalancer(ec, rep)`: Create rebalancer
+- `PlanRebalance(analysis)`: Generate rebalancing plan
+
+## Integration
+
+### Shell Commands
+
+The shell package wraps this distribution package for `ec.balance`:
+
+```go
+import "github.com/seaweedfs/seaweedfs/weed/shell"
+
+rebalancer := shell.NewProportionalECRebalancer(nodes, rp, diskType)
+moves, _ := rebalancer.PlanMoves(volumeId, locations)
+```
+
+### Worker Tasks
+
+Worker tasks can use the distribution package directly:
+
+```go
+import "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/distribution"
+
+ec := distribution.ECConfig{DataShards: 8, ParityShards: 4}
+rep := distribution.NewReplicationConfig(rp)
+dist := distribution.CalculateDistribution(ec, rep)
+```
+
+### seaweed-enterprise
+
+Enterprise features can provide custom EC configurations:
+
+```go
+// Custom EC ratio from license/config
+ec, _ := distribution.NewECConfig(customData, customParity)
+rebalancer := distribution.NewRebalancer(ec, rep)
+```
+

weed/storage/erasure_coding/distribution/analysis.go

@@ -0,0 +1,52 @@
+package distribution
+
+// ShardLocation represents where a shard is located in the topology
+type ShardLocation struct {
+	ShardID    int
+	NodeID     string
+	DataCenter string
+	Rack       string
+}
+
+// TopologyNode represents a node in the topology that can hold EC shards
+type TopologyNode struct {
+	NodeID      string
+	DataCenter  string
+	Rack        string
+	FreeSlots   int   // Available slots for new shards
+	ShardIDs    []int // Shard IDs currently on this node for a specific volume
+	TotalShards int   // Total shards on this node (for all volumes)
+}
+
+// TopologyAnalysis holds the current shard distribution analysis for a volume
+type TopologyAnalysis struct {
+	// Shard counts at each level
+	ShardsByDC   map[string]int
+	ShardsByRack map[string]int
+	ShardsByNode map[string]int
+
+	// Detailed shard locations
+	DCToShards   map[string][]int // DC -> list of shard IDs
+	RackToShards map[string][]int // Rack -> list of shard IDs
+	NodeToShards map[string][]int // NodeID -> list of shard IDs
+
+	// Topology structure
+	DCToRacks   map[string][]string        // DC -> list of rack IDs
+	RackToNodes map[string][]*TopologyNode // Rack -> list of nodes
+	AllNodes    map[string]*TopologyNode   // NodeID -> node info
+
+	// Statistics
+	TotalShards int
+	TotalNodes  int
+	TotalRacks  int
+	TotalDCs    int
+}
+
+// TopologyExcess represents a topology level (DC/rack/node) with excess shards
+type TopologyExcess struct {
+	ID     string          // DC/rack/node ID
+	Level  string          // "dc", "rack", or "node"
+	Excess int             // Number of excess shards (above target)
+	Shards []int           // Shard IDs at this level
+	Nodes  []*TopologyNode // Nodes at this level (for finding sources)
+}

weed/storage/erasure_coding/distribution/config.go

@@ -0,0 +1,19 @@
+// Package distribution provides EC shard distribution algorithms with configurable EC ratios.
+package distribution
+
+// ECConfig holds erasure coding configuration parameters.
+// This replaces hard-coded constants like DataShardsCount=10, ParityShardsCount=4.
+type ECConfig struct {
+	DataShards   int // Number of data shards (e.g., 10)
+	ParityShards int // Number of parity shards (e.g., 4)
+}
+
+// ReplicationConfig holds the parsed replication policy
+type ReplicationConfig struct {
+	MinDataCenters  int // X+1 from XYZ replication (minimum DCs to use)
+	MinRacksPerDC   int // Y+1 from XYZ replication (minimum racks per DC)
+	MinNodesPerRack int // Z+1 from XYZ replication (minimum nodes per rack)
+
+	// Original replication string (for logging/debugging)
+	Original string
+}

weed/storage/erasure_coding/distribution/distribution.go

@@ -0,0 +1,22 @@
+package distribution
+
+// ECDistribution represents the target distribution of EC shards
+// based on EC configuration and replication policy.
+type ECDistribution struct {
+	// EC configuration
+	ECConfig ECConfig
+
+	// Replication configuration
+	ReplicationConfig ReplicationConfig
+
+	// Target shard counts per topology level (balanced distribution)
+	TargetShardsPerDC   int
+	TargetShardsPerRack int
+	TargetShardsPerNode int
+
+	// Maximum shard counts per topology level (fault tolerance limits)
+	// These prevent any single failure domain from having too many shards
+	MaxShardsPerDC   int
+	MaxShardsPerRack int
+	MaxShardsPerNode int
+}

weed/storage/erasure_coding/distribution/rebalancer.go

@@ -0,0 +1,28 @@
+package distribution
+
+// ShardMove represents a planned shard move
+type ShardMove struct {
+	ShardID    int
+	SourceNode *TopologyNode
+	DestNode   *TopologyNode
+	Reason     string
+}
+
+// RebalancePlan contains the complete plan for rebalancing EC shards
+type RebalancePlan struct {
+	Moves        []ShardMove
+	Distribution *ECDistribution
+	Analysis     *TopologyAnalysis
+
+	// Statistics
+	TotalMoves      int
+	MovesAcrossDC   int
+	MovesAcrossRack int
+	MovesWithinRack int
+}
+
+// Rebalancer plans shard moves to achieve proportional distribution
+type Rebalancer struct {
+	ecConfig  ECConfig
+	repConfig ReplicationConfig
+}

weed/storage/erasure_coding/ec_bitrot.go

@@ -0,0 +1,657 @@
+package erasure_coding
+
+// EC bitrot detection — checksum sidecar.
+//
+// A per-volume sidecar file stores a CRC32C (Castagnoli) checksum for every
+// fixed-size block of every EC shard, so a scrub (and the reconstruction path)
+// can detect silent disk corruption in any shard — including cold parity shards
+// that are never read during normal serving.
+//
+// The sidecar is OPTIONAL: an absent or generation-mismatched sidecar simply
+// means "feature off" for that generation, so old binaries, JSON-only Rust
+// nodes, and rollback deployments ignore it and degrade gracefully. See
+// EC_BITROT_DETECTION_DESIGN.md.
+//
+// On-disk layout of <base>.ecsum (legacy/generation 0) and <base>.ecsum.v<N>:
+//
+//	[ magic(4) | format_version(2) | payload_len(4) | payload_crc32c(4) ] [ proto payload ]
+//
+// The header's payload_crc32c lets a loader detect corruption of the sidecar
+// itself BEFORE trusting any contents, so a rotted sidecar can never be
+// mistaken for shard corruption.
+
+import (
+	"crypto/rand"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"math/bits"
+	"os"
+	"path/filepath"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"google.golang.org/protobuf/proto"
+)
+
+const (
+	// BitrotSidecarExt is the canonical extension for the checksum sidecar.
+	// Generation 0 (legacy/fresh encode) uses "<base>.ecsum"; vacuum generation
+	// N uses "<base>.ecsum.v<N>", mirroring the .vif/.ecx versioned convention.
+	BitrotSidecarExt = ".ecsum"
+
+	// DefaultBitrotBlockSize is the default checksum granularity (16 MiB). It is
+	// a power-of-two multiple of ErasureCodingSmallBlockSize (1 MiB) and keeps
+	// the sidecar tiny (~11 KB for a 30 GB volume) while localizing corruption
+	// to a 16 MiB region.
+	DefaultBitrotBlockSize = 16 * 1024 * 1024
+
+	// MaxBitrotBlockSize caps the block granularity so a loaded sidecar cannot
+	// force a huge scrub/verify scratch buffer. Power-of-two multiple of 1 MiB.
+	MaxBitrotBlockSize = 64 * 1024 * 1024
+
+	bitrotMagic         uint32 = 0x45435355 // "ECSU"
+	bitrotFormatVersion uint16 = 1
+	bitrotHeaderSize           = 14 // magic(4)+version(2)+payload_len(4)+payload_crc32c(4)
+)
+
+// Config knobs (wired to volume-server flags). Defaults: protection on for new
+// encodes, 16 MiB block granularity.
+var (
+	// BitrotProtectionEnabled controls whether encode/vacuum write a sidecar.
+	// When false, generations are produced unprotected (no sidecar), which is a
+	// legitimate "off" state. Detection on already-protected generations is
+	// unaffected.
+	BitrotProtectionEnabled = true
+	// BitrotBlockSize is the checksum block granularity used for new sidecars.
+	// Must remain a power-of-two multiple of 1 MiB.
+	BitrotBlockSize int64 = DefaultBitrotBlockSize
+)
+
+// BitrotStatus is the resolved protection state of an EC volume's active
+// generation after loading and validating its sidecar.
+type BitrotStatus int
+
+const (
+	// BitrotOff: no sidecar, or a sidecar that does not describe the active
+	// generation. The generation is unprotected; this is NOT corruption.
+	BitrotOff BitrotStatus = iota
+	// BitrotOn: a complete, well-formed, generation-matching sidecar is loaded.
+	BitrotOn
+	// BitrotInvalid: a generation-matching sidecar that is malformed, incomplete,
+	// or self-integrity-failed. The generation is unprotected pending repair of
+	// the sidecar, and an integrity alarm should fire. The rebuild path treats
+	// this as fail-closed (see ec_encoder rebuild verify).
+	BitrotInvalid
+)
+
+func (s BitrotStatus) String() string {
+	switch s {
+	case BitrotOn:
+		return "on"
+	case BitrotInvalid:
+		return "invalid"
+	default:
+		return "off"
+	}
+}
+
+// BitrotSidecarPath returns the sidecar path for a base file name and EC
+// generation. Generation 0 is the un-suffixed legacy path; generation N>0 is
+// the versioned path, consistent with how .vif/.ecx are versioned by the 2PC
+// switch.
+func BitrotSidecarPath(baseFileName string, generation uint32) string {
+	if generation == 0 {
+		return baseFileName + BitrotSidecarExt
+	}
+	return fmt.Sprintf("%s%s.v%d", baseFileName, BitrotSidecarExt, generation)
+}
+
+// NewEncodeUUID returns a fresh random per-encode identity used to detect a
+// stale sidecar left behind by an in-place re-encode.
+func NewEncodeUUID() []byte {
+	b := make([]byte, 16)
+	if _, err := rand.Read(b); err != nil {
+		// rand.Read essentially never fails; fall back to a zero UUID rather
+		// than aborting an encode over entropy. Stale detection still has the
+		// other defenses (wholesale-mismatch cap + RS arbiter).
+		glog.Warningf("ec bitrot: failed to read encode uuid entropy: %v", err)
+	}
+	return b
+}
+
+// isPow2MultipleOf1MiB reports whether block_size is a power of two in
+// [1 MiB, MaxBitrotBlockSize].
+func isPow2MultipleOf1MiB(blockSize uint32) bool {
+	return blockSize >= (1<<20) && blockSize <= MaxBitrotBlockSize && bits.OnesCount32(blockSize) == 1
+}
+
+// shardChecksumBuilder accumulates the per-block CRC32C of a single shard's byte
+// stream as it is written. It tolerates arbitrary chunk sizes that cross block
+// boundaries, so it works for both the 256 KiB encode buffers and the 1 MiB
+// rebuild buffers.
+type shardChecksumBuilder struct {
+	blockSize int64
+	cur       needle.CRC
+	curLen    int64
+	total     int64
+	blocks    []uint32
+}
+
+func newShardChecksumBuilder(blockSize int64) *shardChecksumBuilder {
+	return &shardChecksumBuilder{blockSize: blockSize}
+}
+
+func (b *shardChecksumBuilder) write(p []byte) {
+	for len(p) > 0 {
+		room := b.blockSize - b.curLen
+		n := int64(len(p))
+		if n > room {
+			n = room
+		}
+		b.cur = b.cur.Update(p[:n])
+		b.curLen += n
+		b.total += n
+		p = p[n:]
+		if b.curLen == b.blockSize {
+			b.blocks = append(b.blocks, uint32(b.cur))
+			b.cur = 0
+			b.curLen = 0
+		}
+	}
+}
+
+// finalize flushes any partial last block and returns the covered size and the
+// packed little-endian uint32 CRC array.
+func (b *shardChecksumBuilder) finalize() (coveredSize int64, packed []byte) {
+	if b.curLen > 0 {
+		b.blocks = append(b.blocks, uint32(b.cur))
+		b.cur = 0
+		b.curLen = 0
+	}
+	return b.total, packUint32LE(b.blocks)
+}
+
+// buildProtectionFromBuilders assembles a generation-0 EcBitrotProtection from
+// the per-shard checksum builders produced during an encode pass. Callers that
+// produce a versioned generation (vacuum) set prot.Generation afterwards. A
+// fresh encode_uuid is minted on every call so an in-place re-encode is
+// distinguishable from a stale sidecar.
+func buildProtectionFromBuilders(ctx *ECContext, builders []*shardChecksumBuilder, blockSize int64) *volume_server_pb.EcBitrotProtection {
+	shards := make([]*volume_server_pb.EcShardChecksums, 0, len(builders))
+	for id, b := range builders {
+		covered, packed := b.finalize()
+		shards = append(shards, &volume_server_pb.EcShardChecksums{
+			ShardId:     uint32(id),
+			CoveredSize: covered,
+			BlockCrc32C: packed,
+		})
+	}
+	return &volume_server_pb.EcBitrotProtection{
+		Algorithm:  volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+		BlockSize:  uint32(blockSize),
+		Generation: 0,
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards:   uint32(ctx.DataShards),
+			ParityShards: uint32(ctx.ParityShards),
+			BlockSize:    ctx.BlockSize,
+		},
+		Shards:     shards,
+		EncodeUuid: NewEncodeUUID(),
+	}
+}
+
+func packUint32LE(vals []uint32) []byte {
+	out := make([]byte, len(vals)*4)
+	for i, v := range vals {
+		binary.LittleEndian.PutUint32(out[i*4:], v)
+	}
+	return out
+}
+
+func unpackUint32LE(b []byte) []uint32 {
+	out := make([]uint32, len(b)/4)
+	for i := range out {
+		out[i] = binary.LittleEndian.Uint32(b[i*4:])
+	}
+	return out
+}
+
+// expectedBlockCount returns ceil(coveredSize/blockSize).
+func expectedBlockCount(coveredSize, blockSize int64) int {
+	if blockSize <= 0 {
+		return 0
+	}
+	return int((coveredSize + blockSize - 1) / blockSize)
+}
+
+// SaveBitrotSidecar atomically writes prot to path, wrapped in the on-disk
+// header with a CRC32C over the serialized payload.
+func SaveBitrotSidecar(path string, prot *volume_server_pb.EcBitrotProtection) error {
+	payload, err := proto.Marshal(prot)
+	if err != nil {
+		return fmt.Errorf("marshal bitrot sidecar: %w", err)
+	}
+	// The header records payload_len as a uint32 and the allocation below adds
+	// it to a constant. Bound the payload well under any overflow (a real
+	// manifest is a few KB) so neither the length field nor make() can wrap.
+	const maxBitrotPayloadSize = 1 << 30 // 1 GiB, vastly above any real sidecar
+	if len(payload) > maxBitrotPayloadSize {
+		return fmt.Errorf("bitrot sidecar payload too large: %d bytes", len(payload))
+	}
+	buf := make([]byte, bitrotHeaderSize+len(payload))
+	binary.BigEndian.PutUint32(buf[0:4], bitrotMagic)
+	binary.BigEndian.PutUint16(buf[4:6], bitrotFormatVersion)
+	binary.BigEndian.PutUint32(buf[6:10], uint32(len(payload)))
+	binary.BigEndian.PutUint32(buf[10:14], uint32(needle.NewCRC(payload)))
+	copy(buf[bitrotHeaderSize:], payload)
+
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, buf, 0644); err != nil {
+		return fmt.Errorf("write bitrot sidecar tmp %s: %w", tmp, err)
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("rename bitrot sidecar %s: %w", path, err)
+	}
+	return nil
+}
+
+// LoadBitrotSidecar reads and self-integrity-checks a sidecar file. It returns
+// the parsed message, or an error if the file is missing, truncated, has a bad
+// magic/version, or fails the payload CRC. A self-integrity failure is a
+// sidecar-integrity problem (caller maps it to BitrotInvalid), never a shard
+// corruption signal.
+func LoadBitrotSidecar(path string) (*volume_server_pb.EcBitrotProtection, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	if len(data) < bitrotHeaderSize {
+		return nil, fmt.Errorf("bitrot sidecar %s too short (%d bytes)", path, len(data))
+	}
+	if magic := binary.BigEndian.Uint32(data[0:4]); magic != bitrotMagic {
+		return nil, fmt.Errorf("bitrot sidecar %s bad magic %#x", path, magic)
+	}
+	if ver := binary.BigEndian.Uint16(data[4:6]); ver != bitrotFormatVersion {
+		return nil, fmt.Errorf("bitrot sidecar %s unsupported format version %d", path, ver)
+	}
+	payloadLen := binary.BigEndian.Uint32(data[6:10])
+	wantCRC := binary.BigEndian.Uint32(data[10:14])
+	payload := data[bitrotHeaderSize:]
+	if int(payloadLen) != len(payload) {
+		return nil, fmt.Errorf("bitrot sidecar %s length mismatch: header %d, actual %d", path, payloadLen, len(payload))
+	}
+	if got := uint32(needle.NewCRC(payload)); got != wantCRC {
+		return nil, fmt.Errorf("bitrot sidecar %s self-integrity CRC mismatch: header %#x, computed %#x", path, wantCRC, got)
+	}
+	prot := &volume_server_pb.EcBitrotProtection{}
+	if err := proto.Unmarshal(payload, prot); err != nil {
+		return nil, fmt.Errorf("unmarshal bitrot sidecar %s: %w", path, err)
+	}
+	return prot, nil
+}
+
+// ValidateBitrotManifest performs the disk-free manifest/syntax checks that
+// every loader runs: supported algorithm, valid block size, exactly one entry
+// per shard id in the active layout (no duplicates, no out-of-range ids),
+// positive covered_size, and a packed-CRC count consistent with covered_size.
+// It does NOT compare covered_size against on-disk shard lengths — that is a
+// per-node physical check done only for locally-held shards.
+func ValidateBitrotManifest(prot *volume_server_pb.EcBitrotProtection, dataShards, parityShards int) error {
+	if prot.Algorithm != volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C {
+		return fmt.Errorf("unsupported checksum algorithm %v", prot.Algorithm)
+	}
+	bs := int64(prot.BlockSize)
+	if !isPow2MultipleOf1MiB(prot.BlockSize) {
+		return fmt.Errorf("invalid block_size %d (must be a power-of-two multiple of 1 MiB, at most %d)", prot.BlockSize, MaxBitrotBlockSize)
+	}
+	total := dataShards + parityShards
+	if total <= 0 || total > MaxShardCount {
+		return fmt.Errorf("invalid active layout: data=%d parity=%d", dataShards, parityShards)
+	}
+	if len(prot.Shards) != total {
+		return fmt.Errorf("incomplete manifest: %d shard entries, expected %d", len(prot.Shards), total)
+	}
+	seen := make([]bool, MaxShardCount)
+	for _, s := range prot.Shards {
+		if s.ShardId >= uint32(total) {
+			return fmt.Errorf("shard id %d out of range [0,%d)", s.ShardId, total)
+		}
+		if seen[s.ShardId] {
+			return fmt.Errorf("duplicate shard id %d", s.ShardId)
+		}
+		seen[s.ShardId] = true
+		if s.CoveredSize <= 0 {
+			return fmt.Errorf("shard %d has non-positive covered_size %d", s.ShardId, s.CoveredSize)
+		}
+		wantCount := expectedBlockCount(s.CoveredSize, bs)
+		if len(s.BlockCrc32C) != wantCount*4 {
+			return fmt.Errorf("shard %d crc count mismatch: %d bytes, expected %d (covered_size=%d block_size=%d)",
+				s.ShardId, len(s.BlockCrc32C), wantCount*4, s.CoveredSize, prot.BlockSize)
+		}
+	}
+	return nil
+}
+
+// shardChecksums returns the EcShardChecksums entry for a shard id, or nil.
+func shardChecksums(prot *volume_server_pb.EcBitrotProtection, shardId uint32) *volume_server_pb.EcShardChecksums {
+	for _, s := range prot.Shards {
+		if s.ShardId == shardId {
+			return s
+		}
+	}
+	return nil
+}
+
+// verifyShardFileBlocks reads a shard file at path in block_size chunks and
+// compares each block's CRC32C against the manifest entry. It returns the list
+// of mismatching block indices (empty == clean) and a fatal error only for I/O
+// problems or a covered_size/length mismatch (which is itself corruption of the
+// shard). It does not interpret the result — the caller (scrub / rebuild)
+// arbitrates shard-vs-sidecar via Reed-Solomon before acting.
+func verifyShardFileBlocks(path string, entry *volume_server_pb.EcShardChecksums, blockSize int64) (mismatched []int, err error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	fi, err := f.Stat()
+	if err != nil {
+		return nil, err
+	}
+	if fi.Size() != entry.CoveredSize {
+		// Length drift (truncation or unexpected trailing bytes) is shard
+		// corruption: report every block as mismatched so the caller treats the
+		// shard as bad.
+		want := unpackUint32LE(entry.BlockCrc32C)
+		all := make([]int, len(want))
+		for i := range all {
+			all[i] = i
+		}
+		return all, nil
+	}
+	want := unpackUint32LE(entry.BlockCrc32C)
+	buf := make([]byte, blockSize)
+	var offset int64
+	for i := 0; i < len(want); i++ {
+		toRead := blockSize
+		if rem := entry.CoveredSize - offset; rem < toRead {
+			toRead = rem
+		}
+		// os.File.ReadAt fills the slice or returns an error; a full read that
+		// lands exactly on EOF may still report io.EOF, which is not corruption.
+		// A short read (n < toRead) means the shard is truncated and the EOF is
+		// propagated as an error.
+		n, rerr := f.ReadAt(buf[:toRead], offset)
+		if rerr == io.EOF && int64(n) == toRead {
+			rerr = nil
+		}
+		if rerr != nil {
+			return nil, rerr
+		}
+		if uint32(needle.NewCRC(buf[:n])) != want[i] {
+			mismatched = append(mismatched, i)
+		}
+		offset += int64(n)
+	}
+	return mismatched, nil
+}
+
+// ComputeProtectionFromShards builds a complete EcBitrotProtection by reading
+// the on-disk bytes of every shard for a generation — the trust-on-first-use
+// backfill primitive for EC volumes that were encoded before this feature. It
+// requires EVERY shard to be reachable (locally or in additionalDirs) and
+// returns an error otherwise, so a partial sidecar is never produced. This is
+// the per-holder building block a coordinator assembles across servers; on a
+// server that holds (or can reach) all shards it yields a complete manifest
+// directly. Caveat: it blesses whatever bytes exist now and cannot detect
+// pre-baseline corruption.
+func ComputeProtectionFromShards(baseFileName string, ctx *ECContext, generation uint32, additionalDirs []string) (*volume_server_pb.EcBitrotProtection, error) {
+	shards := make([]*volume_server_pb.EcShardChecksums, 0, ctx.Total())
+	for id := 0; id < ctx.Total(); id++ {
+		path := findShardFile(baseFileName, ctx.ToExt(id), additionalDirs)
+		if path == "" {
+			return nil, fmt.Errorf("bitrot backfill: shard %d missing for %s; refusing to write a partial sidecar", id, baseFileName)
+		}
+		covered, packed, err := computeShardFileCRCs(path, BitrotBlockSize)
+		if err != nil {
+			return nil, fmt.Errorf("bitrot backfill: read shard %d (%s): %w", id, path, err)
+		}
+		shards = append(shards, &volume_server_pb.EcShardChecksums{
+			ShardId:     uint32(id),
+			CoveredSize: covered,
+			BlockCrc32C: packed,
+		})
+	}
+	return &volume_server_pb.EcBitrotProtection{
+		Algorithm:  volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+		BlockSize:  uint32(BitrotBlockSize),
+		Generation: generation,
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards:   uint32(ctx.DataShards),
+			ParityShards: uint32(ctx.ParityShards),
+			BlockSize:    ctx.BlockSize,
+		},
+		Shards:     shards,
+		EncodeUuid: NewEncodeUUID(),
+	}, nil
+}
+
+// computeShardFileCRCs reads a shard file sequentially and returns its size and
+// packed per-block CRC32C array.
+func computeShardFileCRCs(path string, blockSize int64) (coveredSize int64, packed []byte, err error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return 0, nil, err
+	}
+	defer f.Close()
+	b := newShardChecksumBuilder(blockSize)
+	buf := make([]byte, blockSize)
+	for {
+		n, rerr := f.Read(buf)
+		if n > 0 {
+			b.write(buf[:n])
+		}
+		if rerr == io.EOF {
+			break
+		}
+		if rerr != nil {
+			return 0, nil, rerr
+		}
+		if n == 0 {
+			break
+		}
+	}
+	coveredSize, packed = b.finalize()
+	return coveredSize, packed, nil
+}
+
+// BitrotProtection returns the active-generation bitrot protection and its
+// status. The returned message is read-only; callers must not mutate it.
+func (ev *EcVolume) BitrotProtection() (*volume_server_pb.EcBitrotProtection, BitrotStatus) {
+	ev.bitrotLock.RLock()
+	defer ev.bitrotLock.RUnlock()
+	return ev.bitrot, ev.bitrotStatus
+}
+
+// ReloadBitrotSidecar re-resolves the checksum sidecar for a volume that is
+// already mounted — a shard delivery can bring the manifest with it, and the
+// receive path only writes the file, so without this the in-memory volume
+// keeps the protection state it resolved at mount (off) until a remount.
+func (ev *EcVolume) ReloadBitrotSidecar(additionalDirs ...string) {
+	if err := ev.loadActiveBitrotSidecar(additionalDirs...); err != nil {
+		glog.Warningf("ec volume %d: reload bitrot sidecar: %v", ev.VolumeId, err)
+	}
+}
+
+// loadActiveBitrotSidecar loads the generation-0 checksum sidecar into the
+// volume. Best-effort: any failure leaves protection off/invalid without
+// failing the mount. OSS only produces generation-0 (fresh-encode) sidecars.
+func (ev *EcVolume) loadActiveBitrotSidecar(additionalDirs ...string) error {
+	return ev.loadBitrotForGeneration(0, additionalDirs...)
+}
+
+// loadBitrotForGeneration loads and validates the sidecar describing generation
+// `generation`, setting ev.bitrot/ev.bitrotStatus. Called at mount. Absent or
+// generation-mismatched => BitrotOff; self-integrity/manifest failure =>
+// BitrotInvalid; usable => BitrotOn.
+//
+// A sidecar written FOR THIS generation that disagrees with the volume's shard
+// geometry is different in kind from those: both files record the layout the
+// generation was encoded with, so a disagreement means one of them is wrong and
+// reads through the other would land at the wrong shard offsets. That returns
+// an error and fails the mount instead of quietly dropping to unprotected
+// reads.
+func (ev *EcVolume) loadBitrotForGeneration(generation uint32, additionalDirs ...string) error {
+	ev.bitrotLock.Lock()
+	defer ev.bitrotLock.Unlock()
+	ev.bitrot = nil
+	ev.bitrotStatus = BitrotOff
+
+	if ev.ECContext == nil {
+		return nil
+	}
+	path := findBitrotSidecar(generation, ev.DataBaseFileName(), ev.IndexBaseFileName(), additionalDirs...)
+	if path == "" {
+		return nil
+	}
+	prot, err := LoadBitrotSidecar(path)
+	if err != nil {
+		glog.Warningf("ec volume %d: bitrot sidecar %s self-integrity failed: %v", ev.VolumeId, path, err)
+		ev.bitrotStatus = BitrotInvalid
+		return nil
+	}
+	if prot.Generation != generation {
+		return nil // not for this generation -> off, not corruption
+	}
+	if prot.EcShardConfig == nil {
+		return nil // records no geometry -> nothing to contradict
+	}
+	if int(prot.EcShardConfig.DataShards) != ev.ECContext.DataShards ||
+		int(prot.EcShardConfig.ParityShards) != ev.ECContext.ParityShards ||
+		prot.EcShardConfig.BlockSize != ev.ECContext.BlockSize {
+		return fmt.Errorf("ec volume %d generation %d: %s records layout %d+%d block %d but the volume is mounted as %d+%d block %d; refusing to serve one of the two layouts",
+			ev.VolumeId, generation, path,
+			prot.EcShardConfig.DataShards, prot.EcShardConfig.ParityShards, prot.EcShardConfig.BlockSize,
+			ev.ECContext.DataShards, ev.ECContext.ParityShards, ev.ECContext.BlockSize)
+	}
+	if err := ValidateBitrotManifest(prot, ev.ECContext.DataShards, ev.ECContext.ParityShards); err != nil {
+		glog.Warningf("ec volume %d: bitrot sidecar %s manifest invalid: %v", ev.VolumeId, path, err)
+		ev.bitrotStatus = BitrotInvalid
+		return nil
+	}
+	ev.bitrot = prot
+	ev.bitrotStatus = BitrotOn
+	glog.V(1).Infof("ec volume %d: loaded bitrot protection generation %d (%d shards, block_size %d)",
+		ev.VolumeId, generation, len(prot.Shards), prot.BlockSize)
+	return nil
+}
+
+// layoutFromSidecar resolves a volume's EC layout from its generation-0
+// checksum sidecar, searching the data base and then the index base. A split
+// -dir/-dir.idx layout keeps the sidecar with the index, so probing the data
+// base alone reports "absent" for a volume that has the record right there —
+// and absent is the one answer that selects the legacy 10+4 layout.
+//
+// The three answers stay distinct: (nil, false, nil) means genuinely absent
+// and the caller may fall back to the defaults; a non-nil error means the
+// record exists but cannot establish the layout, which must fail rather than
+// default; otherwise the config is usable.
+func layoutFromSidecar(dataBaseFileName, indexBaseFileName string) (cfg *volume_server_pb.EcShardConfig, found bool, err error) {
+	path := findBitrotSidecar(0, dataBaseFileName, indexBaseFileName)
+	if path == "" {
+		return nil, false, nil
+	}
+	cfg, err = EcShardConfigFromSidecarPath(path)
+	return cfg, true, err
+}
+
+// EcShardConfigFromSidecar reads the generation-0 bitrot sidecar's record of a
+// volume's EC config. The three answers are distinct on purpose: absent means
+// the volume may genuinely predate the sidecar, which is the only case a caller
+// may answer with the legacy layout; present-but-unusable means the one
+// surviving record of the geometry is corrupt, and guessing from there returns
+// wrong bytes rather than no bytes.
+func EcShardConfigFromSidecar(baseFileName string) (cfg *volume_server_pb.EcShardConfig, found bool, err error) {
+	path := BitrotSidecarPath(baseFileName, 0)
+	if _, statErr := os.Stat(path); statErr != nil {
+		if os.IsNotExist(statErr) {
+			return nil, false, nil
+		}
+		return nil, false, fmt.Errorf("stat %s: %w", path, statErr)
+	}
+	cfg, err = EcShardConfigFromSidecarPath(path)
+	return cfg, true, err
+}
+
+// EcShardConfigFromSidecarPath is EcShardConfigFromSidecar for a sidecar whose
+// path the caller already resolved — the rebuild finds it across a multi-disk
+// server's directories, not only next to the base name.
+func EcShardConfigFromSidecarPath(path string) (*volume_server_pb.EcShardConfig, error) {
+	prot, loadErr := LoadBitrotSidecar(path)
+	if loadErr != nil {
+		return nil, fmt.Errorf("read %s: %w", path, loadErr)
+	}
+	// The un-suffixed sidecar describes generation 0 and nothing else; one
+	// stamped for another generation is not a record of these shards.
+	if prot.GetGeneration() != 0 {
+		return nil, fmt.Errorf("%s records generation %d, not generation 0", path, prot.GetGeneration())
+	}
+	cfg := prot.GetEcShardConfig()
+	if cfg == nil {
+		return nil, fmt.Errorf("%s records no EC config", path)
+	}
+	if !ValidEcShardCounts(cfg.GetDataShards(), cfg.GetParityShards()) {
+		return nil, fmt.Errorf("%s records invalid shard counts %d+%d",
+			path, cfg.GetDataShards(), cfg.GetParityShards())
+	}
+	if bsErr := ValidateBlockSize(cfg.GetBlockSize()); bsErr != nil {
+		return nil, fmt.Errorf("%s: %w", path, bsErr)
+	}
+	return cfg, nil
+}
+
+// RemoveBitrotSidecars removes the legacy <base>.ecsum and any versioned
+// <base>.ecsum.v<N> sidecars for a base file name. Best-effort.
+func RemoveBitrotSidecars(base string) {
+	os.Remove(base + BitrotSidecarExt)
+	if matches, _ := filepath.Glob(base + BitrotSidecarExt + ".v*"); matches != nil {
+		for _, m := range matches {
+			os.Remove(m)
+		}
+	}
+}
+
+// FindBitrotSidecar is findBitrotSidecar for callers outside this package. It
+// answers "does this volume already have a manifest, and where" — a question
+// that cannot be settled from one base name, because a split -dir/-dir.idx
+// layout keeps the sidecar with the index and a multi-disk server may keep it
+// on a sibling. Returns "" when no candidate exists.
+func FindBitrotSidecar(generation uint32, dataBase, indexBase string, additionalDirs ...string) string {
+	return findBitrotSidecar(generation, dataBase, indexBase, additionalDirs...)
+}
+
+// findBitrotSidecar resolves the sidecar path for a generation, searching the
+// data base, the index base, and any additional directories — mirroring how
+// shard/.vif lookups handle split data/idx layouts and per-disk mirrors.
+func findBitrotSidecar(generation uint32, dataBase, indexBase string, additionalDirs ...string) string {
+	candidates := []string{
+		BitrotSidecarPath(dataBase, generation),
+		BitrotSidecarPath(indexBase, generation),
+	}
+	base := filepath.Base(dataBase)
+	for _, dir := range additionalDirs {
+		candidates = append(candidates, BitrotSidecarPath(filepath.Join(dir, base), generation))
+	}
+	for _, c := range candidates {
+		if c == "" {
+			continue
+		}
+		if _, err := os.Stat(c); err == nil {
+			return c
+		}
+	}
+	return ""
+}

weed/storage/erasure_coding/ec_bitrot_interop_test.go

@@ -0,0 +1,60 @@
+package erasure_coding
+
+import (
+	"encoding/hex"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+// interopSample is the fully-deterministic sidecar shared with the Rust port's
+// ec_bitrot.rs `sample()`. Both binaries must serialize it to identical bytes.
+func interopSample() *volume_server_pb.EcBitrotProtection {
+	return &volume_server_pb.EcBitrotProtection{
+		Algorithm: volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+		BlockSize: uint32(DefaultBitrotBlockSize),
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards:   10,
+			ParityShards: 4,
+		},
+		Shards: []*volume_server_pb.EcShardChecksums{
+			{ShardId: 0, CoveredSize: 1024 * 1024, BlockCrc32C: packUint32LE([]uint32{0x01020304})},
+			{ShardId: 1, CoveredSize: 1024 * 1024, BlockCrc32C: packUint32LE([]uint32{0x05060708})},
+		},
+		EncodeUuid: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
+	}
+}
+
+// canonicalInteropHex is the exact on-disk bytes of interopSample(). The Rust
+// port's ec_bitrot.rs test_byte_exact_go_interop pins the SAME constant, so a
+// format change in EITHER binary (new always-serialized field, renumber,
+// EcShardConfig default flip, protobuf canonicalization change) fails that
+// binary's test instead of silently desyncing a Go-written sidecar from a
+// Rust-written one. If you intentionally change the format, regenerate this
+// (run with -v to print the hex) and update the Rust constant in lock-step.
+const canonicalInteropHex = "45435355000100000039cc1b826a080110808080082204080a10042a0a108080401a04040302012a0c0801108080401a04080706053210000102030405060708090a0b0c0d0e0f"
+
+// TestBitrotSidecarBytes_RustInterop pins the on-disk bytes so a Go-side format
+// drift fails here, and the Rust port asserts the same constant (cross-binary
+// sidecar interop).
+func TestBitrotSidecarBytes_RustInterop(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "v1.ecsum")
+	if err := SaveBitrotSidecar(path, interopSample()); err != nil {
+		t.Fatalf("save: %v", err)
+	}
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read: %v", err)
+	}
+	if got := hex.EncodeToString(data); got != canonicalInteropHex {
+		t.Fatalf("bitrot sidecar bytes drifted from the cross-binary canonical form;\n got=%s\nwant=%s\n(regenerate and update the Rust constant in lock-step)", got, canonicalInteropHex)
+	}
+
+	// Round-trips through the loader.
+	if _, err := LoadBitrotSidecar(path); err != nil {
+		t.Fatalf("load: %v", err)
+	}
+}

weed/storage/erasure_coding/ec_bitrot_test.go

@@ -0,0 +1,363 @@
+package erasure_coding
+
+import (
+	"math/rand"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+func TestBitrotSidecarRoundTripAndSelfIntegrity(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "vol.ecsum")
+	prot := &volume_server_pb.EcBitrotProtection{
+		Algorithm:     volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+		BlockSize:     DefaultBitrotBlockSize,
+		Generation:    0,
+		EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4},
+		Shards: []*volume_server_pb.EcShardChecksums{
+			{ShardId: 0, CoveredSize: 5, BlockCrc32C: packUint32LE([]uint32{123})},
+		},
+		EncodeUuid: NewEncodeUUID(),
+	}
+	if err := SaveBitrotSidecar(path, prot); err != nil {
+		t.Fatalf("save: %v", err)
+	}
+	got, err := LoadBitrotSidecar(path)
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+	if got.BlockSize != prot.BlockSize || len(got.Shards) != 1 || got.Shards[0].CoveredSize != 5 {
+		t.Fatalf("roundtrip mismatch: %+v", got)
+	}
+
+	// Corrupt a payload byte and confirm the self-integrity check trips.
+	data, _ := os.ReadFile(path)
+	data[bitrotHeaderSize+2] ^= 0xff
+	if err := os.WriteFile(path, data, 0644); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := LoadBitrotSidecar(path); err == nil {
+		t.Fatalf("expected self-integrity failure after payload corruption")
+	}
+}
+
+func TestShardChecksumBuilderCrossesBoundaries(t *testing.T) {
+	blockSize := int64(1 << 20)
+	data := make([]byte, 0, 3*blockSize+12345)
+	r := rand.New(rand.NewSource(1))
+	data = data[:cap(data)]
+	r.Read(data)
+
+	// Feed in odd-sized chunks that cross block boundaries.
+	b := newShardChecksumBuilder(blockSize)
+	for off := 0; off < len(data); off += 7777 {
+		end := off + 7777
+		if end > len(data) {
+			end = len(data)
+		}
+		b.write(data[off:end])
+	}
+	covered, packed := b.finalize()
+	if covered != int64(len(data)) {
+		t.Fatalf("covered=%d want=%d", covered, len(data))
+	}
+	got := unpackUint32LE(packed)
+
+	// Compare to a direct per-block CRC.
+	want := []uint32{}
+	for off := int64(0); off < int64(len(data)); off += blockSize {
+		end := off + blockSize
+		if end > int64(len(data)) {
+			end = int64(len(data))
+		}
+		want = append(want, uint32(needle.NewCRC(data[off:end])))
+	}
+	if len(got) != len(want) {
+		t.Fatalf("block count got=%d want=%d", len(got), len(want))
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Fatalf("block %d crc mismatch got=%x want=%x", i, got[i], want[i])
+		}
+	}
+}
+
+func TestValidateBitrotManifest(t *testing.T) {
+	mk := func(mut func(*volume_server_pb.EcBitrotProtection)) *volume_server_pb.EcBitrotProtection {
+		p := &volume_server_pb.EcBitrotProtection{
+			Algorithm:     volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+			BlockSize:     DefaultBitrotBlockSize,
+			EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 2, ParityShards: 1},
+		}
+		for id := 0; id < 3; id++ {
+			p.Shards = append(p.Shards, &volume_server_pb.EcShardChecksums{
+				ShardId: uint32(id), CoveredSize: 5, BlockCrc32C: packUint32LE([]uint32{1}),
+			})
+		}
+		if mut != nil {
+			mut(p)
+		}
+		return p
+	}
+	if err := ValidateBitrotManifest(mk(nil), 2, 1); err != nil {
+		t.Fatalf("valid manifest rejected: %v", err)
+	}
+	bad := map[string]func(*volume_server_pb.EcBitrotProtection){
+		"missing shard":        func(p *volume_server_pb.EcBitrotProtection) { p.Shards = p.Shards[:2] },
+		"out of range id":      func(p *volume_server_pb.EcBitrotProtection) { p.Shards[2].ShardId = 9 },
+		"duplicate id":         func(p *volume_server_pb.EcBitrotProtection) { p.Shards[2].ShardId = 0 },
+		"zero covered":         func(p *volume_server_pb.EcBitrotProtection) { p.Shards[0].CoveredSize = 0 },
+		"bad crc count":        func(p *volume_server_pb.EcBitrotProtection) { p.Shards[0].BlockCrc32C = []byte{1, 2, 3} },
+		"bad block size":       func(p *volume_server_pb.EcBitrotProtection) { p.BlockSize = 3 << 20 },   // not power of two
+		"oversized block size": func(p *volume_server_pb.EcBitrotProtection) { p.BlockSize = 128 << 20 }, // power of two but > MaxBitrotBlockSize
+		"bad algorithm": func(p *volume_server_pb.EcBitrotProtection) {
+			p.Algorithm = volume_server_pb.ChecksumAlgorithm_CHECKSUM_NONE
+		},
+	}
+	for name, mut := range bad {
+		if err := ValidateBitrotManifest(mk(mut), 2, 1); err == nil {
+			t.Errorf("%s: expected validation error, got nil", name)
+		}
+	}
+}
+
+// writeRandomDat creates a <base>.dat of n bytes for encoding.
+func writeRandomDat(t *testing.T, base string, n int) {
+	t.Helper()
+	data := make([]byte, n)
+	rand.New(rand.NewSource(42)).Read(data)
+	if err := os.WriteFile(base+".dat", data, 0644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestEncodeProducesVerifiableSidecar(t *testing.T) {
+	old := BitrotBlockSize
+	BitrotBlockSize = 1 << 20
+	t.Cleanup(func() { BitrotBlockSize = old })
+
+	dir := t.TempDir()
+	base := filepath.Join(dir, "7")
+	writeRandomDat(t, base, 6*1024*1024) // ~6 MiB
+
+	ctx := &ECContext{DataShards: 10, ParityShards: 4}
+	prot, err := WriteEcFiles(base, ctx)
+	if err != nil {
+		t.Fatalf("encode: %v", err)
+	}
+	if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), prot); err != nil {
+		t.Fatalf("save sidecar: %v", err)
+	}
+	if err := ValidateBitrotManifest(prot, 10, 4); err != nil {
+		t.Fatalf("encode produced invalid manifest: %v", err)
+	}
+
+	// Every shard file verifies clean against the sidecar right after encode.
+	for id := 0; id < ctx.Total(); id++ {
+		entry := shardChecksums(prot, uint32(id))
+		mism, verr := verifyShardFileBlocks(base+ctx.ToExt(id), entry, int64(prot.BlockSize))
+		if verr != nil {
+			t.Fatalf("verify shard %d: %v", id, verr)
+		}
+		if len(mism) != 0 {
+			t.Fatalf("shard %d unexpectedly mismatched at encode time: %v", id, mism)
+		}
+	}
+
+	// Corrupt one byte in shard 12 (a parity shard) and confirm detection.
+	corruptOneByte(t, base+ctx.ToExt(12))
+	entry := shardChecksums(prot, 12)
+	mism, verr := verifyShardFileBlocks(base+ctx.ToExt(12), entry, int64(prot.BlockSize))
+	if verr != nil {
+		t.Fatalf("verify corrupt shard: %v", verr)
+	}
+	if len(mism) == 0 {
+		t.Fatalf("expected to detect corruption in parity shard 12")
+	}
+}
+
+func TestRebuildExcludesCorruptPresentShard(t *testing.T) {
+	old := BitrotBlockSize
+	BitrotBlockSize = 1 << 20
+	t.Cleanup(func() { BitrotBlockSize = old })
+
+	dir := t.TempDir()
+	base := filepath.Join(dir, "9")
+	writeRandomDat(t, base, 6*1024*1024)
+
+	ctx := &ECContext{DataShards: 10, ParityShards: 4}
+	prot, err := WriteEcFiles(base, ctx)
+	if err != nil {
+		t.Fatalf("encode: %v", err)
+	}
+	if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), prot); err != nil {
+		t.Fatalf("save sidecar: %v", err)
+	}
+
+	// Snapshot a good copy of shard 3, then delete shard 0 (missing) and corrupt
+	// shard 3 (present-but-corrupt). Rebuild must regenerate both correctly.
+	good3, _ := os.ReadFile(base + ctx.ToExt(3))
+	if err := os.Remove(base + ctx.ToExt(0)); err != nil {
+		t.Fatal(err)
+	}
+	corruptOneByte(t, base+ctx.ToExt(3))
+
+	generated, err := RebuildEcFiles(base, ctx, false)
+	if err != nil {
+		t.Fatalf("rebuild: %v", err)
+	}
+	if !slices.Contains(generated, 0) || !slices.Contains(generated, 3) {
+		t.Fatalf("expected shards 0 and 3 regenerated, got %v", generated)
+	}
+
+	// Shard 3 should now byte-match its original good content, and every shard
+	// verifies clean against the sidecar.
+	now3, _ := os.ReadFile(base + ctx.ToExt(3))
+	if string(now3) != string(good3) {
+		t.Fatalf("rebuilt shard 3 does not match original good content")
+	}
+	for id := 0; id < ctx.Total(); id++ {
+		entry := shardChecksums(prot, uint32(id))
+		mism, verr := verifyShardFileBlocks(base+ctx.ToExt(id), entry, int64(prot.BlockSize))
+		if verr != nil || len(mism) != 0 {
+			t.Fatalf("post-rebuild shard %d not clean: mism=%v err=%v", id, mism, verr)
+		}
+	}
+}
+
+func TestComputeProtectionFromShardsBackfill(t *testing.T) {
+	old := BitrotBlockSize
+	BitrotBlockSize = 1 << 20
+	t.Cleanup(func() { BitrotBlockSize = old })
+
+	dir := t.TempDir()
+	base := filepath.Join(dir, "11")
+	writeRandomDat(t, base, 6*1024*1024)
+	ctx := &ECContext{DataShards: 10, ParityShards: 4}
+	if _, err := WriteEcFiles(base, ctx); err != nil {
+		t.Fatalf("encode: %v", err)
+	}
+
+	// Backfill from the on-disk shards (no encode pass) yields a complete,
+	// valid manifest that every shard verifies clean against.
+	prot, err := ComputeProtectionFromShards(base, ctx, 0, nil)
+	if err != nil {
+		t.Fatalf("backfill: %v", err)
+	}
+	if err := ValidateBitrotManifest(prot, 10, 4); err != nil {
+		t.Fatalf("backfill manifest invalid: %v", err)
+	}
+	for id := 0; id < ctx.Total(); id++ {
+		entry := shardChecksums(prot, uint32(id))
+		mism, verr := verifyShardFileBlocks(base+ctx.ToExt(id), entry, int64(prot.BlockSize))
+		if verr != nil || len(mism) != 0 {
+			t.Fatalf("backfilled shard %d not clean: mism=%v err=%v", id, mism, verr)
+		}
+	}
+
+	// A missing shard must abort backfill (never write a partial manifest).
+	if err := os.Remove(base + ctx.ToExt(5)); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ComputeProtectionFromShards(base, ctx, 0, nil); err == nil {
+		t.Fatalf("expected backfill error with a missing shard")
+	}
+}
+
+func TestRebuildFailClosedCleansUpGeneratedShard(t *testing.T) {
+	old := BitrotBlockSize
+	BitrotBlockSize = 1 << 20
+	t.Cleanup(func() { BitrotBlockSize = old })
+
+	dir := t.TempDir()
+	base := filepath.Join(dir, "13")
+	writeRandomDat(t, base, 6*1024*1024)
+	ctx := &ECContext{DataShards: 10, ParityShards: 4}
+	prot, err := WriteEcFiles(base, ctx)
+	if err != nil {
+		t.Fatalf("encode: %v", err)
+	}
+
+	// Tamper one block CRC of shard 0 but re-save so the header self-integrity and
+	// manifest stay valid: a well-formed but wrong ("stale") sidecar that loads as ON.
+	for _, s := range prot.Shards {
+		if s.ShardId == 0 && len(s.BlockCrc32C) >= 1 {
+			s.BlockCrc32C[0] ^= 0xff
+		}
+	}
+	if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), prot); err != nil {
+		t.Fatalf("save sidecar: %v", err)
+	}
+
+	// Delete shard 0 so the rebuild regenerates it; the regenerated (correct)
+	// bytes will not match the tampered sidecar entry -> post-verify fails closed.
+	shard0 := base + ctx.ToExt(0)
+	if err := os.Remove(shard0); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, rerr := RebuildEcFiles(base, ctx, false); rerr == nil {
+		t.Fatalf("expected rebuild to fail closed on the stale sidecar")
+	}
+	// The genuinely-missing shard 0 must NOT be published: fail-closed cleanup
+	// returns it to missing.
+	if _, err := os.Stat(shard0); !os.IsNotExist(err) {
+		t.Fatalf("shard 0 should have been cleaned up after a failed rebuild, err=%v", err)
+	}
+
+	// With the unsafe override, the rebuild proceeds and regenerates shard 0.
+	if _, rerr := RebuildEcFiles(base, ctx, true); rerr != nil {
+		t.Fatalf("rebuild with unsafeIgnoreSidecar should succeed: %v", rerr)
+	}
+	if _, err := os.Stat(shard0); err != nil {
+		t.Fatalf("shard 0 should exist after unsafe rebuild: %v", err)
+	}
+}
+
+func TestRemoveBitrotSidecars(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "5")
+	// Legacy + two versioned sidecars, plus an unrelated file that must survive.
+	for _, p := range []string{base + ".ecsum", base + ".ecsum.v1", base + ".ecsum.v7"} {
+		if err := os.WriteFile(p, []byte("x"), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	keep := base + ".ec00"
+	if err := os.WriteFile(keep, []byte("x"), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	RemoveBitrotSidecars(base)
+
+	for _, p := range []string{base + ".ecsum", base + ".ecsum.v1", base + ".ecsum.v7"} {
+		if _, err := os.Stat(p); !os.IsNotExist(err) {
+			t.Errorf("expected %s removed, err=%v", p, err)
+		}
+	}
+	if _, err := os.Stat(keep); err != nil {
+		t.Errorf("unrelated file %s should survive: %v", keep, err)
+	}
+}
+
+func corruptOneByte(t *testing.T, path string) {
+	t.Helper()
+	f, err := os.OpenFile(path, os.O_RDWR, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	var b [1]byte
+	if _, err := f.ReadAt(b[:], 0); err != nil {
+		t.Fatal(err)
+	}
+	b[0] ^= 0xff
+	if _, err := f.WriteAt(b[:], 0); err != nil {
+		t.Fatal(err)
+	}
+}

weed/storage/erasure_coding/ec_consistency_test.go

@@ -0,0 +1,224 @@
+package erasure_coding
+
+import (
+	"bytes"
+	"crypto/rand"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// TestEcConsistency_WritesBetweenEncodeAndEcx reproduces a race condition that
+// existed in VolumeEcShardsGenerate before the fix in this PR.
+//
+// Previously, the order was:
+//  1. WriteEcFiles(baseFileName, ecCtx)        — EC shards from .dat
+//  2. WriteSortedFileFromIdx(v.IndexFileName(), ".ecx")   — .ecx from .idx
+//
+// If a write appended data to .dat/.idx between steps 1 and 2, the .ecx would
+// have entries pointing to data that doesn't exist in the EC shards.
+//
+// The fix reverses the order (write .ecx first, then generate EC shards), so
+// that .ecx is always a subset of what the EC shards contain.
+//
+// This test simulates the old buggy sequence to validate that the problem is real.
+func TestEcConsistency_WritesBetweenEncodeAndEcx(t *testing.T) {
+	dir := t.TempDir()
+	baseFileName := dir + "/consistency"
+
+	ctx := NewDefaultECContext("", 0)
+
+	// Phase 1: Create initial .dat and .idx with known data
+	datSize := int64(largeBlockSize*DataShardsCount + smallBlockSize*DataShardsCount*3) // 1 large row + 3 small rows
+	originalData := make([]byte, datSize)
+	rand.Read(originalData)
+
+	err := os.WriteFile(baseFileName+".dat", originalData, 0644)
+	require.NoError(t, err)
+
+	// Create a minimal .idx with one entry pointing to the data
+	createTestIdx(t, baseFileName+".idx", []idxEntry{
+		{id: 1, offset: 0, size: types.Size(datSize)},
+	})
+
+	// Phase 2: EC encode — generates .ec00-.ec13 from current .dat
+	_, err = generateEcFiles(baseFileName, int(smallBlockSize), largeBlockSize, smallBlockSize, ctx)
+	require.NoError(t, err, "EC encoding")
+
+	// Phase 3: SIMULATE a write between EC encoding and .ecx generation
+	// (reproducing the old buggy order where .ecx was generated after EC shards)
+	extraData := make([]byte, 5000)
+	rand.Read(extraData)
+
+	f, err := os.OpenFile(baseFileName+".dat", os.O_WRONLY|os.O_APPEND, 0644)
+	require.NoError(t, err)
+	_, err = f.Write(extraData)
+	require.NoError(t, err)
+	f.Close()
+
+	// Update .idx with the new entry
+	createTestIdx(t, baseFileName+".idx", []idxEntry{
+		{id: 1, offset: 0, size: types.Size(datSize)},
+		{id: 2, offset: datSize, size: types.Size(len(extraData))},
+	})
+
+	// Phase 4: Generate .ecx from the UPDATED .idx (as the old buggy code did)
+	err = WriteSortedFileFromIdx(baseFileName, ".ecx")
+	require.NoError(t, err, "WriteSortedFileFromIdx")
+
+	// Phase 5: Now try to read needle 2 via EC shards — it should fail
+	// because the EC shards were generated from the OLD .dat (without the extra data)
+	ecFiles, err := openEcFiles(baseFileName, true, ctx)
+	require.NoError(t, err)
+	defer closeEcFiles(ecFiles)
+
+	ecStat, err := ecFiles[0].Stat()
+	require.NoError(t, err)
+	shardSize := ecStat.Size()
+
+	// Read needle 2 (the one added after EC encoding) using LocateData.
+	// Use shardSize-1 to simulate the ecdFileSize fallback path used by
+	// LocateEcShardNeedleInterval when datFileSize is unavailable.
+	actualSize := needle.GetActualSize(types.Size(len(extraData)), needle.Version3)
+	intervals := LocateData(largeBlockSize, smallBlockSize, shardSize-1, datSize, types.Size(actualSize))
+
+	t.Logf("Trying to read needle 2 at offset %d size %d from EC shards (shardSize=%d)", datSize, actualSize, shardSize)
+	t.Logf("Intervals: %+v", intervals)
+
+	// Try to read — this will either fail with an error (offset out of bounds)
+	// or return garbage data (the padded zeros from EC encoding)
+	ecData, readErr := assembleFromIntervalsAllowError(ecFiles, intervals, largeBlockSize, smallBlockSize)
+
+	if readErr != nil {
+		t.Logf("CONFIRMED: Read error for needle written after EC encoding: %v", readErr)
+	} else {
+		// If we got data, it should be zeros (padding) or garbage, not the actual extraData
+		isAllZeros := true
+		for _, b := range ecData {
+			if b != 0 {
+				isAllZeros = false
+				break
+			}
+		}
+		if isAllZeros {
+			t.Logf("CONFIRMED: Read returned zero-padded data (EC shards don't have the needle)")
+		} else if !bytes.Equal(ecData[:len(extraData)], extraData) {
+			t.Logf("CONFIRMED: Read returned wrong data (EC shards don't have the needle)")
+		} else {
+			t.Error("UNEXPECTED: Read returned correct data — needle should NOT be in EC shards")
+		}
+	}
+
+	// Phase 6: Verify a small read from the original data still works.
+	// Use the correct shardDatSize (from the original datSize, not the modified one)
+	// to avoid the fallback heuristic issues.
+	shardDatSize := datSize / int64(DataShardsCount)
+	readSize := types.Size(smallBlockSize)
+	intervals1 := LocateData(largeBlockSize, smallBlockSize, shardDatSize, 0, readSize)
+	ecData1, err := assembleFromIntervalsAllowError(ecFiles, intervals1, largeBlockSize, smallBlockSize)
+	require.NoError(t, err, "reading original data from EC shards")
+
+	assert.True(t, bytes.Equal(originalData[:readSize], ecData1),
+		"Original data at offset 0 should match EC shard data")
+	t.Logf("Original data reads correctly from EC shards")
+}
+
+// TestEcConsistency_ExactLargeRowEncoding verifies that generateEcFiles correctly
+// encodes a .dat file whose size is exactly one large row (DataShardsCount *
+// largeBlockSize), producing shards of exactly largeBlockSize each, and that
+// every chunk of the encoded data can be read back correctly via LocateData.
+func TestEcConsistency_ExactLargeRowEncoding(t *testing.T) {
+	dir := t.TempDir()
+	baseFileName := dir + "/exact"
+	ctx := NewDefaultECContext("", 0)
+
+	datSize := int64(largeBlockSize * DataShardsCount) // exactly 1 large row
+	data := make([]byte, datSize)
+	rand.Read(data)
+	err := os.WriteFile(baseFileName+".dat", data, 0644)
+	require.NoError(t, err)
+
+	// EC encode
+	_, err = generateEcFiles(baseFileName, int(smallBlockSize), largeBlockSize, smallBlockSize, ctx)
+	require.NoError(t, err)
+
+	// Check shard sizes — each shard should be exactly largeBlockSize
+	ecFiles, err := openEcFiles(baseFileName, true, ctx)
+	require.NoError(t, err)
+	defer closeEcFiles(ecFiles)
+
+	for i := 0; i < ctx.DataShards; i++ {
+		stat, err := ecFiles[i].Stat()
+		require.NoError(t, err, "stat shard %d", i)
+		assert.Equal(t, int64(largeBlockSize), stat.Size(),
+			"data shard %d should be exactly largeBlockSize", i)
+	}
+
+	// Verify data reads correctly at every smallBlockSize offset via LocateData
+	shardDatSize := datSize / int64(ctx.DataShards)
+	readSize := types.Size(smallBlockSize)
+	for offset := int64(0); offset+int64(readSize) <= datSize; offset += int64(readSize) {
+		intervals := LocateData(largeBlockSize, smallBlockSize, shardDatSize, offset, readSize)
+		ecData, err := assembleFromIntervalsAllowError(ecFiles, intervals, largeBlockSize, smallBlockSize)
+		require.NoError(t, err, "reading at offset %d", offset)
+		expected := data[offset : offset+int64(readSize)]
+		assert.True(t, bytes.Equal(expected, ecData),
+			"data mismatch at offset %d", offset)
+	}
+}
+
+type idxEntry struct {
+	id     types.NeedleId
+	offset int64
+	size   types.Size
+}
+
+func createTestIdx(t *testing.T, filename string, entries []idxEntry) {
+	t.Helper()
+	f, err := os.Create(filename)
+	require.NoError(t, err)
+	defer f.Close()
+
+	buf := make([]byte, types.NeedleMapEntrySize)
+	for _, e := range entries {
+		types.NeedleIdToBytes(buf[:types.NeedleIdSize], e.id)
+		types.OffsetToBytes(buf[types.NeedleIdSize:types.NeedleIdSize+types.OffsetSize], types.ToOffset(e.offset))
+		types.SizeToBytes(buf[types.NeedleIdSize+types.OffsetSize:], e.size)
+		_, err := f.Write(buf)
+		require.NoError(t, err)
+	}
+}
+
+func assembleFromIntervalsAllowError(ecFiles []*os.File, intervals []Interval, large, small int64) ([]byte, error) {
+	var data []byte
+	for _, interval := range intervals {
+		shardId, shardOffset := interval.ToShardIdAndOffset(large, small)
+		if int(shardId) >= len(ecFiles) {
+			return nil, fmt.Errorf("shard %d out of range (have %d files)", shardId, len(ecFiles))
+		}
+		stat, err := ecFiles[shardId].Stat()
+		if err != nil {
+			return nil, fmt.Errorf("stat shard %d: %v", shardId, err)
+		}
+		if shardOffset+int64(interval.Size) > stat.Size() {
+			return nil, fmt.Errorf("read past end of shard %d: offset %d + size %d > fileSize %d",
+				shardId, shardOffset, interval.Size, stat.Size())
+		}
+		chunk := make([]byte, interval.Size)
+		n, err := ecFiles[shardId].ReadAt(chunk, shardOffset)
+		if err != nil {
+			return nil, fmt.Errorf("read shard %d offset %d: %v", shardId, shardOffset, err)
+		}
+		if n != int(interval.Size) {
+			return nil, fmt.Errorf("short read shard %d: got %d want %d", shardId, n, interval.Size)
+		}
+		data = append(data, chunk...)
+	}
+	return data, nil
+}

weed/storage/erasure_coding/ec_context.go

@@ -0,0 +1,94 @@
+package erasure_coding
+
+import (
+	"fmt"
+
+	"github.com/klauspost/reedsolomon"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// ECContext encapsulates erasure coding parameters for encoding/decoding operations
+type ECContext struct {
+	DataShards   int
+	ParityShards int
+	Collection   string
+	VolumeId     needle.VolumeId
+	// BlockSize > 0 selects the uniform block layout: every shard is a single
+	// contiguous block of this many bytes, so consecutive .dat ranges stay on
+	// one shard instead of striping across all of them at 1MiB granularity.
+	// 0 is the legacy two-tier 1GiB/1MiB layout.
+	BlockSize int64
+	// DatFileSize is the length of the .dat the encode actually read, set by
+	// WriteEcFiles from the same measurement it derives BlockSize from. The
+	// .vif must record the two together: they describe one file, and a live
+	// volume can grow between two separate stats.
+	DatFileSize int64
+}
+
+// Total returns the total number of shards (data + parity)
+func (ctx *ECContext) Total() int {
+	return ctx.DataShards + ctx.ParityShards
+}
+
+// ValidEcShardCounts reports whether a recorded (data, parity) pair could
+// describe a real EC volume. The sum is taken in uint64 deliberately: on a
+// 32-bit build `int` is 32 bits, so converting each count first and adding
+// them wraps for values near the uint32 ceiling — 0x7fffffff + 0x7fffffff
+// lands at -2, which slips under the MaxShardCount bound.
+func ValidEcShardCounts(dataShards, parityShards uint32) bool {
+	return dataShards > 0 && parityShards > 0 &&
+		uint64(dataShards)+uint64(parityShards) <= uint64(MaxShardCount)
+}
+
+// LargeBlockSize returns the large-block length of this context's shard
+// layout; nil-safe so callers can pass an unset context for the legacy layout.
+func (ctx *ECContext) LargeBlockSize() int64 {
+	if ctx != nil && ctx.BlockSize > 0 {
+		return ctx.BlockSize
+	}
+	return ErasureCodingLargeBlockSize
+}
+
+// SmallBlockSize returns the small-block length of this context's shard layout.
+func (ctx *ECContext) SmallBlockSize() int64 {
+	if ctx != nil && ctx.BlockSize > 0 {
+		return ctx.BlockSize
+	}
+	return ErasureCodingSmallBlockSize
+}
+
+// NewDefaultECContext creates a context with default 10+4 shard configuration
+func NewDefaultECContext(collection string, volumeId needle.VolumeId) *ECContext {
+	return &ECContext{
+		DataShards:   DataShardsCount,
+		ParityShards: ParityShardsCount,
+		Collection:   collection,
+		VolumeId:     volumeId,
+	}
+}
+
+// BackgroundECContext returns a non-nil placeholder EC context, analogous to
+// context.Background(): pass it to WriteEcFiles / RebuildEcFiles when the caller
+// has no specific layout, rather than a nil context. Its zero Total() is the
+// "unset" signal — WriteEcFiles resolves it to the default ratio and
+// RebuildEcFiles resolves it from the volume's .vif (falling back to default),
+// so the placeholder itself never reaches the encoder. A fresh value is returned
+// each call so callers cannot mutate a shared default.
+func BackgroundECContext() *ECContext {
+	return &ECContext{}
+}
+
+// CreateEncoder creates a Reed-Solomon encoder for this context
+func (ctx *ECContext) CreateEncoder() (reedsolomon.Encoder, error) {
+	return reedsolomon.New(ctx.DataShards, ctx.ParityShards)
+}
+
+// ToExt returns the file extension for a given shard index
+func (ctx *ECContext) ToExt(shardIndex int) string {
+	return fmt.Sprintf(".ec%02d", shardIndex)
+}
+
+// String returns a human-readable representation of the EC configuration
+func (ctx *ECContext) String() string {
+	return fmt.Sprintf("%d+%d (total: %d)", ctx.DataShards, ctx.ParityShards, ctx.Total())
+}

weed/storage/erasure_coding/ec_decoder.go

@@ -0,0 +1,352 @@
+package erasure_coding
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// EcNoLiveEntriesSubstring is used for server/client coordination when ec.decode determines that
+// decoding should be a no-op (all entries are deleted).
+const EcNoLiveEntriesSubstring = "has no live entries"
+
+// HasLiveNeedles returns whether the EC index (.ecx) contains at least one live (non-deleted) entry.
+// This is used by ec.decode to avoid generating an empty normal volume when all entries were deleted.
+func HasLiveNeedles(indexBaseFileName string) (hasLive bool, err error) {
+	err = iterateEcxFile(indexBaseFileName, func(_ types.NeedleId, _ types.Offset, size types.Size) error {
+		if !size.IsDeleted() {
+			hasLive = true
+			return io.EOF // stop early
+		}
+		return nil
+	})
+	return
+}
+
+// write .idx file from .ecx and .ecj files
+func WriteIdxFileFromEcIndex(baseFileName string) (err error) {
+
+	ecxFile, openErr := os.OpenFile(baseFileName+".ecx", os.O_RDONLY, 0644)
+	if openErr != nil {
+		return fmt.Errorf("cannot open ec index %s.ecx: %v", baseFileName, openErr)
+	}
+	defer ecxFile.Close()
+
+	// Write to a temp file and atomically rename into place, so a crash mid-write
+	// never leaves a partial .idx at the final name beside the source shards.
+	idxFileName := baseFileName + ".idx"
+	tmpFileName := idxFileName + ".tmp"
+	idxFile, openErr := os.OpenFile(tmpFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if openErr != nil {
+		return fmt.Errorf("cannot open %s: %v", tmpFileName, openErr)
+	}
+	committed := false
+	defer func() {
+		idxFile.Close()
+		if !committed {
+			os.Remove(tmpFileName)
+		}
+	}()
+
+	if _, err = io.Copy(idxFile, ecxFile); err != nil {
+		return fmt.Errorf("copy ecx to idx for %s: %v", baseFileName, err)
+	}
+
+	err = iterateEcjFile(baseFileName, func(key types.NeedleId) error {
+		bytes := needle_map.ToBytes(key, types.Offset{}, types.TombstoneFileSize)
+		if _, writeErr := idxFile.Write(bytes); writeErr != nil {
+			return writeErr
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// fsync, rename, then fsync the dir so the decoded .idx is durable and
+	// atomically published before the caller deletes the source shards.
+	if err = idxFile.Sync(); err != nil {
+		return fmt.Errorf("sync idx for %s: %v", baseFileName, err)
+	}
+	if err = idxFile.Close(); err != nil {
+		return fmt.Errorf("close idx for %s: %v", baseFileName, err)
+	}
+	if err = os.Rename(tmpFileName, idxFileName); err != nil {
+		return fmt.Errorf("rename idx for %s: %v", baseFileName, err)
+	}
+	if err = util.FsyncDir(filepath.Dir(idxFileName)); err != nil {
+		return fmt.Errorf("fsync dir for %s: %v", baseFileName, err)
+	}
+	committed = true
+	return nil
+}
+
+// FindDatFileSize calculate .dat file size from max offset entry
+// there may be extra deletions after that entry
+// but they are deletions anyway
+// shard0FileName is the actual path of the .ec00 shard file, which on a
+// multi-disk server may sit on a different disk than the EcVolume's own base
+// path — the store registers shards per disk, so the caller must pass the
+// path CollectEcShards resolved rather than deriving it from a base name.
+func FindDatFileSize(shard0FileName, indexBaseFileName string) (datSize int64, err error) {
+
+	version, err := readEcVolumeVersion(shard0FileName)
+	if err != nil {
+		return 0, fmt.Errorf("read ec volume %s version: %v", shard0FileName, err)
+	}
+
+	// Safety: ensure datSize is at least SuperBlockSize. While the caller typically
+	// checks HasLiveNeedles first, this protects against direct calls to FindDatFileSize
+	// when all needles are deleted (see issue #7748).
+	datSize = int64(super_block.SuperBlockSize)
+
+	err = iterateEcxFile(indexBaseFileName, func(key types.NeedleId, offset types.Offset, size types.Size) error {
+
+		if size.IsDeleted() {
+			return nil
+		}
+
+		entryStopOffset := offset.ToActualOffset() + needle.GetActualSize(size, version)
+		if datSize < entryStopOffset {
+			datSize = entryStopOffset
+		}
+
+		return nil
+	})
+
+	return
+}
+
+// VerifyDecodedDatFile checks that a reconstructed .dat is long enough to hold
+// every needle its index references. datFileSize is the extent the EC index
+// describes (see FindDatFileSize), so a shorter file cannot serve the needles
+// past the cut -- and the caller is about to delete the shards that are their
+// only other copy, which turns a short write into data loss rather than a
+// failed decode.
+func VerifyDecodedDatFile(dataBaseFileName string, datFileSize int64) error {
+	datPath := dataBaseFileName + ".dat"
+	stat, err := os.Stat(datPath)
+	if err != nil {
+		return fmt.Errorf("stat decoded %s: %w", datPath, err)
+	}
+	if stat.Size() < datFileSize {
+		return fmt.Errorf("decoded %s is %d bytes, short of the %d its ec index references", datPath, stat.Size(), datFileSize)
+	}
+	return nil
+}
+
+func readEcVolumeVersion(shard0FileName string) (version needle.Version, err error) {
+
+	// find volume version
+	datFile, err := os.OpenFile(shard0FileName, os.O_RDONLY, 0644)
+	if err != nil {
+		return 0, fmt.Errorf("open ec volume %s superblock: %v", shard0FileName, err)
+	}
+	datBackend := backend.NewDiskFile(datFile)
+
+	superBlock, err := super_block.ReadSuperBlock(datBackend)
+	datBackend.Close()
+	if err != nil {
+		return 0, fmt.Errorf("read ec volume %s superblock: %v", shard0FileName, err)
+	}
+
+	return superBlock.Version, nil
+
+}
+
+func iterateEcxFile(baseFileName string, processNeedleFn func(key types.NeedleId, offset types.Offset, size types.Size) error) error {
+	ecxFile, openErr := os.OpenFile(baseFileName+".ecx", os.O_RDONLY, 0644)
+	if openErr != nil {
+		return fmt.Errorf("cannot open ec index %s.ecx: %v", baseFileName, openErr)
+	}
+	defer ecxFile.Close()
+
+	buf := make([]byte, types.NeedleMapEntrySize)
+	for {
+		// .ecx is a sealed index: a partial trailing record means corruption, not a torn append.
+		_, err := io.ReadFull(ecxFile, buf)
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("read ecx %s.ecx: %w", baseFileName, err)
+		}
+		key, offset, size := idx.IdxFileEntry(buf)
+		if processNeedleFn != nil {
+			err = processNeedleFn(key, offset, size)
+		}
+		if err != nil {
+			if err != io.EOF {
+				return err
+			}
+			return nil
+		}
+	}
+
+}
+
+func iterateEcjFile(baseFileName string, processNeedleFn func(key types.NeedleId) error) error {
+	if !util.FileExists(baseFileName + ".ecj") {
+		return nil
+	}
+	ecjFile, openErr := os.OpenFile(baseFileName+".ecj", os.O_RDONLY, 0644)
+	if openErr != nil {
+		return fmt.Errorf("cannot open ec index %s.ecj: %v", baseFileName, openErr)
+	}
+	defer ecjFile.Close()
+
+	buf := make([]byte, types.NeedleIdSize)
+	for {
+		n, err := ecjFile.Read(buf)
+		if n != types.NeedleIdSize {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+		if processNeedleFn != nil {
+			err = processNeedleFn(types.BytesToNeedleId(buf))
+		}
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+	}
+
+}
+
+// WriteDatFile generates .dat from EC shard files (e.g., .ec00 ~ .ec09 for 10+4).
+// datFileSize is the number of bytes to write, i.e. the live data extent from
+// FindDatFileSize. encodedDatFileSize is the .dat size at encode time, which
+// fixed the shard block layout: deletions can move the live extent below the
+// large-block row boundary, and deriving the layout from the shrunk extent
+// would read the shards in the wrong block order. Pass zero when the .vif does
+// not record the encode-time size to infer the layout from the shard size.
+// largeBlockSize/smallBlockSize are the volume's shard block layout, e.g.
+// ctx.LargeBlockSize()/ctx.SmallBlockSize() from its .vif EC config.
+func WriteDatFile(baseFileName string, datFileSize int64, encodedDatFileSize int64, shardFileNames []string, largeBlockSize int64, smallBlockSize int64) error {
+	return writeDatFile(baseFileName, datFileSize, encodedDatFileSize, shardFileNames, largeBlockSize, smallBlockSize)
+}
+
+func writeDatFile(baseFileName string, datFileSize int64, encodedDatFileSize int64, shardFileNames []string, largeBlockSize int64, smallBlockSize int64) error {
+
+	if len(shardFileNames) == 0 {
+		return fmt.Errorf("no data shard files")
+	}
+
+	// Write to a temp file and atomically rename into place, so a crash mid-write
+	// never leaves a partial .dat at the final name beside the source shards.
+	datFileName := baseFileName + ".dat"
+	tmpFileName := datFileName + ".tmp"
+	datFile, openErr := os.OpenFile(tmpFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if openErr != nil {
+		return fmt.Errorf("cannot write volume %s: %v", tmpFileName, openErr)
+	}
+
+	// Use the actual number of data shards passed in rather than the global
+	// constant, so the de-striping matches the caller's shard set.
+	dataShards := len(shardFileNames)
+	inputFiles := make([]*os.File, dataShards)
+
+	committed := false
+	defer func() {
+		datFile.Close()
+		for shardId := 0; shardId < dataShards; shardId++ {
+			if inputFiles[shardId] != nil {
+				inputFiles[shardId].Close()
+			}
+		}
+		if !committed {
+			os.Remove(tmpFileName)
+		}
+	}()
+
+	for shardId := 0; shardId < dataShards; shardId++ {
+		inputFiles[shardId], openErr = os.OpenFile(shardFileNames[shardId], os.O_RDONLY, 0)
+		if openErr != nil {
+			return openErr
+		}
+	}
+
+	if encodedDatFileSize <= 0 {
+		// .vif without the encode-time size: infer the padded layout from the
+		// physical shard size, which reads the shards in the same block order.
+		shardFileInfo, statErr := inputFiles[0].Stat()
+		if statErr != nil {
+			return fmt.Errorf("stat %s: %v", shardFileNames[0], statErr)
+		}
+		shardSize := shardFileInfo.Size()
+		// A shard size that is an exact multiple of the large block size is
+		// ambiguous: N large rows, or N-1 large rows plus a full small-block
+		// region. The two layouts only agree below the last large row.
+		// A uniform layout has no such ambiguity — its large and small blocks
+		// are the same size, so every reading of the shard is the same one, and
+		// without this precondition the check fires on every volume.
+		if largeBlockSize != smallBlockSize &&
+			shardSize%largeBlockSize == 0 && datFileSize > (shardSize/largeBlockSize-1)*largeBlockSize*int64(dataShards) {
+			return fmt.Errorf("shard size %d of %s does not identify the block layout; re-encode to record the dat size in .vif", shardSize, baseFileName)
+		}
+		encodedDatFileSize = int64(dataShards) * shardSize
+	}
+	if datFileSize > encodedDatFileSize {
+		return fmt.Errorf("dat file size %d exceeds encoded dat file size %d", datFileSize, encodedDatFileSize)
+	}
+
+	for encodedDatFileSize >= int64(dataShards)*largeBlockSize && datFileSize > 0 {
+		for shardId := 0; shardId < dataShards && datFileSize > 0; shardId++ {
+			toRead := min(datFileSize, largeBlockSize)
+			w, err := io.CopyN(datFile, inputFiles[shardId], toRead)
+			if w != toRead {
+				return fmt.Errorf("copy %s large block on shardId %d: %v", baseFileName, shardId, err)
+			}
+			datFileSize -= toRead
+		}
+		encodedDatFileSize -= int64(dataShards) * largeBlockSize
+	}
+
+	for datFileSize > 0 {
+		for shardId := 0; shardId < dataShards && datFileSize > 0; shardId++ {
+			toRead := min(datFileSize, smallBlockSize)
+			w, err := io.CopyN(datFile, inputFiles[shardId], toRead)
+			if w != toRead {
+				return fmt.Errorf("copy %s small block %d: %v", baseFileName, shardId, err)
+			}
+			datFileSize -= toRead
+		}
+	}
+
+	// fsync, rename, then fsync the dir so the decoded .dat is durable and
+	// atomically published before the caller deletes the source shards.
+	if err := datFile.Sync(); err != nil {
+		return fmt.Errorf("sync dat for %s: %v", baseFileName, err)
+	}
+	if err := datFile.Close(); err != nil {
+		return fmt.Errorf("close dat for %s: %v", baseFileName, err)
+	}
+	if err := os.Rename(tmpFileName, datFileName); err != nil {
+		return fmt.Errorf("rename dat for %s: %v", baseFileName, err)
+	}
+	if err := util.FsyncDir(filepath.Dir(datFileName)); err != nil {
+		return fmt.Errorf("fsync dir for %s: %v", baseFileName, err)
+	}
+	committed = true
+	return nil
+}
+
+func min(x, y int64) int64 {
+	if x > y {
+		return y
+	}
+	return x
+}

weed/storage/erasure_coding/ec_decoder_completeness_test.go

@@ -0,0 +1,90 @@
+package erasure_coding
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// A decode hands the caller a volume and the caller then deletes the shards it
+// came from. VerifyDecodedDatFile is what stands between a reconstruction that
+// came up short -- a truncated shard, a partial write, a full disk -- and the
+// deletion of the only other copy of the needles past the cut.
+func TestVerifyDecodedDatFile(t *testing.T) {
+	tests := []struct {
+		name        string
+		written     int64
+		referenced  int64
+		wantErr     bool
+		errContains string
+	}{
+		{
+			name:       "exactly the referenced extent is complete",
+			written:    4096,
+			referenced: 4096,
+		},
+		{
+			name:       "longer than referenced is fine: padding is not missing data",
+			written:    8192,
+			referenced: 4096,
+		},
+		{
+			name:        "one byte short still loses the last needle",
+			written:     4095,
+			referenced:  4096,
+			wantErr:     true,
+			errContains: "short of the 4096",
+		},
+		{
+			name:        "an empty file is short of everything",
+			written:     0,
+			referenced:  4096,
+			wantErr:     true,
+			errContains: "is 0 bytes",
+		},
+		{
+			name:       "an index referencing nothing accepts an empty file",
+			written:    0,
+			referenced: 0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			base := filepath.Join(t.TempDir(), "5")
+			if err := os.WriteFile(base+".dat", make([]byte, tt.written), 0644); err != nil {
+				t.Fatal(err)
+			}
+
+			err := VerifyDecodedDatFile(base, tt.referenced)
+			if tt.wantErr {
+				if err == nil {
+					t.Fatalf("wrote %d bytes against a %d-byte extent, want an error", tt.written, tt.referenced)
+				}
+				if !strings.Contains(err.Error(), tt.errContains) {
+					t.Errorf("error %q does not mention %q", err, tt.errContains)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("wrote %d bytes against a %d-byte extent: %v", tt.written, tt.referenced, err)
+			}
+		})
+	}
+}
+
+// A missing .dat is not a short one, and the two want different answers: the
+// caller cannot tell whether the decode wrote nothing or wrote elsewhere, so
+// the error has to name the file rather than a byte count.
+func TestVerifyDecodedDatFileMissing(t *testing.T) {
+	base := filepath.Join(t.TempDir(), "7")
+
+	err := VerifyDecodedDatFile(base, 4096)
+	if err == nil {
+		t.Fatal("a missing .dat must not verify")
+	}
+	if !strings.Contains(err.Error(), "stat decoded") {
+		t.Errorf("error %q does not say the file could not be read", err)
+	}
+}

weed/storage/erasure_coding/ec_decoder_multidisk_test.go

@@ -0,0 +1,54 @@
+package erasure_coding
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestFindDatFileSizeWithRelocatedShard0: on a multi-disk server a volume's
+// shards can sit on several disks, so the .ec00 is not necessarily beside the
+// EcVolume's base path. FindDatFileSize takes the resolved shard-0 path; it
+// must work when that path points to a different directory than the index.
+func TestFindDatFileSizeWithRelocatedShard0(t *testing.T) {
+	diskA := t.TempDir()
+	diskB := t.TempDir()
+
+	for _, ext := range []string{".dat", ".idx"} {
+		data, err := os.ReadFile("1" + ext)
+		if err != nil {
+			t.Fatalf("read fixture 1%s: %v", ext, err)
+		}
+		if err := os.WriteFile(filepath.Join(diskA, "1"+ext), data, 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	base := filepath.Join(diskA, "1")
+	ctx := NewDefaultECContext("", 0)
+	if _, err := generateEcFiles(base, 50, largeBlockSize, smallBlockSize, ctx); err != nil {
+		t.Fatalf("generateEcFiles: %v", err)
+	}
+	if err := WriteSortedFileFromIdx(base, ".ecx"); err != nil {
+		t.Fatalf("WriteSortedFileFromIdx: %v", err)
+	}
+
+	wantSize, err := FindDatFileSize(base+".ec00", base)
+	if err != nil {
+		t.Fatalf("FindDatFileSize with co-located shard 0: %v", err)
+	}
+
+	// Relocate shard 0 to a sibling disk, as a balance or copy can leave it.
+	moved := filepath.Join(diskB, "1.ec00")
+	if err := os.Rename(base+".ec00", moved); err != nil {
+		t.Fatal(err)
+	}
+
+	gotSize, err := FindDatFileSize(moved, base)
+	if err != nil {
+		t.Fatalf("FindDatFileSize with relocated shard 0: %v", err)
+	}
+	if gotSize != wantSize {
+		t.Fatalf("dat size changed with relocated shard 0: got %d want %d", gotSize, wantSize)
+	}
+}

weed/storage/erasure_coding/ec_decoder_test.go

@@ -0,0 +1,633 @@
+package erasure_coding_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	erasure_coding "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func TestHasLiveNeedles_AllDeletedIsFalse(t *testing.T) {
+	dir := t.TempDir()
+
+	collection := "foo"
+	base := filepath.Join(dir, collection+"_1")
+
+	// Build an ecx file with only deleted entries.
+	// ecx file entries are the same format as .idx entries.
+	ecx := makeNeedleMapEntry(types.NeedleId(1), types.Offset{}, types.TombstoneFileSize)
+	if err := os.WriteFile(base+".ecx", ecx, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+
+	hasLive, err := erasure_coding.HasLiveNeedles(base)
+	if err != nil {
+		t.Fatalf("HasLiveNeedles: %v", err)
+	}
+	if hasLive {
+		t.Fatalf("expected no live entries")
+	}
+}
+
+func TestHasLiveNeedles_WithLiveEntryIsTrue(t *testing.T) {
+	dir := t.TempDir()
+
+	collection := "foo"
+	base := filepath.Join(dir, collection+"_1")
+
+	// Build an ecx file containing at least one live entry.
+	// ecx file entries are the same format as .idx entries.
+	live := makeNeedleMapEntry(types.NeedleId(1), types.Offset{}, types.Size(1))
+	if err := os.WriteFile(base+".ecx", live, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+
+	hasLive, err := erasure_coding.HasLiveNeedles(base)
+	if err != nil {
+		t.Fatalf("HasLiveNeedles: %v", err)
+	}
+	if !hasLive {
+		t.Fatalf("expected live entries")
+	}
+}
+
+func TestHasLiveNeedles_EmptyFileIsFalse(t *testing.T) {
+	dir := t.TempDir()
+
+	base := filepath.Join(dir, "foo_1")
+
+	// Create an empty ecx file.
+	if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+
+	hasLive, err := erasure_coding.HasLiveNeedles(base)
+	if err != nil {
+		t.Fatalf("HasLiveNeedles: %v", err)
+	}
+	if hasLive {
+		t.Fatalf("expected no live entries for empty file")
+	}
+}
+
+func TestHasLiveNeedles_TruncatedEcxErrors(t *testing.T) {
+	dir := t.TempDir()
+
+	base := filepath.Join(dir, "foo_1")
+
+	entry := makeNeedleMapEntry(types.NeedleId(1), types.Offset{}, types.TombstoneFileSize)
+	truncated := append(entry, entry[:types.NeedleIdSize]...)
+	if err := os.WriteFile(base+".ecx", truncated, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+
+	if _, err := erasure_coding.HasLiveNeedles(base); err == nil {
+		t.Fatalf("expected error for truncated ecx")
+	}
+}
+
+func makeNeedleMapEntry(key types.NeedleId, offset types.Offset, size types.Size) []byte {
+	b := make([]byte, types.NeedleIdSize+types.OffsetSize+types.SizeSize)
+	types.NeedleIdToBytes(b[0:types.NeedleIdSize], key)
+	types.OffsetToBytes(b[types.NeedleIdSize:types.NeedleIdSize+types.OffsetSize], offset)
+	types.SizeToBytes(b[types.NeedleIdSize+types.OffsetSize:types.NeedleIdSize+types.OffsetSize+types.SizeSize], size)
+	return b
+}
+
+// TestWriteIdxFileFromEcIndex_PreservesDeletedNeedles verifies that WriteIdxFileFromEcIndex
+// correctly marks deleted needles in the generated .idx file.
+// This tests the fix for issue #7751 where deleted files in encoded volumes
+// were not properly marked as deleted when decoded.
+func TestWriteIdxFileFromEcIndex_PreservesDeletedNeedles(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "foo_1")
+
+	// Create an .ecx file with one live needle and one deleted needle
+	needle1 := makeNeedleMapEntry(types.NeedleId(1), types.ToOffset(64), types.Size(100))
+	needle2 := makeNeedleMapEntry(types.NeedleId(2), types.ToOffset(128), types.TombstoneFileSize) // deleted
+
+	ecxData := append(needle1, needle2...)
+	if err := os.WriteFile(base+".ecx", ecxData, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+
+	// Generate .idx from .ecx
+	if err := erasure_coding.WriteIdxFileFromEcIndex(base); err != nil {
+		t.Fatalf("WriteIdxFileFromEcIndex: %v", err)
+	}
+
+	// Verify .idx file has the same content
+	idxData, err := os.ReadFile(base + ".idx")
+	if err != nil {
+		t.Fatalf("read idx: %v", err)
+	}
+
+	if len(idxData) != len(ecxData) {
+		t.Fatalf("idx file size mismatch: got %d, want %d", len(idxData), len(ecxData))
+	}
+
+	// Verify the second needle is still marked as deleted
+	entrySize := types.NeedleIdSize + types.OffsetSize + types.SizeSize
+	entry2 := idxData[entrySize : entrySize*2]
+	size2 := types.BytesToSize(entry2[types.NeedleIdSize+types.OffsetSize:])
+	if !size2.IsDeleted() {
+		t.Fatalf("expected needle 2 to be marked as deleted, got size: %d", size2)
+	}
+}
+
+// TestWriteIdxFileFromEcIndex_ProcessesEcjJournal verifies that WriteIdxFileFromEcIndex
+// correctly processes deletions from the .ecj journal file.
+func TestWriteIdxFileFromEcIndex_ProcessesEcjJournal(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "foo_1")
+
+	// Create an .ecx file with two live needles
+	needle1 := makeNeedleMapEntry(types.NeedleId(1), types.ToOffset(64), types.Size(100))
+	needle2 := makeNeedleMapEntry(types.NeedleId(2), types.ToOffset(128), types.Size(200))
+
+	ecxData := append(needle1, needle2...)
+	if err := os.WriteFile(base+".ecx", ecxData, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+
+	// Create an .ecj file that records needle 2 as deleted
+	ecjData := make([]byte, types.NeedleIdSize)
+	types.NeedleIdToBytes(ecjData, types.NeedleId(2))
+	if err := os.WriteFile(base+".ecj", ecjData, 0644); err != nil {
+		t.Fatalf("write ecj: %v", err)
+	}
+
+	// Generate .idx from .ecx and .ecj
+	if err := erasure_coding.WriteIdxFileFromEcIndex(base); err != nil {
+		t.Fatalf("WriteIdxFileFromEcIndex: %v", err)
+	}
+
+	// Verify .idx file has 3 entries: 2 from .ecx + 1 deletion from .ecj
+	idxData, err := os.ReadFile(base + ".idx")
+	if err != nil {
+		t.Fatalf("read idx: %v", err)
+	}
+
+	entrySize := types.NeedleIdSize + types.OffsetSize + types.SizeSize
+	expectedSize := entrySize * 3 // 2 from ecx + 1 deletion append from ecj
+	if len(idxData) != expectedSize {
+		t.Fatalf("idx file size mismatch: got %d, want %d", len(idxData), expectedSize)
+	}
+
+	// The third entry should be the deletion record for needle 2
+	entry3 := idxData[entrySize*2 : entrySize*3]
+	key3 := types.BytesToNeedleId(entry3[0:types.NeedleIdSize])
+	size3 := types.BytesToSize(entry3[types.NeedleIdSize+types.OffsetSize:])
+
+	if key3 != types.NeedleId(2) {
+		t.Fatalf("expected needle id 2 in deletion record, got: %d", key3)
+	}
+	if !size3.IsDeleted() {
+		t.Fatalf("expected deletion record to have tombstone size, got: %d", size3)
+	}
+}
+
+// TestDecodeAtomicPublish verifies the decoded .idx/.dat are published via a
+// temp file plus rename: a successful write leaves the final file with no
+// leftover .tmp, and a failed write leaves neither the final file nor a
+// partial .tmp beside the source shards.
+func TestDecodeAtomicPublish(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "foo_1")
+
+	// .ecx with one live needle so WriteIdxFileFromEcIndex succeeds.
+	ecxData := makeNeedleMapEntry(types.NeedleId(1), types.ToOffset(64), types.Size(100))
+	if err := os.WriteFile(base+".ecx", ecxData, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+	if err := erasure_coding.WriteIdxFileFromEcIndex(base); err != nil {
+		t.Fatalf("WriteIdxFileFromEcIndex: %v", err)
+	}
+	if _, err := os.Stat(base + ".idx"); err != nil {
+		t.Fatalf("decoded .idx missing: %v", err)
+	}
+	if _, err := os.Stat(base + ".idx.tmp"); !os.IsNotExist(err) {
+		t.Fatalf("expected no leftover .idx.tmp, stat err=%v", err)
+	}
+
+	// A WriteDatFile pointed at a missing shard must fail and leave neither the
+	// final .dat nor a partial .dat.tmp behind.
+	datBase := filepath.Join(dir, "bar_2")
+	missingShards := []string{filepath.Join(dir, "does_not_exist.ec00")}
+	if err := erasure_coding.WriteDatFile(datBase, 100, 100, missingShards, erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize); err == nil {
+		t.Fatalf("expected WriteDatFile to fail on missing shard")
+	}
+	if _, err := os.Stat(datBase + ".dat"); !os.IsNotExist(err) {
+		t.Fatalf("failed decode must not leave a .dat, stat err=%v", err)
+	}
+	if _, err := os.Stat(datBase + ".dat.tmp"); !os.IsNotExist(err) {
+		t.Fatalf("failed decode must not leave a .dat.tmp, stat err=%v", err)
+	}
+}
+
+// TestDecodeWithNonEmptyEcj_AllDeleted verifies the full decode pre-processing
+// when .ecj contains deletions for ALL live entries in .ecx.
+// After RebuildEcxFile merges .ecj into .ecx, HasLiveNeedles must return false
+// and WriteIdxFileFromEcIndex must produce an .idx where every entry is tombstoned.
+func TestDecodeWithNonEmptyEcj_AllDeleted(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "test_1")
+
+	// .ecx: two live entries
+	needle1 := makeNeedleMapEntry(types.NeedleId(1), types.ToOffset(64), types.Size(100))
+	needle2 := makeNeedleMapEntry(types.NeedleId(2), types.ToOffset(128), types.Size(200))
+	ecxData := append(needle1, needle2...)
+	if err := os.WriteFile(base+".ecx", ecxData, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+
+	// .ecj: both needles deleted
+	ecjData := make([]byte, 2*types.NeedleIdSize)
+	types.NeedleIdToBytes(ecjData[0:types.NeedleIdSize], types.NeedleId(1))
+	types.NeedleIdToBytes(ecjData[types.NeedleIdSize:], types.NeedleId(2))
+	if err := os.WriteFile(base+".ecj", ecjData, 0644); err != nil {
+		t.Fatalf("write ecj: %v", err)
+	}
+
+	// Before rebuild, ecx entries look live
+	hasLive, err := erasure_coding.HasLiveNeedles(base)
+	if err != nil {
+		t.Fatalf("HasLiveNeedles before rebuild: %v", err)
+	}
+	if !hasLive {
+		t.Fatal("expected live entries before rebuild")
+	}
+
+	// Simulate what VolumeEcShardsToVolume now does: merge .ecj into .ecx
+	if err := erasure_coding.RebuildEcxFile(base); err != nil {
+		t.Fatalf("RebuildEcxFile: %v", err)
+	}
+
+	// .ecj should be removed after rebuild
+	if _, err := os.Stat(base + ".ecj"); !os.IsNotExist(err) {
+		t.Fatal("expected .ecj to be removed after RebuildEcxFile")
+	}
+
+	// After rebuild, HasLiveNeedles must return false
+	hasLive, err = erasure_coding.HasLiveNeedles(base)
+	if err != nil {
+		t.Fatalf("HasLiveNeedles after rebuild: %v", err)
+	}
+	if hasLive {
+		t.Fatal("expected no live entries after rebuild merged all deletions")
+	}
+
+	// WriteIdxFileFromEcIndex should still work (no .ecj to process)
+	if err := erasure_coding.WriteIdxFileFromEcIndex(base); err != nil {
+		t.Fatalf("WriteIdxFileFromEcIndex: %v", err)
+	}
+
+	idxData, err := os.ReadFile(base + ".idx")
+	if err != nil {
+		t.Fatalf("read idx: %v", err)
+	}
+
+	// .idx should have exactly 2 entries (copied from .ecx, both now tombstoned)
+	entrySize := types.NeedleIdSize + types.OffsetSize + types.SizeSize
+	if len(idxData) != 2*entrySize {
+		t.Fatalf("idx file size: got %d, want %d", len(idxData), 2*entrySize)
+	}
+
+	// Both entries must be tombstoned
+	for i := 0; i < 2; i++ {
+		entry := idxData[i*entrySize : (i+1)*entrySize]
+		size := types.BytesToSize(entry[types.NeedleIdSize+types.OffsetSize:])
+		if !size.IsDeleted() {
+			t.Fatalf("entry %d: expected tombstone, got size %d", i+1, size)
+		}
+	}
+}
+
+// TestDecodeWithNonEmptyEcj_PartiallyDeleted verifies decode pre-processing
+// when .ecj deletes only some entries. After RebuildEcxFile, HasLiveNeedles
+// must still return true for the surviving entries, and WriteIdxFileFromEcIndex
+// must produce an .idx that correctly distinguishes live from deleted needles.
+func TestDecodeWithNonEmptyEcj_PartiallyDeleted(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "test_1")
+
+	// .ecx: three live entries
+	needle1 := makeNeedleMapEntry(types.NeedleId(1), types.ToOffset(64), types.Size(100))
+	needle2 := makeNeedleMapEntry(types.NeedleId(2), types.ToOffset(128), types.Size(200))
+	needle3 := makeNeedleMapEntry(types.NeedleId(3), types.ToOffset(256), types.Size(300))
+	ecxData := append(append(needle1, needle2...), needle3...)
+	if err := os.WriteFile(base+".ecx", ecxData, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+
+	// .ecj: only needle 2 is deleted
+	ecjData := make([]byte, types.NeedleIdSize)
+	types.NeedleIdToBytes(ecjData, types.NeedleId(2))
+	if err := os.WriteFile(base+".ecj", ecjData, 0644); err != nil {
+		t.Fatalf("write ecj: %v", err)
+	}
+
+	// Merge .ecj into .ecx
+	if err := erasure_coding.RebuildEcxFile(base); err != nil {
+		t.Fatalf("RebuildEcxFile: %v", err)
+	}
+
+	// HasLiveNeedles must still return true (needles 1 and 3 survive)
+	hasLive, err := erasure_coding.HasLiveNeedles(base)
+	if err != nil {
+		t.Fatalf("HasLiveNeedles: %v", err)
+	}
+	if !hasLive {
+		t.Fatal("expected live entries after partial deletion")
+	}
+
+	// WriteIdxFileFromEcIndex
+	if err := erasure_coding.WriteIdxFileFromEcIndex(base); err != nil {
+		t.Fatalf("WriteIdxFileFromEcIndex: %v", err)
+	}
+
+	idxData, err := os.ReadFile(base + ".idx")
+	if err != nil {
+		t.Fatalf("read idx: %v", err)
+	}
+
+	entrySize := types.NeedleIdSize + types.OffsetSize + types.SizeSize
+	if len(idxData) != 3*entrySize {
+		t.Fatalf("idx file size: got %d, want %d", len(idxData), 3*entrySize)
+	}
+
+	// Verify each entry
+	for i := 0; i < 3; i++ {
+		entry := idxData[i*entrySize : (i+1)*entrySize]
+		key := types.BytesToNeedleId(entry[0:types.NeedleIdSize])
+		size := types.BytesToSize(entry[types.NeedleIdSize+types.OffsetSize:])
+
+		switch key {
+		case types.NeedleId(1), types.NeedleId(3):
+			if size.IsDeleted() {
+				t.Fatalf("needle %d: should be live, got tombstone", key)
+			}
+		case types.NeedleId(2):
+			if !size.IsDeleted() {
+				t.Fatalf("needle %d: should be tombstoned, got size %d", key, size)
+			}
+		default:
+			t.Fatalf("unexpected needle id %d", key)
+		}
+	}
+}
+
+// TestDecodeWithEmptyEcj verifies that the decode flow is a no-op when
+// .ecj exists but is empty (no deletions recorded).
+func TestDecodeWithEmptyEcj(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "test_1")
+
+	// .ecx: one live entry
+	needle1 := makeNeedleMapEntry(types.NeedleId(1), types.ToOffset(64), types.Size(100))
+	if err := os.WriteFile(base+".ecx", needle1, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+
+	// .ecj: empty
+	if err := os.WriteFile(base+".ecj", []byte{}, 0644); err != nil {
+		t.Fatalf("write ecj: %v", err)
+	}
+
+	// RebuildEcxFile with empty .ecj should not change anything
+	if err := erasure_coding.RebuildEcxFile(base); err != nil {
+		t.Fatalf("RebuildEcxFile: %v", err)
+	}
+
+	// HasLiveNeedles must still return true
+	hasLive, err := erasure_coding.HasLiveNeedles(base)
+	if err != nil {
+		t.Fatalf("HasLiveNeedles: %v", err)
+	}
+	if !hasLive {
+		t.Fatal("expected live entries with empty .ecj")
+	}
+}
+
+// TestDecodeWithNoEcjFile verifies that the decode flow works when no .ecj
+// file exists at all.
+func TestDecodeWithNoEcjFile(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "test_1")
+
+	// .ecx: one live entry
+	needle1 := makeNeedleMapEntry(types.NeedleId(1), types.ToOffset(64), types.Size(100))
+	if err := os.WriteFile(base+".ecx", needle1, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+
+	// No .ecj file
+
+	// RebuildEcxFile should be a no-op
+	if err := erasure_coding.RebuildEcxFile(base); err != nil {
+		t.Fatalf("RebuildEcxFile: %v", err)
+	}
+
+	hasLive, err := erasure_coding.HasLiveNeedles(base)
+	if err != nil {
+		t.Fatalf("HasLiveNeedles: %v", err)
+	}
+	if !hasLive {
+		t.Fatal("expected live entries without .ecj file")
+	}
+}
+
+// TestEcxFileDeletionVisibleAfterSync verifies that deletions made to .ecx
+// via MarkNeedleDeleted are visible to other readers after Sync().
+// This is a regression test for issue #7751.
+func TestEcxFileDeletionVisibleAfterSync(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "foo_1")
+
+	// Create an .ecx file with one live needle
+	needle1 := makeNeedleMapEntry(types.NeedleId(1), types.ToOffset(64), types.Size(100))
+	if err := os.WriteFile(base+".ecx", needle1, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+
+	// Open the file for writing (simulating what EcVolume does)
+	ecxFile, err := os.OpenFile(base+".ecx", os.O_RDWR, 0644)
+	if err != nil {
+		t.Fatalf("open ecx: %v", err)
+	}
+
+	// Mark needle as deleted using MarkNeedleDeleted
+	err = erasure_coding.MarkNeedleDeleted(ecxFile, 0)
+	if err != nil {
+		ecxFile.Close()
+		t.Fatalf("MarkNeedleDeleted: %v", err)
+	}
+
+	// Sync the file to ensure changes are visible to other readers
+	if err := ecxFile.Sync(); err != nil {
+		ecxFile.Close()
+		t.Fatalf("Sync: %v", err)
+	}
+	ecxFile.Close()
+
+	// Now open with a new file handle and verify deletion is visible
+	data, err := os.ReadFile(base + ".ecx")
+	if err != nil {
+		t.Fatalf("read ecx: %v", err)
+	}
+
+	size := types.BytesToSize(data[types.NeedleIdSize+types.OffsetSize:])
+	if !size.IsDeleted() {
+		t.Fatalf("expected needle to be marked as deleted after sync, got size: %d", size)
+	}
+}
+
+// TestEcxFileDeletionNotVisibleWithoutSync verifies that without Sync(),
+// deletions may not be visible to other readers (demonstrating the bug).
+// Note: This test may be flaky depending on OS caching behavior, but it
+// documents the expected behavior.
+func TestEcxFileDeletionWithSeparateHandles(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "foo_1")
+
+	// Create an .ecx file with one live needle
+	needle1 := makeNeedleMapEntry(types.NeedleId(1), types.ToOffset(64), types.Size(100))
+	if err := os.WriteFile(base+".ecx", needle1, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+
+	// Open the file for writing (writer handle)
+	writerFile, err := os.OpenFile(base+".ecx", os.O_RDWR, 0644)
+	if err != nil {
+		t.Fatalf("open ecx for write: %v", err)
+	}
+	defer writerFile.Close()
+
+	// Open the file for reading (reader handle - simulating CopyFile behavior)
+	readerFile, err := os.OpenFile(base+".ecx", os.O_RDONLY, 0644)
+	if err != nil {
+		t.Fatalf("open ecx for read: %v", err)
+	}
+	defer readerFile.Close()
+
+	// Mark needle as deleted via writer handle
+	err = erasure_coding.MarkNeedleDeleted(writerFile, 0)
+	if err != nil {
+		t.Fatalf("MarkNeedleDeleted: %v", err)
+	}
+
+	// Sync the writer to flush changes
+	if err := writerFile.Sync(); err != nil {
+		t.Fatalf("Sync: %v", err)
+	}
+
+	// Read via reader handle - after sync, changes should be visible
+	data := make([]byte, types.NeedleIdSize+types.OffsetSize+types.SizeSize)
+	if _, err := readerFile.ReadAt(data, 0); err != nil {
+		t.Fatalf("ReadAt: %v", err)
+	}
+
+	size := types.BytesToSize(data[types.NeedleIdSize+types.OffsetSize:])
+	if !size.IsDeleted() {
+		t.Fatalf("expected deletion to be visible after Sync(), got size: %d", size)
+	}
+}
+
+// TestEcVolumeDeleteDurableToJournal tracks issue #7751: a runtime needle
+// delete must be observable by the ec.decode CopyFile path. Under the
+// current design .ecx is an immutable sealed index at runtime — deletes
+// are journaled to .ecj and tracked in an in-memory set — so the
+// durability chain decode relies on is:
+//
+//  1. DeleteNeedleFromEcx appends the needle id to .ecj and fsyncs it.
+//  2. Runtime reads via FindNeedleFromEcx consult the in-memory set and
+//     return TombstoneFileSize even though the sealed .ecx record on
+//     disk still shows the original size.
+//  3. ec.decode later closes the EcVolume and calls RebuildEcxFile on
+//     the now-quiescent files, which walks .ecj and writes tombstones
+//     into .ecx. CopyFile then reads the rebuilt .ecx.
+//
+// This test exercises the full chain on a tempdir fixture.
+func TestEcVolumeDeleteDurableToJournal(t *testing.T) {
+	dir := t.TempDir()
+
+	collection := "test"
+	vid := 1
+	base := filepath.Join(dir, collection+"_1")
+
+	// Seed .ecx with two live needles.
+	needle1 := makeNeedleMapEntry(types.NeedleId(1), types.ToOffset(64), types.Size(100))
+	needle2 := makeNeedleMapEntry(types.NeedleId(2), types.ToOffset(128), types.Size(200))
+	ecxData := append(needle1, needle2...)
+	if err := os.WriteFile(base+".ecx", ecxData, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+	if err := os.WriteFile(base+".ecj", []byte{}, 0644); err != nil {
+		t.Fatalf("write ecj: %v", err)
+	}
+	if err := os.WriteFile(base+".ec00", make([]byte, 8), 0644); err != nil {
+		t.Fatalf("write ec00: %v", err)
+	}
+	if err := os.WriteFile(base+".vif", []byte{}, 0644); err != nil {
+		t.Fatalf("write vif: %v", err)
+	}
+
+	ecVolume, err := erasure_coding.NewEcVolume("hdd", dir, dir, collection, needle.VolumeId(vid))
+	if err != nil {
+		t.Fatalf("NewEcVolume: %v", err)
+	}
+
+	// Runtime delete must not mutate .ecx.
+	if err := ecVolume.DeleteNeedleFromEcx(types.NeedleId(2)); err != nil {
+		t.Fatalf("DeleteNeedleFromEcx: %v", err)
+	}
+
+	// FindNeedleFromEcx masks the id via the in-memory set.
+	_, size, err := ecVolume.FindNeedleFromEcx(types.NeedleId(2))
+	if err != nil {
+		t.Fatalf("FindNeedleFromEcx(2): %v", err)
+	}
+	if !size.IsDeleted() {
+		t.Fatalf("expected FindNeedleFromEcx to return tombstone for deleted needle, got size=%d", size)
+	}
+
+	// Direct .ecx reader should still see the original size — .ecx is
+	// immutable at runtime.
+	entrySize := int64(types.NeedleIdSize + types.OffsetSize + types.SizeSize)
+	rawBuf := make([]byte, entrySize)
+	rawReader, err := os.Open(base + ".ecx")
+	if err != nil {
+		t.Fatalf("open ecx raw: %v", err)
+	}
+	if _, err := rawReader.ReadAt(rawBuf, entrySize); err != nil {
+		t.Fatalf("read raw ecx entry: %v", err)
+	}
+	rawReader.Close()
+	rawSize := types.BytesToSize(rawBuf[types.NeedleIdSize+types.OffsetSize:])
+	if rawSize.IsDeleted() {
+		t.Fatalf("runtime delete must not mutate .ecx on disk; got tombstone in raw entry")
+	}
+
+	// Close the volume so RebuildEcxFile can operate on the files, then
+	// fold .ecj into .ecx as the decode path does and verify the rebuilt
+	// index has the tombstone visible to external readers.
+	ecVolume.Close()
+	if err := erasure_coding.RebuildEcxFile(base); err != nil {
+		t.Fatalf("RebuildEcxFile: %v", err)
+	}
+	rebuilt, err := os.Open(base + ".ecx")
+	if err != nil {
+		t.Fatalf("open rebuilt ecx: %v", err)
+	}
+	defer rebuilt.Close()
+	if _, err := rebuilt.ReadAt(rawBuf, entrySize); err != nil {
+		t.Fatalf("read rebuilt ecx entry: %v", err)
+	}
+	rebuiltSize := types.BytesToSize(rawBuf[types.NeedleIdSize+types.OffsetSize:])
+	if !rebuiltSize.IsDeleted() {
+		t.Fatalf("expected needle 2 to be tombstoned in rebuilt .ecx, got size=%d", rebuiltSize)
+	}
+}

weed/storage/erasure_coding/ec_encoder.go

@@ -0,0 +1,756 @@
+package erasure_coding
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/klauspost/reedsolomon"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const (
+	DataShardsCount             = 10
+	ParityShardsCount           = 4
+	TotalShardsCount            = DataShardsCount + ParityShardsCount
+	MaxShardCount               = 32 // Maximum number of shards since ShardBits is uint32 (bits 0-31)
+	MinTotalDisks               = TotalShardsCount/ParityShardsCount + 1
+	ErasureCodingLargeBlockSize = 1024 * 1024 * 1024 // 1GB
+	ErasureCodingSmallBlockSize = 1024 * 1024        // 1MB
+)
+
+// WriteSortedFileFromIdx generates .ecx file from existing .idx file
+// all keys are sorted in ascending order
+func WriteSortedFileFromIdx(baseFileName string, ext string) (e error) {
+
+	nm, err := readNeedleMap(baseFileName)
+	if nm != nil {
+		defer nm.Close()
+	}
+	if err != nil {
+		return fmt.Errorf("readNeedleMap: %w", err)
+	}
+
+	ecxFile, err := os.OpenFile(baseFileName+ext, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open ecx file: %w", err)
+	}
+	defer ecxFile.Close()
+
+	err = nm.AscendingVisit(func(value needle_map.NeedleValue) error {
+		bytes := value.ToBytes()
+		_, writeErr := ecxFile.Write(bytes)
+		return writeErr
+	})
+
+	if err != nil {
+		return fmt.Errorf("failed to visit idx file: %w", err)
+	}
+
+	return nil
+}
+
+// WriteEcFiles generates .ec00 ~ .ec13 files from baseFileName.dat. Pass
+// BackgroundECContext for the default ratio, or an explicit ctx for a configured
+// (e.g. custom-ratio) layout. It returns the bitrot protection (per-shard block
+// CRC32C) computed during the single encode pass; the caller persists it as a
+// <base>.ecsum sidecar, and persists ctx.BlockSize and ctx.DatFileSize (both
+// set here, from one measurement of the .dat) to the .vif so readers resolve
+// the shard block layout.
+func WriteEcFiles(baseFileName string, ctx *ECContext) (*volume_server_pb.EcBitrotProtection, error) {
+	if ctx == nil {
+		ctx = NewDefaultECContext("", 0)
+	} else if ctx.Total() == 0 {
+		// Fill the placeholder in place rather than swapping the pointer: the
+		// caller reads BlockSize and DatFileSize back off the context it
+		// passed, and a replacement leaves it holding the zero values.
+		ctx.DataShards, ctx.ParityShards = DataShardsCount, ParityShardsCount
+	}
+	// Always encode with the uniform block layout, sized for this .dat. Both
+	// the block size and the .dat length it was derived from are left on ctx,
+	// so the caller persists a .vif whose two fields describe one measurement
+	// — a second stat could see a different size on a volume still taking
+	// writes.
+	fi, err := os.Stat(baseFileName + ".dat")
+	if err != nil {
+		return nil, fmt.Errorf("failed to stat dat file: %w", err)
+	}
+	ctx.DatFileSize = fi.Size()
+	ctx.BlockSize = UniformBlockSize(fi.Size(), ctx.DataShards)
+	return generateEcFiles(baseFileName, 256*1024, ctx.BlockSize, ctx.BlockSize, ctx)
+}
+
+// ValidateBlockSize reports whether a `.vif`-recorded shard block size is one
+// an encoder could have produced. 0 means the legacy two-tier layout, which is
+// always valid; anything positive must be a whole number of small blocks,
+// because that is what UniformBlockSize rounds to. A negative or unaligned
+// value is corruption, and using it would map every read to the wrong shard
+// offset.
+func ValidateBlockSize(blockSize int64) error {
+	if blockSize == 0 {
+		return nil
+	}
+	if blockSize < 0 || blockSize%ErasureCodingSmallBlockSize != 0 {
+		return fmt.Errorf("invalid shard block size %d: expected 0 (legacy) or a multiple of %d",
+			blockSize, ErasureCodingSmallBlockSize)
+	}
+	return nil
+}
+
+// UniformBlockSize returns the per-shard block size of the uniform layout for
+// a .dat of the given size: ceil(datFileSize/dataShards) rounded up to a whole
+// small block. For every input this equals the legacy layout's padded shard
+// size, so only the byte placement differs between the two layouts, never the
+// shard length.
+func UniformBlockSize(datFileSize int64, dataShards int) int64 {
+	perShard := (datFileSize + int64(dataShards) - 1) / int64(dataShards)
+	blocks := (perShard + ErasureCodingSmallBlockSize - 1) / ErasureCodingSmallBlockSize
+	if blocks < 1 {
+		blocks = 1
+	}
+	return blocks * ErasureCodingSmallBlockSize
+}
+
+// ResolveRebuildECContext answers which shard layout a rebuild of baseFileName
+// will use: the caller's context when it already states one, else the volume's
+// own metadata — its `.vif` in any of the directories the rebuild can read,
+// then the generation-0 bitrot sidecar, and only then the build defaults.
+// Exported so a caller that must agree with the rebuild (the post-rebuild
+// bitrot backfill writes a manifest describing these very shards) resolves
+// once and uses the same answer, instead of re-deriving it from a narrower
+// search and recording a layout the rebuild did not use.
+func ResolveRebuildECContext(baseFileName string, ctx *ECContext, additionalDirs []string) (*ECContext, error) {
+	if ctx == nil || ctx.Total() == 0 {
+		// Resolve the layout from the .vif to preserve the original configuration.
+		vifPath := findVifPath(baseFileName, additionalDirs)
+		volumeInfo, _, foundVif, vifErr := volume_info.MaybeLoadVolumeInfo(vifPath)
+		if vifErr != nil {
+			// The .vif exists but cannot be read or parsed. Fail closed rather
+			// than silently falling back to the default ratio, which would
+			// rebuild a custom-ratio volume with the wrong layout. Pass an
+			// explicit ctx to override.
+			return nil, fmt.Errorf("RebuildEcFiles %s: cannot load .vif: %w", baseFileName, vifErr)
+		}
+		switch {
+		case foundVif && volumeInfo.EcShardConfig != nil &&
+			ValidEcShardCounts(volumeInfo.EcShardConfig.DataShards, volumeInfo.EcShardConfig.ParityShards):
+			if bsErr := ValidateBlockSize(volumeInfo.EcShardConfig.GetBlockSize()); bsErr != nil {
+				return nil, fmt.Errorf("RebuildEcFiles %s: %s: %w", baseFileName, vifPath, bsErr)
+			}
+			ctx = &ECContext{
+				DataShards:   int(volumeInfo.EcShardConfig.DataShards),
+				ParityShards: int(volumeInfo.EcShardConfig.ParityShards),
+				BlockSize:    volumeInfo.EcShardConfig.GetBlockSize(),
+			}
+			glog.V(0).Infof("Rebuilding EC files for %s with config from .vif: %s", baseFileName, ctx.String())
+		case foundVif && volumeInfo.EcShardConfig != nil:
+			// A recorded-but-impossible ratio is corruption, not a reason to
+			// substitute the default one: a 12+4 volume rebuilt as 10+4
+			// reconstructs from the wrong matrix and never regenerates shards
+			// 14-15. Pass an explicit ctx to override.
+			return nil, fmt.Errorf("RebuildEcFiles %s: %s records invalid shard counts %d+%d",
+				baseFileName, vifPath, volumeInfo.EcShardConfig.DataShards, volumeInfo.EcShardConfig.ParityShards)
+		default:
+			// No usable .vif: the bitrot sidecar records the same config at
+			// encode time and is then the surviving authority. Reading the
+			// default ratio and the legacy block size instead would rebuild
+			// from the wrong geometry AND make the sidecar look like it
+			// disagrees, which silently skips every checksum check below.
+			if sidecarPath := findBitrotSidecar(0, baseFileName, baseFileName, additionalDirs...); sidecarPath != "" {
+				cfg, cfgErr := EcShardConfigFromSidecarPath(sidecarPath)
+				if cfgErr != nil {
+					return nil, fmt.Errorf("RebuildEcFiles %s: no usable .vif and %w", baseFileName, cfgErr)
+				}
+				ctx = &ECContext{
+					DataShards:   int(cfg.GetDataShards()),
+					ParityShards: int(cfg.GetParityShards()),
+					BlockSize:    cfg.GetBlockSize(),
+				}
+				glog.V(0).Infof("Rebuilding EC files for %s with config from the bitrot sidecar: %s", baseFileName, ctx.String())
+				break
+			}
+			glog.V(0).Infof("Rebuilding EC files for %s with default config", baseFileName)
+			ctx = NewDefaultECContext("", 0)
+		}
+	}
+	return ctx, nil
+}
+
+// RebuildEcFiles rebuilds missing EC shard files. Pass BackgroundECContext to
+// resolve the layout from the volume's .vif (falling back to the default ratio),
+// or an explicit ctx when the caller already knows the shard layout.
+// additionalDirs are extra directories to search for existing shard files,
+// which handles multi-disk servers where shards may be spread across disks.
+// When a bitrot checksum sidecar is present for the (generation-0) volume,
+// present input shards are verified against it and corrupt ones are excluded
+// from Reed-Solomon and regenerated; unsafeIgnoreSidecar bypasses that guard.
+func RebuildEcFiles(baseFileName string, ctx *ECContext, unsafeIgnoreSidecar bool, additionalDirs ...string) ([]uint32, error) {
+	ctx, err := ResolveRebuildECContext(baseFileName, ctx, additionalDirs)
+	if err != nil {
+		return nil, err
+	}
+	return generateMissingEcFiles(baseFileName, 256*1024, ctx, unsafeIgnoreSidecar, additionalDirs)
+}
+
+func ToExt(ecIndex int) string {
+	return fmt.Sprintf(".ec%02d", ecIndex)
+}
+
+func generateEcFiles(baseFileName string, bufferSize int, largeBlockSize int64, smallBlockSize int64, ctx *ECContext) (*volume_server_pb.EcBitrotProtection, error) {
+	file, err := os.OpenFile(baseFileName+".dat", os.O_RDONLY, 0)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open dat file: %w", err)
+	}
+	defer file.Close()
+
+	fi, err := file.Stat()
+	if err != nil {
+		return nil, fmt.Errorf("failed to stat dat file: %w", err)
+	}
+
+	// One rolling-CRC builder per shard; fed as each shard's bytes are written.
+	builders := make([]*shardChecksumBuilder, ctx.Total())
+	for i := range builders {
+		builders[i] = newShardChecksumBuilder(BitrotBlockSize)
+	}
+
+	glog.V(0).Infof("encodeDatFile %s.dat size:%d with EC context %s", baseFileName, fi.Size(), ctx.String())
+	err = encodeDatFile(fi.Size(), baseFileName, bufferSize, largeBlockSize, file, smallBlockSize, ctx, builders)
+	if err != nil {
+		return nil, fmt.Errorf("encodeDatFile: %w", err)
+	}
+	return buildProtectionFromBuilders(ctx, builders, BitrotBlockSize), nil
+}
+
+// findShardFile looks for a shard file at baseFileName+ext, then in additionalDirs.
+func findShardFile(baseFileName string, ext string, additionalDirs []string) string {
+	primary := baseFileName + ext
+	if util.FileExists(primary) {
+		return primary
+	}
+	baseName := filepath.Base(baseFileName)
+	for _, dir := range additionalDirs {
+		candidate := filepath.Join(dir, baseName+ext)
+		if util.FileExists(candidate) {
+			return candidate
+		}
+	}
+	return ""
+}
+
+// generateMissingEcFiles takes no block sizes: Reed-Solomon reconstruction is
+// layout-agnostic — it rebuilds a missing shard from the same offsets of the
+// survivors — so the shard layout only ever reaches it through ctx.
+func generateMissingEcFiles(baseFileName string, bufferSize int, ctx *ECContext, unsafeIgnoreSidecar bool, additionalDirs []string) (generatedShardIds []uint32, err error) {
+
+	// Pass 1: discover which shards exist and which are missing,
+	// opening input files but NOT creating output files yet.
+	shardHasData := make([]bool, ctx.Total())
+	shardPaths := make([]string, ctx.Total()) // non-empty for present shards (also the in-place output for a reclassified-corrupt shard)
+	inputFiles := make([]*os.File, ctx.Total())
+	presentCount := 0
+	var zeroSized []int
+	for shardId := 0; shardId < ctx.Total(); shardId++ {
+		ext := ctx.ToExt(shardId)
+		shardPath := findShardFile(baseFileName, ext, additionalDirs)
+		if shardPath == "" {
+			generatedShardIds = append(generatedShardIds, uint32(shardId))
+			continue
+		}
+		if fi, statErr := os.Stat(shardPath); statErr == nil && fi.Size() == 0 {
+			// A zero-sized shard file is residue of a failed operation, not a
+			// shard; feeding it to Reed-Solomon fails the whole rebuild with a
+			// size mismatch. Treat it as missing and regenerate over it in
+			// place, like a reclassified-corrupt shard.
+			glog.Warningf("shard %d for %s is zero-sized at %s; excluding from rebuild inputs and regenerating", shardId, baseFileName, shardPath)
+			shardPaths[shardId] = shardPath
+			zeroSized = append(zeroSized, shardId)
+			generatedShardIds = append(generatedShardIds, uint32(shardId))
+			continue
+		}
+		shardHasData[shardId] = true
+		shardPaths[shardId] = shardPath
+		inputFiles[shardId], err = os.OpenFile(shardPath, os.O_RDONLY, 0)
+		if err != nil {
+			return nil, err
+		}
+		defer inputFiles[shardId].Close()
+		presentCount++
+	}
+
+	// Bitrot verify-and-exclude: when a generation-0 checksum sidecar is present
+	// and valid, verify each present input shard against it and reclassify
+	// corrupt ones as missing so Reed-Solomon regenerates them instead of
+	// silently consuming corrupt bytes. corruptOwned marks shards whose
+	// (corrupt) original file must be replaced in place at its discovered path.
+	corruptOwned := make([]bool, ctx.Total())
+	for _, shardId := range zeroSized {
+		corruptOwned[shardId] = true
+	}
+	prot, status := loadRebuildSidecar(baseFileName, ctx, additionalDirs)
+	switch status {
+	case BitrotInvalid:
+		if !unsafeIgnoreSidecar {
+			return nil, fmt.Errorf("bitrot sidecar for %s is malformed/unverifiable; refusing to rebuild (pass unsafeIgnoreSidecar to override)", baseFileName)
+		}
+		glog.Warningf("bitrot sidecar for %s is malformed/unverifiable; proceeding because unsafeIgnoreSidecar is set", baseFileName)
+	case BitrotOn:
+		corrupt := make([]int, 0, ctx.Total())
+		for shardId := 0; shardId < ctx.Total(); shardId++ {
+			if !shardHasData[shardId] {
+				continue
+			}
+			entry := shardChecksums(prot, uint32(shardId))
+			if entry == nil {
+				continue
+			}
+			mismatched, verr := verifyShardFileBlocks(shardPaths[shardId], entry, int64(prot.BlockSize))
+			if verr != nil {
+				// A read error means we cannot trust this shard as a Reed-Solomon
+				// input. Exclude it (treat as corrupt) rather than silently
+				// feeding possibly-corrupt bytes into reconstruction.
+				glog.Warningf("bitrot: failed to verify present shard %d for %s: %v; excluding it", shardId, baseFileName, verr)
+				corrupt = append(corrupt, shardId)
+				continue
+			}
+			if len(mismatched) > 0 {
+				corrupt = append(corrupt, shardId)
+			}
+		}
+		if len(corrupt) > 0 {
+			// Wholesale-mismatch guard (RS-arbiter conservative form): localized
+			// bitrot touches a few shards; a stale/wrong sidecar mismatches more
+			// than parity_shards. In that case refuse rather than excluding good
+			// shards en masse.
+			if len(corrupt) > ctx.ParityShards && !unsafeIgnoreSidecar {
+				return nil, fmt.Errorf("bitrot sidecar suspect for %s: %d/%d present shards mismatch (> parity %d); refusing to rebuild (pass unsafeIgnoreSidecar to override)",
+					baseFileName, len(corrupt), presentCount, ctx.ParityShards)
+			}
+			if presentCount-len(corrupt) < ctx.DataShards && !unsafeIgnoreSidecar {
+				return nil, fmt.Errorf("bitrot: only %d verified-good shards for %s, need %d data shards; sidecar may be stale (pass unsafeIgnoreSidecar to override)",
+					presentCount-len(corrupt), baseFileName, ctx.DataShards)
+			}
+			if !unsafeIgnoreSidecar {
+				for _, shardId := range corrupt {
+					glog.Warningf("bitrot: present shard %d for %s fails checksum; excluding from rebuild inputs and regenerating", shardId, baseFileName)
+					shardHasData[shardId] = false
+					corruptOwned[shardId] = true
+					generatedShardIds = append(generatedShardIds, uint32(shardId))
+					presentCount--
+				}
+			}
+		}
+	}
+
+	// Pre-check: bail out before creating any output files.
+	if presentCount < ctx.DataShards {
+		return nil, fmt.Errorf("not enough shards to rebuild %s: found %d shards, need at least %d (data shards), missing shards: %v",
+			baseFileName, presentCount, ctx.DataShards, generatedShardIds)
+	}
+
+	glog.V(0).Infof("rebuilding %s: %d shards present, %d missing %v, config %s",
+		baseFileName, presentCount, len(generatedShardIds), generatedShardIds, ctx.String())
+
+	// Pass 2: create output files for missing shards. A genuinely-absent shard
+	// is written at baseFileName+ext; a reclassified-corrupt shard is written to
+	// a temp file beside its discovered location and atomically renamed over the
+	// corrupt original after the rebuild (and checksum) succeed, so we never
+	// leave a duplicate shard id or a half-written file.
+	outputFiles := make([]*os.File, ctx.Total())
+	writePaths := make([]string, ctx.Total())
+	finalPaths := make([]string, ctx.Total())
+	for shardId := 0; shardId < ctx.Total(); shardId++ {
+		if shardHasData[shardId] {
+			continue
+		}
+		finalPath := baseFileName + ctx.ToExt(shardId)
+		writePath := finalPath
+		if corruptOwned[shardId] && shardPaths[shardId] != "" {
+			finalPath = shardPaths[shardId]
+			writePath = shardPaths[shardId] + ".rebuilding"
+		}
+		outputFiles[shardId], err = os.OpenFile(writePath, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			return nil, err
+		}
+		defer outputFiles[shardId].Close()
+		writePaths[shardId] = writePath
+		finalPaths[shardId] = finalPath
+	}
+
+	if err = rebuildEcFiles(shardHasData, inputFiles, outputFiles, ctx); err != nil {
+		return nil, fmt.Errorf("rebuildEcFiles: %w", err)
+	}
+
+	// Verify regenerated shards against the sidecar. Reed-Solomon is
+	// deterministic, so a regenerated shard that does NOT match the sidecar
+	// means the sidecar is wrong/stale (not the shard) — fail closed rather than
+	// publishing bytes we cannot trust. On ANY verification failure (sync, read
+	// error, or mismatch) remove every generated output so the rebuild publishes
+	// nothing: a genuinely-missing shard returns to missing; a reclassified-
+	// corrupt shard keeps its untouched original.
+	if status == BitrotOn && !unsafeIgnoreSidecar {
+		for shardId := 0; shardId < ctx.Total(); shardId++ {
+			if writePaths[shardId] == "" {
+				continue
+			}
+			entry := shardChecksums(prot, uint32(shardId))
+			if entry == nil {
+				continue
+			}
+			if err = outputFiles[shardId].Sync(); err != nil {
+				cleanupRebuildOutputs(outputFiles, writePaths)
+				return nil, fmt.Errorf("sync regenerated shard %d: %w", shardId, err)
+			}
+			mismatched, verr := verifyShardFileBlocks(writePaths[shardId], entry, int64(prot.BlockSize))
+			if verr != nil {
+				cleanupRebuildOutputs(outputFiles, writePaths)
+				return nil, fmt.Errorf("bitrot: verify regenerated shard %d for %s: %w", shardId, baseFileName, verr)
+			}
+			if len(mismatched) > 0 {
+				cleanupRebuildOutputs(outputFiles, writePaths)
+				return nil, fmt.Errorf("bitrot: regenerated shard %d for %s does not match sidecar (%d blocks differ); sidecar likely stale — aborting (pass unsafeIgnoreSidecar to override)",
+					shardId, baseFileName, len(mismatched))
+			}
+		}
+	}
+
+	// Atomically move reclassified-corrupt rebuilds over their originals.
+	for shardId := 0; shardId < ctx.Total(); shardId++ {
+		if writePaths[shardId] != "" && writePaths[shardId] != finalPaths[shardId] {
+			outputFiles[shardId].Close()
+			if rerr := os.Rename(writePaths[shardId], finalPaths[shardId]); rerr != nil {
+				return nil, fmt.Errorf("bitrot: replace corrupt shard %d (%s -> %s): %w", shardId, writePaths[shardId], finalPaths[shardId], rerr)
+			}
+		}
+	}
+	return
+}
+
+// cleanupRebuildOutputs removes every generated output on a failed fail-closed
+// rebuild: temp replacements AND genuinely-missing shards written directly at
+// their final path, so no unverified bytes are published. A reclassified-corrupt
+// shard's untouched original (at finalPath, distinct from its temp writePath) is
+// left in place; a genuinely-missing shard (writePath == finalPath) returns to
+// missing.
+func cleanupRebuildOutputs(outputFiles []*os.File, writePaths []string) {
+	for i := range writePaths {
+		if writePaths[i] == "" {
+			continue
+		}
+		if outputFiles[i] != nil {
+			outputFiles[i].Close()
+		}
+		os.Remove(writePaths[i])
+	}
+}
+
+// findVifPath locates the volume's `.vif` for a rebuild: next to the shards
+// first, then in every directory the caller also handed us — the index
+// directory of a split `-dir`/`-dir.idx` layout, and the sibling disks of a
+// multi-disk server, where `.ecx`/`.ecj`/`.vif` may live while this disk holds
+// only shards. Returns the data-base path when nothing exists, so the caller's
+// "not found" handling stays on the canonical name.
+func findVifPath(baseFileName string, additionalDirs []string) string {
+	dataPath := baseFileName + ".vif"
+	candidates := []string{dataPath}
+	base := filepath.Base(baseFileName)
+	for _, dir := range additionalDirs {
+		if dir == "" {
+			continue
+		}
+		candidates = append(candidates, filepath.Join(dir, base)+".vif")
+	}
+	for _, c := range candidates {
+		if _, err := os.Stat(c); err == nil {
+			return c
+		}
+	}
+	return dataPath
+}
+
+// loadRebuildSidecar loads and validates the generation-0 checksum sidecar for a
+// rebuild. RebuildEcFiles operates on the un-suffixed (generation 0) shard
+// names, so only the legacy sidecar is relevant here. Returns BitrotOff when
+// absent or describing a different generation/config, BitrotInvalid on a
+// self-integrity/manifest failure, BitrotOn when usable.
+func loadRebuildSidecar(baseFileName string, ctx *ECContext, additionalDirs []string) (*volume_server_pb.EcBitrotProtection, BitrotStatus) {
+	path := findBitrotSidecar(0, baseFileName, baseFileName, additionalDirs...)
+	if path == "" {
+		return nil, BitrotOff
+	}
+	prot, err := LoadBitrotSidecar(path)
+	if err != nil {
+		glog.Warningf("bitrot: sidecar %s self-integrity failed: %v", path, err)
+		return nil, BitrotInvalid
+	}
+	if prot.Generation != 0 {
+		return nil, BitrotOff
+	}
+	if prot.EcShardConfig == nil {
+		return nil, BitrotOff // records no geometry -> nothing to contradict
+	}
+	if int(prot.EcShardConfig.DataShards) != ctx.DataShards ||
+		int(prot.EcShardConfig.ParityShards) != ctx.ParityShards ||
+		prot.EcShardConfig.BlockSize != ctx.BlockSize {
+		// Both records describe the same encode, so a disagreement means the
+		// rebuild is about to reconstruct under a geometry the checksums do not
+		// cover. Treating that as "no protection" skipped every input and
+		// output check exactly when they matter most.
+		glog.Warningf("bitrot: sidecar %s records layout %d+%d block %d but the rebuild uses %d+%d block %d",
+			path, prot.EcShardConfig.DataShards, prot.EcShardConfig.ParityShards, prot.EcShardConfig.BlockSize,
+			ctx.DataShards, ctx.ParityShards, ctx.BlockSize)
+		return nil, BitrotInvalid
+	}
+	if err := ValidateBitrotManifest(prot, ctx.DataShards, ctx.ParityShards); err != nil {
+		glog.Warningf("bitrot: sidecar %s manifest invalid: %v", path, err)
+		return nil, BitrotInvalid
+	}
+	return prot, BitrotOn
+}
+
+func encodeData(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize int64, buffers [][]byte, outputs []*os.File, ctx *ECContext, builders []*shardChecksumBuilder) error {
+
+	bufferSize := int64(len(buffers[0]))
+	if bufferSize == 0 {
+		glog.Fatal("unexpected zero buffer size")
+	}
+
+	batchCount := blockSize / bufferSize
+	if blockSize%bufferSize != 0 {
+		glog.Fatalf("unexpected block size %d buffer size %d", blockSize, bufferSize)
+	}
+
+	for b := int64(0); b < batchCount; b++ {
+		err := encodeDataOneBatch(file, enc, startOffset+b*bufferSize, blockSize, buffers, outputs, ctx, builders)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func openEcFiles(baseFileName string, forRead bool, ctx *ECContext) (files []*os.File, err error) {
+	for i := 0; i < ctx.Total(); i++ {
+		fname := baseFileName + ctx.ToExt(i)
+		openOption := os.O_TRUNC | os.O_CREATE | os.O_WRONLY
+		if forRead {
+			openOption = os.O_RDONLY
+		}
+		f, err := os.OpenFile(fname, openOption, 0644)
+		if err != nil {
+			return files, fmt.Errorf("failed to open file %s: %v", fname, err)
+		}
+		files = append(files, f)
+	}
+	return
+}
+
+func closeEcFiles(files []*os.File) {
+	for _, f := range files {
+		if f != nil {
+			f.Close()
+		}
+	}
+}
+
+func encodeDataOneBatch(file *os.File, enc reedsolomon.Encoder, startOffset, blockSize int64, buffers [][]byte, outputs []*os.File, ctx *ECContext, builders []*shardChecksumBuilder) error {
+
+	// read data into buffers
+	for i := 0; i < ctx.DataShards; i++ {
+		n, err := file.ReadAt(buffers[i], startOffset+blockSize*int64(i))
+		if err != nil {
+			if err != io.EOF {
+				return err
+			}
+		}
+		if n < len(buffers[i]) {
+			for t := len(buffers[i]) - 1; t >= n; t-- {
+				buffers[i][t] = 0
+			}
+		}
+	}
+
+	err := enc.Encode(buffers)
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < ctx.Total(); i++ {
+		_, err := outputs[i].Write(buffers[i])
+		if err != nil {
+			return err
+		}
+		// Accumulate this shard's block CRC over exactly the bytes written.
+		if builders != nil && builders[i] != nil {
+			builders[i].write(buffers[i])
+		}
+	}
+
+	return nil
+}
+
+func encodeDatFile(remainingSize int64, baseFileName string, bufferSize int, largeBlockSize int64, file *os.File, smallBlockSize int64, ctx *ECContext, builders []*shardChecksumBuilder) error {
+
+	var processedSize int64
+
+	enc, err := ctx.CreateEncoder()
+	if err != nil {
+		return fmt.Errorf("failed to create encoder: %w", err)
+	}
+
+	buffers := make([][]byte, ctx.Total())
+	for i := range buffers {
+		buffers[i] = make([]byte, bufferSize)
+	}
+
+	outputs, err := openEcFiles(baseFileName, false, ctx)
+	defer closeEcFiles(outputs)
+	if err != nil {
+		return fmt.Errorf("failed to open ec files %s: %v", baseFileName, err)
+	}
+
+	// Pre-calculate row sizes to avoid redundant calculations in loops
+	largeRowSize := largeBlockSize * int64(ctx.DataShards)
+	smallRowSize := smallBlockSize * int64(ctx.DataShards)
+
+	for remainingSize >= largeRowSize {
+		err = encodeData(file, enc, processedSize, largeBlockSize, buffers, outputs, ctx, builders)
+		if err != nil {
+			return fmt.Errorf("failed to encode large chunk data: %w", err)
+		}
+		remainingSize -= largeRowSize
+		processedSize += largeRowSize
+	}
+	for remainingSize > 0 {
+		err = encodeData(file, enc, processedSize, smallBlockSize, buffers, outputs, ctx, builders)
+		if err != nil {
+			return fmt.Errorf("failed to encode small chunk data: %w", err)
+		}
+		remainingSize -= smallRowSize
+		processedSize += smallRowSize
+	}
+	return nil
+}
+
+func rebuildEcFiles(shardHasData []bool, inputFiles []*os.File, outputFiles []*os.File, ctx *ECContext) error {
+
+	enc, err := ctx.CreateEncoder()
+	if err != nil {
+		return fmt.Errorf("failed to create encoder: %w", err)
+	}
+
+	// The output shard size equals the present input shards' size (all EC
+	// shards are equal length). Deriving it up front turns a short read on a
+	// truncated/corrupt input into an error instead of a silent early return
+	// that would publish truncated shards as restored redundancy.
+	var expectedShardSize int64 = -1
+	for i := 0; i < ctx.Total(); i++ {
+		if !shardHasData[i] {
+			continue
+		}
+		fi, statErr := inputFiles[i].Stat()
+		if statErr != nil {
+			return fmt.Errorf("stat input shard %d: %w", i, statErr)
+		}
+		if expectedShardSize < 0 {
+			expectedShardSize = fi.Size()
+		} else if fi.Size() != expectedShardSize {
+			return fmt.Errorf("ec rebuild: input shard %d size %d != %d (truncated input?)", i, fi.Size(), expectedShardSize)
+		}
+	}
+	if expectedShardSize <= 0 {
+		return fmt.Errorf("ec rebuild: no input shard data (expected shard size %d)", expectedShardSize)
+	}
+
+	buffers := make([][]byte, ctx.Total())
+	for i := range buffers {
+		if shardHasData[i] {
+			buffers[i] = make([]byte, ErasureCodingSmallBlockSize)
+		}
+	}
+
+	for startOffset := int64(0); startOffset < expectedShardSize; {
+		thisBlock := int64(ErasureCodingSmallBlockSize)
+		if remaining := expectedShardSize - startOffset; remaining < thisBlock {
+			thisBlock = remaining
+		}
+
+		// read the input data; a short read means a truncated input shard.
+		shards := make([][]byte, ctx.Total())
+		for i := 0; i < ctx.Total(); i++ {
+			if !shardHasData[i] {
+				continue // nil: reconstructed below
+			}
+			b := buffers[i][:thisBlock]
+			n, readErr := inputFiles[i].ReadAt(b, startOffset)
+			if readErr != nil && readErr != io.EOF {
+				return fmt.Errorf("ec rebuild read shard %d at %d: %w", i, startOffset, readErr)
+			}
+			if int64(n) != thisBlock {
+				return fmt.Errorf("ec rebuild short read shard %d at %d: got %d want %d", i, startOffset, n, thisBlock)
+			}
+			shards[i] = b
+		}
+
+		if err = enc.Reconstruct(shards); err != nil {
+			return fmt.Errorf("reconstruct: %w", err)
+		}
+
+		for i := 0; i < ctx.Total(); i++ {
+			if shardHasData[i] {
+				continue
+			}
+			n, writeErr := outputFiles[i].WriteAt(shards[i][:thisBlock], startOffset)
+			if writeErr != nil {
+				return fmt.Errorf("ec rebuild write shard %d at %d: %w", i, startOffset, writeErr)
+			}
+			if int64(n) != thisBlock {
+				return fmt.Errorf("ec rebuild short write shard %d at %d: got %d want %d", i, startOffset, n, thisBlock)
+			}
+		}
+		startOffset += thisBlock
+	}
+
+	// Flush every regenerated shard before it is mounted/renamed and published
+	// as restored redundancy, so a crash cannot leave a peer trusting a shard
+	// whose bytes never reached disk.
+	for i := 0; i < ctx.Total(); i++ {
+		if shardHasData[i] {
+			continue
+		}
+		if err = outputFiles[i].Sync(); err != nil {
+			return fmt.Errorf("ec rebuild sync shard %d: %w", i, err)
+		}
+	}
+
+	return nil
+}
+
+func readNeedleMap(baseFileName string) (*needle_map.MemDb, error) {
+	indexFile, err := os.OpenFile(baseFileName+".idx", os.O_RDONLY, 0644)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read Volume Index %s.idx: %v", baseFileName, err)
+	}
+	defer indexFile.Close()
+
+	cm := needle_map.NewMemDb()
+	err = idx.WalkIndexFile(indexFile, 0, func(key types.NeedleId, offset types.Offset, size types.Size) error {
+		if !offset.IsZero() && !size.IsDeleted() {
+			cm.Set(key, offset, size)
+		} else {
+			cm.Delete(key)
+		}
+		return nil
+	})
+	return cm, err
+}

weed/storage/erasure_coding/ec_locate.go

@@ -0,0 +1,98 @@
+package erasure_coding
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+type Interval struct {
+	BlockIndex          int // the index of the block in either the large blocks or the small blocks
+	InnerBlockOffset    int64
+	Size                types.Size
+	IsLargeBlock        bool // whether the block is a large block or a small block
+	LargeBlockRowsCount int
+}
+
+// LocateData finds the intervals of data within erasure coding blocks for a given offset and size.
+func LocateData(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64, size types.Size) (intervals []Interval) {
+	blockIndex, isLargeBlock, nLargeBlockRows, innerBlockOffset := locateOffset(largeBlockLength, smallBlockLength, shardDatSize, offset)
+
+	for size > 0 {
+		blockRemaining := largeBlockLength - innerBlockOffset
+		if !isLargeBlock {
+			blockRemaining = smallBlockLength - innerBlockOffset
+		}
+
+		if blockRemaining <= 0 {
+			// move to next block
+			blockIndex, isLargeBlock = moveToNextBlock(blockIndex, isLargeBlock, nLargeBlockRows)
+			innerBlockOffset = 0
+			continue
+		}
+
+		interval := Interval{
+			BlockIndex:          blockIndex,
+			InnerBlockOffset:    innerBlockOffset,
+			IsLargeBlock:        isLargeBlock,
+			LargeBlockRowsCount: int(nLargeBlockRows),
+		}
+
+		if int64(size) <= blockRemaining {
+			interval.Size = size
+			intervals = append(intervals, interval)
+			return
+		}
+		interval.Size = types.Size(blockRemaining)
+		intervals = append(intervals, interval)
+
+		size -= interval.Size
+		blockIndex, isLargeBlock = moveToNextBlock(blockIndex, isLargeBlock, nLargeBlockRows)
+		innerBlockOffset = 0
+
+	}
+	return
+}
+
+func moveToNextBlock(blockIndex int, isLargeBlock bool, nLargeBlockRows int64) (int, bool) {
+	nextBlockIndex := blockIndex + 1
+	nextIsLargeBlock := isLargeBlock
+	if isLargeBlock && int64(nextBlockIndex) == nLargeBlockRows*DataShardsCount {
+		nextIsLargeBlock = false
+		nextBlockIndex = 0
+	}
+	return nextBlockIndex, nextIsLargeBlock
+}
+
+func locateOffset(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64) (blockIndex int, isLargeBlock bool, nLargeBlockRows int64, innerBlockOffset int64) {
+	largeRowSize := largeBlockLength * DataShardsCount
+	nLargeBlockRows = shardDatSize / largeBlockLength
+
+	// if offset is within the large block area
+	if offset < nLargeBlockRows*largeRowSize {
+		isLargeBlock = true
+		blockIndex, innerBlockOffset = locateOffsetWithinBlocks(largeBlockLength, offset)
+		return
+	}
+
+	isLargeBlock = false
+	offset -= nLargeBlockRows * largeRowSize
+	blockIndex, innerBlockOffset = locateOffsetWithinBlocks(smallBlockLength, offset)
+	return
+}
+
+func locateOffsetWithinBlocks(blockLength int64, offset int64) (blockIndex int, innerBlockOffset int64) {
+	blockIndex = int(offset / blockLength)
+	innerBlockOffset = offset % blockLength
+	return
+}
+
+func (interval Interval) ToShardIdAndOffset(largeBlockSize, smallBlockSize int64) (ShardId, int64) {
+	ecFileOffset := interval.InnerBlockOffset
+	rowIndex := interval.BlockIndex / DataShardsCount
+	if interval.IsLargeBlock {
+		ecFileOffset += int64(rowIndex) * largeBlockSize
+	} else {
+		ecFileOffset += int64(interval.LargeBlockRowsCount)*largeBlockSize + int64(rowIndex)*smallBlockSize
+	}
+	ecFileIndex := interval.BlockIndex % DataShardsCount
+	return ShardId(ecFileIndex), ecFileOffset
+}

weed/storage/erasure_coding/ec_parity_shards_ratio_test.go

@@ -0,0 +1,49 @@
+package erasure_coding
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestMinusParityShardsRatio covers the data/parity split for the default 10+4
+// layout and other ratios. The pre-fix code hardcoded ids 10..13 as parity,
+// which for a wide ratio (12+4, 16+6) DROPPED real data ids >= 10 and for a
+// narrow ratio (9+3) kept parity id 9. The method now takes the data-shard
+// count (<= 0 falls back to DataShardsCount).
+func TestMinusParityShardsRatio(t *testing.T) {
+	all := func(n int) []ShardId {
+		ids := make([]ShardId, n)
+		for i := range ids {
+			ids[i] = ShardId(i)
+		}
+		return ids
+	}
+	tests := []struct {
+		name          string
+		dataShards    int
+		inputShards   []ShardId
+		expectedCount int
+		expectedIds   []ShardId
+	}{
+		{"default 10+4 all present", DataShardsCount, all(14), 10, all(10)},
+		{"zero ratio falls back to default", 0, all(14), 10, all(10)},
+		{"9+3 all present", 9, all(12), 9, all(9)},
+		{"9+3 drops parity id 9", 9, []ShardId{0, 1, 2, 9, 10, 11}, 3, []ShardId{0, 1, 2}},
+		{"12+4 keeps data ids >= 10", 12, all(16), 12, all(12)},
+		{"16+6 keeps data ids >= 10", 16, all(22), 16, all(16)},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			si := NewShardsInfo()
+			for _, id := range tt.inputShards {
+				si.Set(NewShardInfo(id, ShardSize(1000)))
+			}
+			result := si.MinusParityShards(tt.dataShards)
+			assert.Equal(t, tt.expectedCount, result.Count(), "count for ratio %d", tt.dataShards)
+			for _, id := range tt.expectedIds {
+				assert.True(t, result.Has(id), "data shard %d must remain for ratio %d", id, tt.dataShards)
+			}
+		})
+	}
+}

weed/storage/erasure_coding/ec_rebuild_safety_test.go

@@ -0,0 +1,398 @@
+package erasure_coding
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+)
+
+// readEcxSizeField returns the size field of the .ecx entry for needleId, or
+// (false) if the id is absent. Entries are fixed-width (id, offset, size).
+func readEcxSizeField(t *testing.T, ecxPath string, needleId types.NeedleId) (types.Size, bool) {
+	t.Helper()
+	data, err := os.ReadFile(ecxPath)
+	if err != nil {
+		t.Fatalf("read ecx: %v", err)
+	}
+	entry := types.NeedleIdSize + types.OffsetSize + types.SizeSize
+	for off := 0; off+entry <= len(data); off += entry {
+		id := types.BytesToNeedleId(data[off : off+types.NeedleIdSize])
+		if id == needleId {
+			return types.BytesToSize(data[off+types.NeedleIdSize+types.OffsetSize : off+entry]), true
+		}
+	}
+	return 0, false
+}
+
+func writeEcxEntries(t *testing.T, ecxPath string, ids []types.NeedleId) {
+	t.Helper()
+	var buf []byte
+	for i, id := range ids {
+		b := make([]byte, types.NeedleIdSize+types.OffsetSize+types.SizeSize)
+		types.NeedleIdToBytes(b[0:types.NeedleIdSize], id)
+		// A plausible non-zero offset/size; the fold only rewrites the size field.
+		types.OffsetToBytes(b[types.NeedleIdSize:types.NeedleIdSize+types.OffsetSize], types.ToOffset(int64((i+1)*1024)))
+		types.SizeToBytes(b[types.NeedleIdSize+types.OffsetSize:], types.Size(100))
+		buf = append(buf, b...)
+	}
+	if err := os.WriteFile(ecxPath, buf, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+}
+
+func packEcjIds(ids []types.NeedleId) []byte {
+	var buf []byte
+	for _, id := range ids {
+		b := make([]byte, types.NeedleIdSize)
+		types.NeedleIdToBytes(b, id)
+		buf = append(buf, b...)
+	}
+	return buf
+}
+
+// TestRebuildEcxFile_AppliesTombstonesAndRemovesJournal: a clean fold marks the
+// journaled needle deleted in .ecx and removes the journal.
+func TestRebuildEcxFile_AppliesTombstonesAndRemovesJournal(t *testing.T) {
+	base := filepath.Join(t.TempDir(), "vol")
+	writeEcxEntries(t, base+".ecx", []types.NeedleId{1, 2, 3})
+	if err := os.WriteFile(base+".ecj", packEcjIds([]types.NeedleId{2}), 0644); err != nil {
+		t.Fatalf("write ecj: %v", err)
+	}
+
+	if err := RebuildEcxFile(base); err != nil {
+		t.Fatalf("RebuildEcxFile: %v", err)
+	}
+
+	if _, err := os.Stat(base + ".ecj"); !os.IsNotExist(err) {
+		t.Errorf(".ecj should be removed after a clean fold, stat err=%v", err)
+	}
+	if sz, ok := readEcxSizeField(t, base+".ecx", 2); !ok || sz != types.TombstoneFileSize {
+		t.Errorf("needle 2 should be tombstoned in .ecx, got size=%d ok=%v", sz, ok)
+	}
+	for _, live := range []types.NeedleId{1, 3} {
+		if sz, ok := readEcxSizeField(t, base+".ecx", live); !ok || sz == types.TombstoneFileSize {
+			t.Errorf("needle %d should remain live, got size=%d ok=%v", live, sz, ok)
+		}
+	}
+}
+
+// TestRebuildEcxFile_TornJournalAborts: a journal whose tail is a partial record
+// (e.g. interrupted append) must abort the fold and leave .ecj in place for a
+// retry, rather than silently dropping the remaining tombstones before unlink.
+func TestRebuildEcxFile_TornJournalAborts(t *testing.T) {
+	base := filepath.Join(t.TempDir(), "vol")
+	writeEcxEntries(t, base+".ecx", []types.NeedleId{1, 2, 3})
+	// One full id followed by a truncated trailing record.
+	torn := append(packEcjIds([]types.NeedleId{2}), []byte{0x01, 0x02, 0x03}...)
+	if err := os.WriteFile(base+".ecj", torn, 0644); err != nil {
+		t.Fatalf("write ecj: %v", err)
+	}
+
+	if err := RebuildEcxFile(base); err == nil {
+		t.Fatal("expected an error folding a torn journal, got nil")
+	}
+	if _, err := os.Stat(base + ".ecj"); err != nil {
+		t.Errorf(".ecj must be retained after a torn-journal abort, stat err=%v", err)
+	}
+}
+
+// TestRebuildEcxFile_NoJournalNoop: with no .ecj the fold is a no-op and does
+// not error (idempotent re-run after a prior successful fold).
+func TestRebuildEcxFile_NoJournalNoop(t *testing.T) {
+	base := filepath.Join(t.TempDir(), "vol")
+	writeEcxEntries(t, base+".ecx", []types.NeedleId{1, 2, 3})
+	before, _ := os.ReadFile(base + ".ecx")
+
+	if err := RebuildEcxFile(base); err != nil {
+		t.Fatalf("RebuildEcxFile with no journal: %v", err)
+	}
+	after, _ := os.ReadFile(base + ".ecx")
+	if len(before) != len(after) {
+		t.Errorf(".ecx changed despite no journal: %d -> %d bytes", len(before), len(after))
+	}
+}
+
+// makeRebuildInputs creates `present` equal-size input shard files of `size`
+// bytes and one missing output file, returning slices sized to ctx.Total().
+func makeRebuildInputs(t *testing.T, dir string, present, missingIdx int, size int) ([]bool, []*os.File, []*os.File) {
+	t.Helper()
+	ctx := NewDefaultECContext("", 0)
+	shardHasData := make([]bool, ctx.Total())
+	inputFiles := make([]*os.File, ctx.Total())
+	outputFiles := make([]*os.File, ctx.Total())
+	for i := 0; i < present; i++ {
+		p := filepath.Join(dir, "in"+ToExt(i))
+		if err := os.WriteFile(p, make([]byte, size), 0644); err != nil {
+			t.Fatalf("write input %d: %v", i, err)
+		}
+		f, err := os.Open(p)
+		if err != nil {
+			t.Fatalf("open input %d: %v", i, err)
+		}
+		t.Cleanup(func() { f.Close() })
+		inputFiles[i] = f
+		shardHasData[i] = true
+	}
+	outPath := filepath.Join(dir, "out"+ToExt(missingIdx))
+	of, err := os.OpenFile(outPath, os.O_CREATE|os.O_RDWR, 0644)
+	if err != nil {
+		t.Fatalf("open output: %v", err)
+	}
+	t.Cleanup(func() { of.Close() })
+	outputFiles[missingIdx] = of
+	shardHasData[missingIdx] = false
+	return shardHasData, inputFiles, outputFiles
+}
+
+// TestRebuildEcFiles_TruncatedInputErrorsWithoutPublishing: a truncated input
+// shard must abort the rebuild before any output is written, rather than
+// reconstructing over a zero-padded tail and publishing a corrupt shard.
+func TestRebuildEcFiles_TruncatedInputErrorsWithoutPublishing(t *testing.T) {
+	dir := t.TempDir()
+	ctx := NewDefaultECContext("", 0)
+	const size = 4096
+	shardHasData, inputFiles, outputFiles := makeRebuildInputs(t, dir, ctx.Total()-1, ctx.Total()-1, size)
+
+	// Truncate one present input to half.
+	if err := os.Truncate(filepath.Join(dir, "in"+ToExt(5)), size/2); err != nil {
+		t.Fatalf("truncate: %v", err)
+	}
+
+	err := rebuildEcFiles(shardHasData, inputFiles, outputFiles, ctx)
+	if err == nil {
+		t.Fatal("expected an error on a truncated input shard, got nil")
+	}
+	if fi, _ := os.Stat(filepath.Join(dir, "out"+ToExt(ctx.Total()-1))); fi != nil && fi.Size() != 0 {
+		t.Errorf("output shard must not be published on a truncated input, got %d bytes", fi.Size())
+	}
+}
+
+// TestRebuildEcFiles_ZeroSizeInputsErrors: all-empty input shards must error
+// rather than producing and publishing zero-length output shards.
+func TestRebuildEcFiles_ZeroSizeInputsErrors(t *testing.T) {
+	dir := t.TempDir()
+	ctx := NewDefaultECContext("", 0)
+	shardHasData, inputFiles, outputFiles := makeRebuildInputs(t, dir, ctx.Total()-1, ctx.Total()-1, 0)
+
+	if err := rebuildEcFiles(shardHasData, inputFiles, outputFiles, ctx); err == nil {
+		t.Fatal("expected an error on zero-size input shards, got nil")
+	}
+}
+
+// TestRebuildEcFiles_HappyPathRebuildsByteIdentical: a real encode, drop one
+// shard, rebuild, and confirm the regenerated shard is byte-identical. The tiny
+// .dat exercises the sub-1MB final-block path (the guard must not false-positive
+// on a legitimate short tail).
+func TestRebuildEcFiles_HappyPathRebuildsByteIdentical(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "vol")
+	ctx := NewDefaultECContext("", 0)
+	writeRandomDat(t, base, 7000)
+
+	if _, err := generateEcFiles(base, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx); err != nil {
+		t.Fatalf("generateEcFiles: %v", err)
+	}
+	const dropped = 5
+	want, err := os.ReadFile(base + ToExt(dropped))
+	if err != nil {
+		t.Fatalf("read shard %d: %v", dropped, err)
+	}
+	if err := os.Remove(base + ToExt(dropped)); err != nil {
+		t.Fatalf("remove shard: %v", err)
+	}
+
+	if _, err := RebuildEcFiles(base, ctx, true); err != nil {
+		t.Fatalf("RebuildEcFiles: %v", err)
+	}
+	got, err := os.ReadFile(base + ToExt(dropped))
+	if err != nil {
+		t.Fatalf("read rebuilt shard: %v", err)
+	}
+	if string(got) != string(want) {
+		t.Errorf("rebuilt shard %d differs from original (%d vs %d bytes)", dropped, len(got), len(want))
+	}
+}
+
+// TestRebuildEcFiles_ZeroSizedShardRegeneratedInPlace: a zero-sized shard file
+// (residue of a failed operation, issue 10730) must be excluded from the
+// rebuild inputs and regenerated byte-identical over the empty file, instead
+// of failing the whole rebuild with an input size mismatch.
+func TestRebuildEcFiles_ZeroSizedShardRegeneratedInPlace(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "vol")
+	ctx := NewDefaultECContext("", 0)
+	writeRandomDat(t, base, 7000)
+
+	if _, err := generateEcFiles(base, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx); err != nil {
+		t.Fatalf("generateEcFiles: %v", err)
+	}
+
+	const emptied = 2
+	want, err := os.ReadFile(base + ToExt(emptied))
+	if err != nil {
+		t.Fatalf("read shard %d: %v", emptied, err)
+	}
+	if err := os.Truncate(base+ToExt(emptied), 0); err != nil {
+		t.Fatalf("truncate shard: %v", err)
+	}
+
+	generated, err := RebuildEcFiles(base, ctx, true)
+	if err != nil {
+		t.Fatalf("RebuildEcFiles with a zero-sized shard: %v", err)
+	}
+	found := false
+	for _, id := range generated {
+		if id == emptied {
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("shard %d not reported as regenerated: %v", emptied, generated)
+	}
+	got, err := os.ReadFile(base + ToExt(emptied))
+	if err != nil {
+		t.Fatalf("read regenerated shard: %v", err)
+	}
+	if string(got) != string(want) {
+		t.Errorf("regenerated shard %d differs from original (%d vs %d bytes)", emptied, len(got), len(want))
+	}
+}
+
+// TestRebuildEcFiles_CustomRatioRebuildsByteIdentical: the same for a 9+3 ratio,
+// confirming no 10+4 assumption leaks into the destructive rebuild path.
+func TestRebuildEcFiles_CustomRatioRebuildsByteIdentical(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "vol")
+	ctx := &ECContext{DataShards: 9, ParityShards: 3}
+	writeRandomDat(t, base, 9000)
+
+	if _, err := generateEcFiles(base, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx); err != nil {
+		t.Fatalf("generateEcFiles: %v", err)
+	}
+	const dropped = 7
+	want, err := os.ReadFile(base + ToExt(dropped))
+	if err != nil {
+		t.Fatalf("read shard %d: %v", dropped, err)
+	}
+	if err := os.Remove(base + ToExt(dropped)); err != nil {
+		t.Fatalf("remove shard: %v", err)
+	}
+
+	if _, err := RebuildEcFiles(base, ctx, true); err != nil {
+		t.Fatalf("RebuildEcFiles: %v", err)
+	}
+	got, err := os.ReadFile(base + ToExt(dropped))
+	if err != nil {
+		t.Fatalf("read rebuilt shard: %v", err)
+	}
+	if string(got) != string(want) {
+		t.Errorf("rebuilt 9+3 shard %d differs (%d vs %d bytes)", dropped, len(got), len(want))
+	}
+	// No shard position beyond the 9+3 layout must have been created.
+	if _, err := os.Stat(base + ToExt(12)); err == nil {
+		t.Errorf("unexpected shard .ec12 for a 9+3 volume")
+	}
+}
+
+// With no .vif, the bitrot sidecar is the surviving record of the volume's
+// geometry. Defaulting to 10+4 and the legacy block layout instead would
+// reconstruct a custom-ratio volume from the wrong matrix — and would make the
+// sidecar look like it disagrees, silently skipping every checksum check.
+func TestRebuildEcFiles_ResolvesGeometryFromSidecarWithoutVif(t *testing.T) {
+	if !BitrotProtectionEnabled {
+		t.Skip("bitrot protection is compiled out")
+	}
+	dir := t.TempDir()
+	base := filepath.Join(dir, "vol")
+	ctx := &ECContext{DataShards: 12, ParityShards: 4}
+	writeRandomDat(t, base, 7000)
+
+	prot, err := WriteEcFiles(base, ctx)
+	if err != nil {
+		t.Fatalf("WriteEcFiles: %v", err)
+	}
+	if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), prot); err != nil {
+		t.Fatalf("save sidecar: %v", err)
+	}
+	// No .vif at all: the sidecar has to answer.
+	const dropped = 11
+	want, err := os.ReadFile(base + ToExt(dropped))
+	if err != nil {
+		t.Fatalf("read shard: %v", err)
+	}
+	if err := os.Remove(base + ToExt(dropped)); err != nil {
+		t.Fatalf("remove shard: %v", err)
+	}
+
+	// nil ctx: the production RPC's BackgroundECContext path.
+	if _, err := RebuildEcFiles(base, nil, false); err != nil {
+		t.Fatalf("RebuildEcFiles: %v", err)
+	}
+	got, err := os.ReadFile(base + ToExt(dropped))
+	if err != nil {
+		t.Fatalf("read rebuilt shard: %v", err)
+	}
+	if string(got) != string(want) {
+		t.Errorf("rebuilt shard %d differs; the rebuild used the wrong geometry", dropped)
+	}
+	// The 12+4 shards past the default ratio must still be there.
+	for _, id := range []int{14, 15} {
+		if _, err := os.Stat(base + ToExt(id)); err != nil {
+			t.Errorf("shard %d missing after rebuild: %v", id, err)
+		}
+	}
+}
+
+// A split -dir/-dir.idx layout keeps the .vif with the index, and a multi-disk
+// server may leave this disk holding only shards. Probing just the data base
+// resolves such a volume to the default ratio and the legacy block layout.
+func TestRebuildEcFiles_FindsVifInAnAdditionalDir(t *testing.T) {
+	dataDir := t.TempDir()
+	idxDir := t.TempDir()
+	base := filepath.Join(dataDir, "vol")
+	ctx := &ECContext{DataShards: 12, ParityShards: 4}
+	writeRandomDat(t, base, 7000)
+
+	if _, err := WriteEcFiles(base, ctx); err != nil {
+		t.Fatalf("WriteEcFiles: %v", err)
+	}
+	// The volume's .vif lives with the index, not the shards.
+	if err := volume_info.SaveVolumeInfo(filepath.Join(idxDir, "vol")+".vif", &volume_server_pb.VolumeInfo{
+		Version: uint32(needle.GetCurrentVersion()),
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards: 12, ParityShards: 4, BlockSize: ctx.BlockSize,
+		},
+	}); err != nil {
+		t.Fatalf("save .vif: %v", err)
+	}
+
+	const dropped = 11
+	want, err := os.ReadFile(base + ToExt(dropped))
+	if err != nil {
+		t.Fatalf("read shard: %v", err)
+	}
+	if err := os.Remove(base + ToExt(dropped)); err != nil {
+		t.Fatalf("remove shard: %v", err)
+	}
+
+	if _, err := RebuildEcFiles(base, nil, true, idxDir); err != nil {
+		t.Fatalf("RebuildEcFiles: %v", err)
+	}
+	got, err := os.ReadFile(base + ToExt(dropped))
+	if err != nil {
+		t.Fatalf("read rebuilt shard: %v", err)
+	}
+	if string(got) != string(want) {
+		t.Errorf("rebuilt shard %d differs; the rebuild used the wrong geometry", dropped)
+	}
+	for _, id := range []int{14, 15} {
+		if _, err := os.Stat(base + ToExt(id)); err != nil {
+			t.Errorf("shard %d missing after rebuild: %v", id, err)
+		}
+	}
+}

weed/storage/erasure_coding/ec_roundtrip_test.go

@@ -0,0 +1,509 @@
+package erasure_coding
+
+import (
+	"bytes"
+	"crypto/rand"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// TestEcReadRoundTrip tests the EC encode→read cycle via LocateData for various
+// .dat file sizes, paying special attention to the large/small block boundary.
+//
+// The nLargeBlockRows calculation must correctly distinguish between large and small
+// blocks. A previous bug (issue #8947) caused an off-by-one error when
+// shardDatSize was an exact multiple of largeBlockSize, leading to data corruption.
+func TestEcReadRoundTrip(t *testing.T) {
+	const (
+		large = largeBlockSize // 10000
+		small = smallBlockSize // 100
+	)
+
+	largeRowSize := large * DataShardsCount // 100000
+	smallRowSize := small * DataShardsCount // 1000
+
+	testCases := []struct {
+		name    string
+		datSize int64
+	}{
+		// Exact multiples of largeRowSize — triggers the nLargeBlockRows off-by-one bug
+		{"1_large_row_exact", int64(largeRowSize)},
+		{"2_large_rows_exact", int64(2 * largeRowSize)},
+		{"3_large_rows_exact", int64(3 * largeRowSize)},
+
+		// Just over a large row boundary — has small blocks
+		{"1_large_row_plus_1", int64(largeRowSize + 1)},
+		{"2_large_rows_plus_small", int64(2*largeRowSize + smallRowSize)},
+		{"1_large_row_plus_half_small", int64(largeRowSize + smallRowSize/2)},
+
+		// Just under a large row boundary — all small blocks
+		{"just_under_1_large_row", int64(largeRowSize - 1)},
+		{"just_under_2_large_rows", int64(2*largeRowSize - 1)},
+
+		// Small data — no large blocks at all
+		{"small_only", int64(smallRowSize * 3)},
+		{"small_single_row", int64(smallRowSize)},
+
+		// Boundary with mixed large and small
+		{"boundary_spanning", int64(largeRowSize + smallRowSize*5 + 50)},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			testEcRead(t, large, small, tc.datSize)
+		})
+	}
+}
+
+// testEcRead creates a .dat file, EC-encodes it, then verifies LocateData-based reads
+// return correct data at positions throughout the file (especially near the large/small
+// block boundary).
+func testEcRead(t *testing.T, large, small, datSize int64) {
+	t.Helper()
+
+	dir := t.TempDir()
+	baseFileName := fmt.Sprintf("%s/rt_%d", dir, datSize)
+
+	// 1. Create a .dat file with deterministic random data
+	originalData := make([]byte, datSize)
+	_, err := rand.Read(originalData)
+	require.NoError(t, err, "generating random data")
+
+	err = os.WriteFile(baseFileName+".dat", originalData, 0644)
+	require.NoError(t, err, "writing .dat file")
+
+	ctx := NewDefaultECContext("", 0)
+
+	// 2. EC encode with test block sizes
+	_, err = generateEcFiles(baseFileName, int(small), large, small, ctx)
+	require.NoError(t, err, "EC encoding")
+
+	// 3. Open EC shard files for reading
+	ecFiles, err := openEcFiles(baseFileName, true, ctx)
+	require.NoError(t, err, "opening EC files")
+	defer closeEcFiles(ecFiles)
+
+	ecStat, err := ecFiles[0].Stat()
+	require.NoError(t, err)
+	shardFileSize := ecStat.Size()
+
+	// Compute shardDatSize as the production code does when datFileSize is known
+	shardDatSizeFromDat := datSize / int64(ctx.DataShards)
+
+	// 4. Verify EC reads at various positions
+	largeRowSize := large * DataShardsCount
+	encoderLargeRows := datSize / int64(largeRowSize)
+	boundaryOffset := encoderLargeRows * int64(largeRowSize)
+
+	readSize := types.Size(small / 2) // read half a small block
+	testOffsets := collectTestOffsets(datSize, int64(readSize), boundaryOffset, large, small)
+
+	for _, offset := range testOffsets {
+		// Test with shardDatSize from datFileSize (the production path with fix)
+		intervals := LocateData(large, small, shardDatSizeFromDat, offset, readSize)
+		ecData, err := assembleFromIntervals(ecFiles, intervals, large, small)
+		require.NoError(t, err, "reading EC data at offset %d (datFileSize path)", offset)
+
+		expected := originalData[offset : offset+int64(readSize)]
+		if !bytes.Equal(expected, ecData) {
+			t.Errorf("EC read mismatch at offset %d (datFileSize path, shardDatSize=%d, nLargeBlockRows=%d)",
+				offset, shardDatSizeFromDat, shardDatSizeFromDat/large)
+		}
+
+		// Test with shardDatSize from ecdFileSize-1 (the fallback path for old volumes)
+		intervalsFallback := LocateData(large, small, shardFileSize-1, offset, readSize)
+		ecDataFallback, err := assembleFromIntervals(ecFiles, intervalsFallback, large, small)
+		if err == nil && !bytes.Equal(expected, ecDataFallback) {
+			// The fallback path may fail for exact multiples — log as warning
+			t.Logf("WARN: EC read mismatch at offset %d (fallback path, shardFileSize=%d)",
+				offset, shardFileSize)
+		}
+	}
+}
+
+// locateOffsetBuggy reimplements locateOffset with the old buggy formula:
+//
+//	nLargeBlockRows = (shardDatSize - 1) / largeBlockLength
+//
+// This caused an off-by-one error when shardDatSize was an exact multiple of
+// largeBlockLength, miscounting the number of large block rows.
+func locateOffsetBuggy(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64) (blockIndex int, isLargeBlock bool, nLargeBlockRows int64, innerBlockOffset int64) {
+	largeRowSize := largeBlockLength * DataShardsCount
+	nLargeBlockRows = (shardDatSize - 1) / largeBlockLength // THE BUG
+
+	if offset < nLargeBlockRows*largeRowSize {
+		isLargeBlock = true
+		blockIndex = int(offset / largeBlockLength)
+		innerBlockOffset = offset % largeBlockLength
+		return
+	}
+
+	isLargeBlock = false
+	offset -= nLargeBlockRows * largeRowSize
+	blockIndex = int(offset / smallBlockLength)
+	innerBlockOffset = offset % smallBlockLength
+	return
+}
+
+// locateDataBuggy is LocateData using the old buggy locateOffset.
+func locateDataBuggy(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64, size types.Size) []Interval {
+	blockIndex, isLargeBlock, nLargeBlockRows, innerBlockOffset := locateOffsetBuggy(largeBlockLength, smallBlockLength, shardDatSize, offset)
+
+	var intervals []Interval
+	for size > 0 {
+		blockRemaining := largeBlockLength - innerBlockOffset
+		if !isLargeBlock {
+			blockRemaining = smallBlockLength - innerBlockOffset
+		}
+		if blockRemaining <= 0 {
+			blockIndex, isLargeBlock = moveToNextBlock(blockIndex, isLargeBlock, nLargeBlockRows)
+			innerBlockOffset = 0
+			continue
+		}
+		interval := Interval{
+			BlockIndex:          blockIndex,
+			InnerBlockOffset:    innerBlockOffset,
+			IsLargeBlock:        isLargeBlock,
+			LargeBlockRowsCount: int(nLargeBlockRows),
+		}
+		if int64(size) <= blockRemaining {
+			interval.Size = size
+			intervals = append(intervals, interval)
+			return intervals
+		}
+		interval.Size = types.Size(blockRemaining)
+		intervals = append(intervals, interval)
+		size -= interval.Size
+		blockIndex, isLargeBlock = moveToNextBlock(blockIndex, isLargeBlock, nLargeBlockRows)
+		innerBlockOffset = 0
+	}
+	return intervals
+}
+
+// TestEcOffByOneBug_Issue8947 directly demonstrates the off-by-one bug.
+//
+// It creates a .dat file whose size is an exact multiple of (largeBlockSize * DataShards),
+// EC-encodes it, then shows that:
+//   - The OLD buggy formula produces WRONG data (data corruption)
+//   - The FIXED formula produces CORRECT data
+func TestEcOffByOneBug_Issue8947(t *testing.T) {
+	const (
+		large = largeBlockSize // 10000
+		small = smallBlockSize // 100
+	)
+
+	// datSize is exactly 2 large rows — each shard gets exactly 2*largeBlockSize bytes.
+	// The encoder produces 2 large block rows and 0 small block rows.
+	datSize := int64(2 * large * DataShardsCount) // 200000
+
+	dir := t.TempDir()
+	baseFileName := fmt.Sprintf("%s/bug_%d", dir, datSize)
+
+	originalData := make([]byte, datSize)
+	_, err := rand.Read(originalData)
+	require.NoError(t, err)
+
+	err = os.WriteFile(baseFileName+".dat", originalData, 0644)
+	require.NoError(t, err)
+
+	ctx := NewDefaultECContext("", 0)
+	_, err = generateEcFiles(baseFileName, int(small), large, small, ctx)
+	require.NoError(t, err, "EC encoding")
+
+	ecFiles, err := openEcFiles(baseFileName, true, ctx)
+	require.NoError(t, err)
+	defer closeEcFiles(ecFiles)
+
+	// shardDatSize = datFileSize / DataShards = 2 * largeBlockSize
+	// This is an EXACT multiple of largeBlockSize.
+	shardDatSize := datSize / int64(ctx.DataShards) // = 2 * large = 20000
+
+	// The encoder used 2 large block rows, 0 small block rows.
+	// Correct: nLargeBlockRows = 20000 / 10000 = 2
+	// Buggy:   nLargeBlockRows = (20000 - 1) / 10000 = 1  ← OFF BY ONE
+	fixedRows := shardDatSize / large
+	buggyRows := (shardDatSize - 1) / large
+	assert.Equal(t, int64(2), fixedRows, "fixed formula should give 2 large block rows")
+	assert.Equal(t, int64(1), buggyRows, "buggy formula gives only 1 (the bug)")
+
+	// Test reading from the 2nd large block row (offsets 100000–199999).
+	// With the buggy formula (nLargeBlockRows=1), this region is incorrectly
+	// treated as small blocks, causing reads from the WRONG shard positions.
+	readSize := types.Size(small / 2)
+
+	// Pick an offset well into the 2nd large block row so that the buggy formula
+	// computes a different (shard, offset) than the correct formula.
+	// At the very start of the 2nd row, both formulas coincidentally hit the same
+	// shard position. But further in, the small-block vs large-block addressing diverges.
+	offset := int64(large*DataShardsCount) + large + 50 // 110050: in 2nd large row, shard 1
+
+	// --- Fixed formula: reads correct data ---
+	fixedIntervals := LocateData(large, small, shardDatSize, offset, readSize)
+	fixedData, err := assembleFromIntervals(ecFiles, fixedIntervals, large, small)
+	require.NoError(t, err, "fixed LocateData read")
+
+	expected := originalData[offset : offset+int64(readSize)]
+	assert.True(t, bytes.Equal(expected, fixedData),
+		"FIXED formula should read correct data from 2nd large block row")
+
+	// --- Buggy formula: reads WRONG data ---
+	buggyIntervals := locateDataBuggy(large, small, shardDatSize, offset, readSize)
+	buggyData, err := assembleFromIntervals(ecFiles, buggyIntervals, large, small)
+	// The buggy formula might read from wrong offsets (possibly out of bounds),
+	// so an error is also evidence of the bug.
+	if err != nil {
+		t.Logf("Buggy formula caused read error (expected): %v", err)
+	} else {
+		assert.False(t, bytes.Equal(expected, buggyData),
+			"BUGGY formula should return WRONG data from 2nd large block row (demonstrating the corruption)")
+		n := 8
+		if len(expected) < n {
+			n = len(expected)
+		}
+		t.Logf("Buggy formula returned wrong data: expected first bytes %x, got %x",
+			expected[:n], buggyData[:n])
+	}
+
+	// Verify the bug mechanism: buggy formula misclassifies the 2nd large row as small blocks
+	assert.True(t, fixedIntervals[0].IsLargeBlock,
+		"fixed: offset %d should be in large blocks", offset)
+	assert.False(t, buggyIntervals[0].IsLargeBlock,
+		"buggy: offset %d is incorrectly classified as small blocks (the bug)", offset)
+
+	t.Logf("Fixed: nLargeBlockRows=%d, interval=%+v", fixedRows, fixedIntervals[0])
+	t.Logf("Buggy: nLargeBlockRows=%d, interval=%+v", buggyRows, buggyIntervals[0])
+}
+
+// TestEcDecodeDatRoundTrip tests the full WriteDatFile decode path using the production
+// block sizes with a small .dat file that fits within the small block region.
+func TestEcDecodeDatRoundTrip(t *testing.T) {
+	// With production sizes, datFileSize must be < DataShardsCount * ErasureCodingLargeBlockSize (10GB)
+	// to avoid needing huge test files. We test small block decode only.
+	// Each shard gets datSize/DataShards bytes in small 1MB blocks.
+	datSizes := []int64{
+		1000, // tiny
+		int64(DataShardsCount) * ErasureCodingSmallBlockSize,     // exactly 1 small row (10MB)
+		int64(DataShardsCount)*ErasureCodingSmallBlockSize + 500, // 1 small row + partial
+	}
+
+	for _, datSize := range datSizes {
+		t.Run(fmt.Sprintf("size_%d", datSize), func(t *testing.T) {
+			testDecodeDat(t, datSize)
+		})
+	}
+}
+
+func testDecodeDat(t *testing.T, datSize int64) {
+	t.Helper()
+
+	dir := t.TempDir()
+	baseFileName := fmt.Sprintf("%s/dec_%d", dir, datSize)
+
+	// 1. Create .dat with random data
+	originalData := make([]byte, datSize)
+	_, err := rand.Read(originalData)
+	require.NoError(t, err)
+
+	err = os.WriteFile(baseFileName+".dat", originalData, 0644)
+	require.NoError(t, err)
+
+	ctx := NewDefaultECContext("", 0)
+
+	// 2. EC encode with PRODUCTION block sizes
+	_, err = generateEcFiles(baseFileName, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx)
+	require.NoError(t, err, "EC encoding")
+
+	// 3. Decode via WriteDatFile
+	decodedBase := baseFileName + "_decoded"
+	shardFileNames := make([]string, DataShardsCount)
+	for i := 0; i < DataShardsCount; i++ {
+		shardFileNames[i] = fmt.Sprintf("%s%s", baseFileName, ctx.ToExt(i))
+	}
+
+	err = WriteDatFile(decodedBase, datSize, datSize, shardFileNames, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize)
+	require.NoError(t, err, "WriteDatFile")
+
+	// The atomic publish must rename the temp file away, never leaving it behind.
+	_, statErr := os.Stat(decodedBase + ".dat.tmp")
+	require.True(t, os.IsNotExist(statErr), "WriteDatFile must not leave a .dat.tmp")
+
+	// 4. Verify decoded .dat matches original
+	decodedData, err := os.ReadFile(decodedBase + ".dat")
+	require.NoError(t, err)
+
+	assert.Equal(t, len(originalData), len(decodedData), "decoded .dat size mismatch")
+	if !bytes.Equal(originalData, decodedData) {
+		for i := 0; i < len(originalData) && i < len(decodedData); i++ {
+			if originalData[i] != decodedData[i] {
+				t.Fatalf("decoded .dat mismatch at byte %d (datSize=%d)", i, datSize)
+			}
+		}
+	}
+}
+
+// collectTestOffsets generates offsets to test, focusing on the large/small block boundary.
+func collectTestOffsets(datSize, readSize, boundaryOffset, large, small int64) []int64 {
+	offsets := []int64{0}
+
+	if datSize > readSize {
+		offsets = append(offsets, datSize/2)
+	}
+
+	// Near the large/small block boundary
+	if boundaryOffset > 0 && boundaryOffset < datSize {
+		for _, delta := range []int64{-large, -small, -1, 0, 1, small, large} {
+			off := boundaryOffset + delta
+			if off >= 0 && off+readSize <= datSize {
+				offsets = append(offsets, off)
+			}
+		}
+	}
+
+	// Near end of file
+	if datSize > readSize {
+		offsets = append(offsets, datSize-readSize)
+	}
+
+	return offsets
+}
+
+// assembleFromIntervals reads data from EC shard files according to the given intervals.
+func assembleFromIntervals(ecFiles []*os.File, intervals []Interval, large, small int64) ([]byte, error) {
+	var data []byte
+	for _, interval := range intervals {
+		shardId, shardOffset := interval.ToShardIdAndOffset(large, small)
+		chunk := make([]byte, interval.Size)
+		n, err := ecFiles[shardId].ReadAt(chunk, shardOffset)
+		if err != nil {
+			return nil, fmt.Errorf("read shard %d offset %d size %d: %v", shardId, shardOffset, interval.Size, err)
+		}
+		if n != int(interval.Size) {
+			return nil, fmt.Errorf("short read from shard %d: got %d, want %d", shardId, n, interval.Size)
+		}
+		data = append(data, chunk...)
+	}
+	return data, nil
+}
+
+// TestWriteDatFileAfterTailDeletion decodes after deletions moved the live
+// extent below the large-block row boundary. The shard block layout is fixed
+// by the encode-time .dat size, so de-striping must not derive it from the
+// shrunk live extent.
+func TestWriteDatFileAfterTailDeletion(t *testing.T) {
+	const (
+		large = int64(largeBlockSize) // 10000
+		small = int64(smallBlockSize) // 100
+	)
+	largeRowSize := large * DataShardsCount
+	smallRowSize := small * DataShardsCount
+
+	// one full large-block row plus a small-block tail
+	datSize := largeRowSize + 2*smallRowSize + 530
+
+	dir := t.TempDir()
+	baseFileName := fmt.Sprintf("%s/tail_del", dir)
+
+	originalData := make([]byte, datSize)
+	_, err := rand.Read(originalData)
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(baseFileName+".dat", originalData, 0644))
+
+	ctx := NewDefaultECContext("", 0)
+	_, err = generateEcFiles(baseFileName, 50, large, small, ctx)
+	require.NoError(t, err, "EC encoding")
+
+	shardFileNames := make([]string, DataShardsCount)
+	for i := 0; i < DataShardsCount; i++ {
+		shardFileNames[i] = baseFileName + ctx.ToExt(i)
+	}
+	shardFileInfo, err := os.Stat(shardFileNames[0])
+	require.NoError(t, err)
+	paddedSize := int64(DataShardsCount) * shardFileInfo.Size()
+
+	decodedBase := baseFileName + "_decoded"
+	decode := func(liveSize, encodedSize int64) []byte {
+		require.NoError(t, writeDatFile(decodedBase, liveSize, encodedSize, shardFileNames, large, small))
+		decoded, readErr := os.ReadFile(decodedBase + ".dat")
+		require.NoError(t, readErr)
+		require.NoError(t, os.Remove(decodedBase+".dat"))
+		return decoded
+	}
+
+	liveSizes := []int64{
+		large - 1,              // within the first large block
+		large + 42,             // partial second large block
+		largeRowSize / 2,       // mid large row
+		largeRowSize,           // exactly the large row
+		largeRowSize + 5*small, // into the small-block tail
+		datSize,                // nothing deleted
+	}
+	for _, liveSize := range liveSizes {
+		assert.Equal(t, originalData[:liveSize], decode(liveSize, datSize), "live size %d, encode-time layout", liveSize)
+		assert.Equal(t, originalData[:liveSize], decode(liveSize, paddedSize), "live size %d, padded layout", liveSize)
+	}
+
+	// deriving the layout from the shrunk live extent reorders the data
+	assert.NotEqual(t, originalData[:largeRowSize/2], decode(largeRowSize/2, largeRowSize/2))
+
+	// the live extent can never exceed the encode-time size
+	require.Error(t, writeDatFile(decodedBase, datSize+1, datSize, shardFileNames, large, small))
+}
+
+// TestWriteDatFileFallbackLayout covers decoding when .vif does not record the
+// encode-time size: the layout is inferred from the shard size, except when
+// that is an exact large-block multiple and the live extent reaches the
+// ambiguous region.
+func TestWriteDatFileFallbackLayout(t *testing.T) {
+	const (
+		large = int64(largeBlockSize) // 10000
+		small = int64(smallBlockSize) // 100
+	)
+	largeRowSize := large * DataShardsCount
+
+	dir := t.TempDir()
+
+	encode := func(name string, datSize int64) (string, []string, []byte) {
+		baseFileName := fmt.Sprintf("%s/%s", dir, name)
+		originalData := make([]byte, datSize)
+		_, err := rand.Read(originalData)
+		require.NoError(t, err)
+		require.NoError(t, os.WriteFile(baseFileName+".dat", originalData, 0644))
+		ctx := NewDefaultECContext("", 0)
+		_, err = generateEcFiles(baseFileName, 50, large, small, ctx)
+		require.NoError(t, err, "EC encoding")
+		shardFileNames := make([]string, DataShardsCount)
+		for i := 0; i < DataShardsCount; i++ {
+			shardFileNames[i] = baseFileName + ctx.ToExt(i)
+		}
+		return baseFileName, shardFileNames, originalData
+	}
+
+	// a small-block tail that is not a large-block multiple is unambiguous
+	base, shards, original := encode("plain", largeRowSize+2530)
+	liveSize := largeRowSize / 2
+	require.NoError(t, writeDatFile(base+"_d", liveSize, 0, shards, large, small))
+	decoded, err := os.ReadFile(base + "_d.dat")
+	require.NoError(t, err)
+	assert.Equal(t, original[:liveSize], decoded)
+
+	// datSize just under one large row: the full small-block region makes each
+	// shard exactly largeBlockSize, indistinguishable from one large row
+	base, shards, _ = encode("ambiguous1", largeRowSize-1)
+	err = writeDatFile(base+"_d", largeRowSize/2, 0, shards, large, small)
+	require.ErrorContains(t, err, "does not identify the block layout")
+
+	// two-row equivalent: decoding within the agreed prefix still works
+	base, shards, original = encode("ambiguous2", 2*largeRowSize-1)
+	require.NoError(t, writeDatFile(base+"_d", largeRowSize, 0, shards, large, small))
+	decoded, err = os.ReadFile(base + "_d.dat")
+	require.NoError(t, err)
+	assert.Equal(t, original[:largeRowSize], decoded)
+	err = writeDatFile(base+"_d2", largeRowSize+1, 0, shards, large, small)
+	require.ErrorContains(t, err, "does not identify the block layout")
+}

weed/storage/erasure_coding/ec_scrub_local_test.go

@@ -0,0 +1,67 @@
+package erasure_coding_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// setupScrubLocalVolume seeds one live .ecx entry and a shard 0 whose bytes form
+// the needle's on-disk header, then returns a loaded EcVolume with shard 0 added.
+func setupScrubLocalVolume(t *testing.T, shard0 []byte) *erasure_coding.EcVolume {
+	t.Helper()
+	dir := t.TempDir()
+	collection := "test"
+	vid := needle.VolumeId(1)
+	base := filepath.Join(dir, collection+"_1")
+
+	if err := os.WriteFile(base+".ecx", makeNeedleMapEntry(types.NeedleId(1), types.ToOffset(0), types.Size(100)), 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+	if err := os.WriteFile(base+".ecj", []byte{}, 0644); err != nil {
+		t.Fatalf("write ecj: %v", err)
+	}
+	if err := os.WriteFile(base+".vif", []byte{}, 0644); err != nil {
+		t.Fatalf("write vif: %v", err)
+	}
+	if err := os.WriteFile(base+".ec00", shard0, 0644); err != nil {
+		t.Fatalf("write ec00: %v", err)
+	}
+
+	ecv, err := erasure_coding.NewEcVolume("hdd", dir, dir, collection, vid)
+	if err != nil {
+		t.Fatalf("NewEcVolume: %v", err)
+	}
+	shard, err := erasure_coding.NewEcVolumeShard("hdd", dir, collection, vid, 0)
+	if err != nil {
+		t.Fatalf("NewEcVolumeShard: %v", err)
+	}
+	if _, err := ecv.AddEcVolumeShard(shard); err != nil {
+		t.Fatalf("AddEcVolumeShard: %v", err)
+	}
+	return ecv
+}
+
+// A needle the .ecx still reports as live (size 100) but whose reassembled
+// on-disk header carries size 0 is a delete-state disagreement, not corruption.
+func TestScrubLocal_SuppressesDeleteStateDisagreement(t *testing.T) {
+	ecv := setupScrubLocalVolume(t, make([]byte, 256)) // all-zero header => size 0
+	if _, _, errs := ecv.ScrubLocal(); len(errs) != 0 {
+		t.Fatalf("delete-state disagreement must be suppressed, got %v", errs)
+	}
+}
+
+// A non-zero on-disk header size that disagrees with the index is genuine
+// corruption and must still be reported.
+func TestScrubLocal_ReportsGenuineSizeCorruption(t *testing.T) {
+	shard0 := make([]byte, 256)
+	shard0[15] = 50 // header size field (big-endian uint32) = 50, != index 100 and != 0
+	ecv := setupScrubLocalVolume(t, shard0)
+	if _, _, errs := ecv.ScrubLocal(); len(errs) == 0 {
+		t.Fatal("a non-zero size mismatch is genuine corruption and must be reported")
+	}
+}

weed/storage/erasure_coding/ec_shard.go

@@ -0,0 +1,168 @@
+package erasure_coding
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+type ShardId uint8
+
+// Converts a slice of uint32s to ShardId.
+func Uint32ToShardIds(ids []uint32) []ShardId {
+	res := make([]ShardId, len(ids))
+	for i, id := range ids {
+		res[i] = ShardId(id)
+	}
+	return res
+}
+
+// Converts a slice of ShardIds to uint32
+func ShardIdsToUint32(ids []ShardId) []uint32 {
+	res := make([]uint32, len(ids))
+	for i, id := range ids {
+		res[i] = uint32(id)
+	}
+	return res
+}
+
+// Returns a slice of all possible ShardIds.
+func AllShardIds() []ShardId {
+	res := make([]ShardId, TotalShardsCount)
+	for i := range res {
+		res[i] = ShardId(i)
+	}
+	return res
+}
+
+// Compares a pair of EcShardInfo protos for sorting.
+func CmpEcShardInfo(a, b *volume_server_pb.EcShardInfo) int {
+	if a.VolumeId < b.VolumeId {
+		return -1
+	}
+	if a.VolumeId > b.VolumeId {
+		return 1
+	}
+	if a.ShardId < b.ShardId {
+		return -1
+	}
+	if a.ShardId > b.ShardId {
+		return 1
+	}
+	return 0
+}
+
+type EcVolumeShard struct {
+	VolumeId    needle.VolumeId
+	ShardId     ShardId
+	Collection  string
+	dir         string
+	ecdFile     *os.File
+	ecdFileSize int64
+	DiskType    types.DiskType
+}
+
+func NewEcVolumeShard(diskType types.DiskType, dirname string, collection string, id needle.VolumeId, shardId ShardId) (v *EcVolumeShard, e error) {
+
+	v = &EcVolumeShard{dir: dirname, Collection: collection, VolumeId: id, ShardId: shardId, DiskType: diskType}
+
+	baseFileName := v.FileName()
+
+	// open ecd file
+	if v.ecdFile, e = backend.OpenVolumeFile(baseFileName+ToExt(int(shardId)), os.O_RDONLY); e != nil {
+		if e == os.ErrNotExist || strings.Contains(e.Error(), "no such file or directory") {
+			return nil, os.ErrNotExist
+		}
+		return nil, fmt.Errorf("cannot read ec volume shard %s%s: %v", baseFileName, ToExt(int(shardId)), e)
+	}
+	ecdFi, statErr := v.ecdFile.Stat()
+	if statErr != nil {
+		_ = v.ecdFile.Close()
+		return nil, fmt.Errorf("can not stat ec volume shard %s%s: %v", baseFileName, ToExt(int(shardId)), statErr)
+	}
+	v.ecdFileSize = ecdFi.Size()
+
+	v.Mount()
+
+	return
+}
+
+func (shard *EcVolumeShard) Mount() {
+	stats.VolumeServerVolumeGauge.WithLabelValues(shard.Collection, "ec_shards").Inc()
+}
+
+func (shard *EcVolumeShard) Unmount() {
+	stats.VolumeServerVolumeGauge.WithLabelValues(shard.Collection, "ec_shards").Dec()
+}
+
+func (shard *EcVolumeShard) Size() int64 {
+	return shard.ecdFileSize
+}
+
+func (shard *EcVolumeShard) String() string {
+	return fmt.Sprintf("ec shard %v:%v, dir:%s, Collection:%s", shard.VolumeId, shard.ShardId, shard.dir, shard.Collection)
+}
+
+func (shard *EcVolumeShard) FileName() (fileName string) {
+	return EcShardFileName(shard.Collection, shard.dir, int(shard.VolumeId))
+}
+
+func EcShardFileName(collection string, dir string, id int) (fileName string) {
+	idString := strconv.Itoa(id)
+	if collection == "" {
+		fileName = path.Join(dir, idString)
+	} else {
+		fileName = path.Join(dir, collection+"_"+idString)
+	}
+	return
+}
+
+func EcShardBaseFileName(collection string, id int) (baseFileName string) {
+	baseFileName = strconv.Itoa(id)
+	if collection != "" {
+		baseFileName = collection + "_" + baseFileName
+	}
+	return
+}
+
+func (shard *EcVolumeShard) Close() {
+	// Close the fd but do NOT nil it: a reader resolves the shard under the
+	// ecVolumesLock then calls ReadAt after the lock is released, so a
+	// concurrent eviction that nils the field would race that read and panic.
+	// Leaving the field set means ReadAt hits a closed fd and returns a clean
+	// error (the caller recovers from parity) with no data race on the field.
+	if shard.ecdFile != nil {
+		_ = shard.ecdFile.Close()
+	}
+}
+
+func (shard *EcVolumeShard) Destroy() {
+	shard.Unmount()
+	os.Remove(shard.FileName() + ToExt(int(shard.ShardId)))
+}
+
+func (shard *EcVolumeShard) ReadAt(buf []byte, offset int64) (int, error) {
+	n, err := shard.ecdFile.ReadAt(buf, offset)
+	if err == io.EOF && n == len(buf) {
+		err = nil
+	}
+	return n, err
+}
+
+func (shard *EcVolumeShard) ToEcShardInfo() *volume_server_pb.EcShardInfo {
+	return &volume_server_pb.EcShardInfo{
+		ShardId:    uint32(shard.ShardId),
+		Size:       int64(shard.Size()),
+		Collection: shard.Collection,
+		VolumeId:   uint32(shard.VolumeId),
+	}
+}

weed/storage/erasure_coding/ec_shard_info.go

@@ -0,0 +1,23 @@
+package erasure_coding
+
+// ShardSize represents the size of a shard in bytes
+type ShardSize int64
+
+// ShardInfo holds information about a single shard
+type ShardInfo struct {
+	Id   ShardId
+	Size ShardSize
+}
+
+// NewShardInfo creates a new ShardInfo with the given ID and size
+func NewShardInfo(id ShardId, size ShardSize) ShardInfo {
+	return ShardInfo{
+		Id:   id,
+		Size: size,
+	}
+}
+
+// IsValid checks if the shard info has a valid ID
+func (si ShardInfo) IsValid() bool {
+	return si.Id < MaxShardCount
+}

weed/storage/erasure_coding/ec_shards_info.go

@@ -0,0 +1,449 @@
+package erasure_coding
+
+import (
+	"fmt"
+	"iter"
+	"math/bits"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/dustin/go-humanize"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+// ShardBits is a bitmap representing which shards are present (bit 0 = shard 0, etc.)
+type ShardBits uint32
+
+// Has checks if a shard ID is present in the bitmap
+func (sb ShardBits) Has(id ShardId) bool {
+	return id < MaxShardCount && sb&(1<<id) != 0
+}
+
+// Set sets a shard ID in the bitmap
+func (sb ShardBits) Set(id ShardId) ShardBits {
+	if id >= MaxShardCount {
+		return sb
+	}
+	return sb | (1 << id)
+}
+
+// Clear clears a shard ID from the bitmap
+func (sb ShardBits) Clear(id ShardId) ShardBits {
+	if id >= MaxShardCount {
+		return sb
+	}
+	return sb &^ (1 << id)
+}
+
+// Count returns the number of set bits using popcount
+func (sb ShardBits) Count() int {
+	return bits.OnesCount32(uint32(sb))
+}
+
+// All iterates the shard ids present in the bitmap, in ascending order. It walks
+// only the set bits (trailing-zero scan), so cost scales with the number of
+// shards present rather than the full id range. Prefer this over scanning
+// 0..MaxShardCount and calling Has on each id.
+func (sb ShardBits) All() iter.Seq[ShardId] {
+	return func(yield func(ShardId) bool) {
+		for b := uint32(sb); b != 0; b &= b - 1 {
+			if !yield(ShardId(bits.TrailingZeros32(b))) {
+				return
+			}
+		}
+	}
+}
+
+// ShardsInfo encapsulates information for EC shards with memory-efficient storage
+type ShardsInfo struct {
+	mu        sync.RWMutex
+	shards    []ShardInfo // Sorted by Id
+	shardBits ShardBits
+}
+
+func NewShardsInfo() *ShardsInfo {
+	return &ShardsInfo{
+		shards: make([]ShardInfo, 0, TotalShardsCount),
+	}
+}
+
+// Initializes a ShardsInfo from a VolumeEcShardInformationMessage proto.
+func ShardsInfoFromVolumeEcShardInformationMessage(vi *master_pb.VolumeEcShardInformationMessage) *ShardsInfo {
+	res := NewShardsInfo()
+	if vi == nil {
+		return res
+	}
+
+	var id ShardId
+	var j int
+	// Build shards directly to avoid locking in Set() since res is not yet shared
+	newShards := make([]ShardInfo, 0, 8)
+	for bitmap := vi.EcIndexBits; bitmap != 0; bitmap >>= 1 {
+		if bitmap&1 != 0 {
+			var size ShardSize
+			if j < len(vi.ShardSizes) {
+				size = ShardSize(vi.ShardSizes[j])
+			}
+			j++
+			newShards = append(newShards, NewShardInfo(id, size))
+		}
+		id++
+	}
+	res.shards = newShards
+	res.shardBits = ShardBits(vi.EcIndexBits)
+
+	return res
+}
+
+// Returns a count of shards from a VolumeEcShardInformationMessage proto.
+func GetShardCount(vi *master_pb.VolumeEcShardInformationMessage) int {
+	if vi == nil {
+		return 0
+	}
+	return ShardBits(vi.EcIndexBits).Count()
+}
+
+// VolumeSlots converts an EC shard count to the number of volume slots those
+// shards occupy: every DataShardsCount shards hold one volume's worth of data,
+// rounded up.
+func VolumeSlots(ecShardCount int64) int64 {
+	return (ecShardCount + DataShardsCount - 1) / DataShardsCount
+}
+
+// EcShardsTotalSize returns the sum of all shard sizes (data + parity) in
+// the message. Walks vi.ShardSizes directly rather than materializing a
+// ShardsInfo, which is significantly cheaper for callers that only need the
+// aggregate size.
+func EcShardsTotalSize(vi *master_pb.VolumeEcShardInformationMessage) int64 {
+	if vi == nil {
+		return 0
+	}
+	var total int64
+	for _, s := range vi.ShardSizes {
+		total += s
+	}
+	return total
+}
+
+// EcShardsVolumeDataShards returns the number of data shards for the EC volume
+// described by vi. Open-source SeaweedFS always uses the fixed 10+4 layout, so
+// this returns DataShardsCount. It is defined as a per-volume accessor (rather
+// than referencing DataShardsCount directly at every call site) so callers such
+// as ec.check.replication stay correct when a build derives the ratio per volume instead
+// of from the global constant.
+func EcShardsVolumeDataShards(vi *master_pb.VolumeEcShardInformationMessage) int {
+	return DataShardsCount
+}
+
+// EcShardsVolumeParityShards returns the number of parity shards for the EC
+// volume described by vi. As with EcShardsVolumeDataShards, open-source
+// SeaweedFS uses the fixed 10+4 layout, so this returns ParityShardsCount.
+func EcShardsVolumeParityShards(vi *master_pb.VolumeEcShardInformationMessage) int {
+	return ParityShardsCount
+}
+
+// EcShardsDataSize returns the sum of sizes for data shards only (parity
+// shards excluded). Data shards are those with id < dataShards; all higher
+// shard ids are treated as parity. Passing dataShards <= 0 falls back to
+// the upstream default of DataShardsCount (10), which is correct for the
+// fixed 10+4 layout. Forks with per-volume ratio metadata (e.g. the
+// data_shards field carried on an extended VolumeEcShardInformationMessage)
+// should pass the per-volume value so logical sizes remain accurate under
+// custom EC policies like 6+3 or 16+6.
+func EcShardsDataSize(vi *master_pb.VolumeEcShardInformationMessage, dataShards int) int64 {
+	if vi == nil {
+		return 0
+	}
+	if dataShards <= 0 {
+		dataShards = DataShardsCount
+	}
+	var total int64
+	var id ShardId
+	var j int
+	for bitmap := vi.EcIndexBits; bitmap != 0; bitmap >>= 1 {
+		if bitmap&1 != 0 {
+			if int(id) < dataShards && j < len(vi.ShardSizes) {
+				total += vi.ShardSizes[j]
+			}
+			j++
+		}
+		id++
+	}
+	return total
+}
+
+// Returns a string representation for a ShardsInfo.
+func (sp *ShardsInfo) String() string {
+	sp.mu.RLock()
+	defer sp.mu.RUnlock()
+	var sb strings.Builder
+	for i, s := range sp.shards {
+		if i > 0 {
+			sb.WriteString(" ")
+		}
+		fmt.Fprintf(&sb, "%d:%s", s.Id, humanize.Bytes(uint64(s.Size)))
+	}
+	return sb.String()
+}
+
+// AsSlice converts a ShardsInfo to a slice of ShardInfo structs, ordered by shard ID.
+func (si *ShardsInfo) AsSlice() []ShardInfo {
+	si.mu.RLock()
+	defer si.mu.RUnlock()
+	res := make([]ShardInfo, len(si.shards))
+	copy(res, si.shards)
+	return res
+}
+
+// Count returns the number of EC shards using popcount on the bitmap.
+func (si *ShardsInfo) Count() int {
+	si.mu.RLock()
+	defer si.mu.RUnlock()
+	return si.shardBits.Count()
+}
+
+// Has verifies if a shard ID is present using bitmap check.
+func (si *ShardsInfo) Has(id ShardId) bool {
+	si.mu.RLock()
+	defer si.mu.RUnlock()
+	return si.shardBits.Has(id)
+}
+
+// Ids returns a list of shard IDs, in ascending order.
+func (si *ShardsInfo) Ids() []ShardId {
+	si.mu.RLock()
+	defer si.mu.RUnlock()
+	ids := make([]ShardId, len(si.shards))
+	for i, s := range si.shards {
+		ids[i] = s.Id
+	}
+	return ids
+}
+
+// IdsInt returns a list of shards ID as int, in ascending order.
+func (si *ShardsInfo) IdsInt() []int {
+	ids := si.Ids()
+	res := make([]int, len(ids))
+	for i, id := range ids {
+		res[i] = int(id)
+	}
+	return res
+}
+
+// IdsUint32 returns a list of shards ID as uint32, in ascending order.
+func (si *ShardsInfo) IdsUint32() []uint32 {
+	return ShardIdsToUint32(si.Ids())
+}
+
+// Set sets or updates a shard's information.
+func (si *ShardsInfo) Set(shard ShardInfo) {
+	if shard.Id >= MaxShardCount {
+		return
+	}
+	si.mu.Lock()
+	defer si.mu.Unlock()
+
+	// Check if already exists
+	if si.shardBits.Has(shard.Id) {
+		// Find and update
+		idx := si.findIndex(shard.Id)
+		if idx >= 0 {
+			si.shards[idx] = shard
+		}
+		return
+	}
+
+	// Add new shard
+	si.shardBits = si.shardBits.Set(shard.Id)
+
+	// Find insertion point to keep sorted
+	idx := sort.Search(len(si.shards), func(i int) bool {
+		return si.shards[i].Id > shard.Id
+	})
+
+	// Insert at idx
+	si.shards = append(si.shards, ShardInfo{})
+	copy(si.shards[idx+1:], si.shards[idx:])
+	si.shards[idx] = shard
+}
+
+// Delete deletes a shard by ID.
+func (si *ShardsInfo) Delete(id ShardId) {
+	if id >= MaxShardCount {
+		return
+	}
+	si.mu.Lock()
+	defer si.mu.Unlock()
+
+	if !si.shardBits.Has(id) {
+		return // Not present
+	}
+
+	si.shardBits = si.shardBits.Clear(id)
+
+	// Find and remove from slice
+	idx := si.findIndex(id)
+	if idx >= 0 {
+		si.shards = append(si.shards[:idx], si.shards[idx+1:]...)
+	}
+}
+
+// Bitmap returns a bitmap for all existing shard IDs.
+func (si *ShardsInfo) Bitmap() uint32 {
+	si.mu.RLock()
+	defer si.mu.RUnlock()
+	return uint32(si.shardBits)
+}
+
+// Size returns the size of a given shard ID, if present.
+func (si *ShardsInfo) Size(id ShardId) ShardSize {
+	if id >= MaxShardCount {
+		return 0
+	}
+	si.mu.RLock()
+	defer si.mu.RUnlock()
+
+	if !si.shardBits.Has(id) {
+		return 0
+	}
+
+	idx := si.findIndex(id)
+	if idx >= 0 {
+		return si.shards[idx].Size
+	}
+	return 0
+}
+
+// TotalSize returns the size for all shards.
+func (si *ShardsInfo) TotalSize() ShardSize {
+	si.mu.RLock()
+	defer si.mu.RUnlock()
+	var total ShardSize
+	for _, s := range si.shards {
+		total += s.Size
+	}
+	return total
+}
+
+// Sizes returns a compact slice of present shard sizes, from first to last.
+func (si *ShardsInfo) Sizes() []ShardSize {
+	si.mu.RLock()
+	defer si.mu.RUnlock()
+
+	res := make([]ShardSize, len(si.shards))
+	for i, s := range si.shards {
+		res[i] = s.Size
+	}
+	return res
+}
+
+// SizesInt64 returns a compact slice of present shard sizes, from first to last, as int64.
+func (si *ShardsInfo) SizesInt64() []int64 {
+	sizes := si.Sizes()
+	res := make([]int64, len(sizes))
+	for i, s := range sizes {
+		res[i] = int64(s)
+	}
+	return res
+}
+
+// Copy creates a copy of a ShardInfo.
+func (si *ShardsInfo) Copy() *ShardsInfo {
+	si.mu.RLock()
+	defer si.mu.RUnlock()
+
+	newShards := make([]ShardInfo, len(si.shards))
+	copy(newShards, si.shards)
+
+	return &ShardsInfo{
+		shards:    newShards,
+		shardBits: si.shardBits,
+	}
+}
+
+// DeleteParityShards removes parity shards (those with id >= dataShards) from
+// a ShardInfo. dataShards is the volume's data-shard count; passing <= 0 falls
+// back to DataShardsCount (the fixed 10+4 layout). The upper bound is
+// MaxShardCount, not TotalShardsCount, so a custom ratio's high parity ids
+// (e.g. 16+6 reaches id 21) are cleared too; Delete no-ops on absent ids.
+func (si *ShardsInfo) DeleteParityShards(dataShards int) {
+	if dataShards <= 0 {
+		dataShards = DataShardsCount
+	}
+	if dataShards >= MaxShardCount {
+		return // every id is a data shard; nothing to remove
+	}
+	si.mu.Lock()
+	defer si.mu.Unlock()
+
+	// Parity ids are >= dataShards. shards stays sorted by Id and shardBits is
+	// a bitmap, so clear them in one locked pass instead of a per-id Delete:
+	// mask off the high bits, then truncate the sorted slice at the first
+	// parity id via binary search.
+	si.shardBits &= ShardBits((uint32(1) << uint(dataShards)) - 1)
+	idx := sort.Search(len(si.shards), func(i int) bool {
+		return si.shards[i].Id >= ShardId(dataShards)
+	})
+	si.shards = si.shards[:idx]
+}
+
+// MinusParityShards creates a ShardInfo copy with parity shards removed for the
+// given data-shard count (<= 0 falls back to DataShardsCount).
+func (si *ShardsInfo) MinusParityShards(dataShards int) *ShardsInfo {
+	result := si.Copy()
+	result.DeleteParityShards(dataShards)
+	return result
+}
+
+// Add merges all shards from another ShardInfo into this one.
+func (si *ShardsInfo) Add(other *ShardsInfo) {
+	other.mu.RLock()
+	// Copy shards to avoid holding lock on 'other' while calling si.Set, which could deadlock.
+	shardsToAdd := make([]ShardInfo, len(other.shards))
+	copy(shardsToAdd, other.shards)
+	other.mu.RUnlock()
+
+	for _, s := range shardsToAdd {
+		si.Set(s)
+	}
+}
+
+// Subtract removes all shards present on another ShardInfo.
+func (si *ShardsInfo) Subtract(other *ShardsInfo) {
+	other.mu.RLock()
+	// Copy shards to avoid holding lock on 'other' while calling si.Delete, which could deadlock.
+	shardsToRemove := make([]ShardInfo, len(other.shards))
+	copy(shardsToRemove, other.shards)
+	other.mu.RUnlock()
+
+	for _, s := range shardsToRemove {
+		si.Delete(s.Id)
+	}
+}
+
+// Plus returns a new ShardInfo consisting of (this + other).
+func (si *ShardsInfo) Plus(other *ShardsInfo) *ShardsInfo {
+	result := si.Copy()
+	result.Add(other)
+	return result
+}
+
+// Minus returns a new ShardInfo consisting of (this - other).
+func (si *ShardsInfo) Minus(other *ShardsInfo) *ShardsInfo {
+	result := si.Copy()
+	result.Subtract(other)
+	return result
+}
+
+// findIndex finds the index of a shard by ID using binary search.
+// Must be called with lock held. Returns -1 if not found.
+func (si *ShardsInfo) findIndex(id ShardId) int {
+	idx := sort.Search(len(si.shards), func(i int) bool {
+		return si.shards[i].Id >= id
+	})
+	if idx < len(si.shards) && si.shards[idx].Id == id {
+		return idx
+	}
+	return -1
+}

weed/storage/erasure_coding/ec_shards_info_test.go

@@ -0,0 +1,532 @@
+package erasure_coding
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func TestShardsInfo_SetAndGet(t *testing.T) {
+	si := NewShardsInfo()
+
+	// Test setting shards
+	si.Set(ShardInfo{Id: 0, Size: 1000})
+	si.Set(ShardInfo{Id: 5, Size: 2000})
+	si.Set(ShardInfo{Id: 13, Size: 3000})
+
+	// Verify Has
+	if !si.Has(0) {
+		t.Error("Expected shard 0 to exist")
+	}
+	if !si.Has(5) {
+		t.Error("Expected shard 5 to exist")
+	}
+	if !si.Has(13) {
+		t.Error("Expected shard 13 to exist")
+	}
+	if si.Has(1) {
+		t.Error("Expected shard 1 to not exist")
+	}
+
+	// Verify Size
+	if got := si.Size(0); got != 1000 {
+		t.Errorf("Expected size 1000, got %d", got)
+	}
+	if got := si.Size(5); got != 2000 {
+		t.Errorf("Expected size 2000, got %d", got)
+	}
+	if got := si.Size(13); got != 3000 {
+		t.Errorf("Expected size 3000, got %d", got)
+	}
+
+	// Verify Count
+	if got := si.Count(); got != 3 {
+		t.Errorf("Expected count 3, got %d", got)
+	}
+
+	// Verify Bitmap
+	expectedBitmap := uint32((1 << 0) | (1 << 5) | (1 << 13))
+	if got := si.Bitmap(); got != expectedBitmap {
+		t.Errorf("Expected bitmap %b, got %b", expectedBitmap, got)
+	}
+}
+
+func TestShardsInfo_SortedOrder(t *testing.T) {
+	si := NewShardsInfo()
+
+	// Add shards in non-sequential order
+	si.Set(ShardInfo{Id: 10, Size: 1000})
+	si.Set(ShardInfo{Id: 2, Size: 2000})
+	si.Set(ShardInfo{Id: 7, Size: 3000})
+	si.Set(ShardInfo{Id: 0, Size: 4000})
+
+	// Verify Ids returns sorted order
+	ids := si.Ids()
+	expected := []ShardId{0, 2, 7, 10}
+	if len(ids) != len(expected) {
+		t.Fatalf("Expected %d ids, got %d", len(expected), len(ids))
+	}
+	for i, id := range ids {
+		if id != expected[i] {
+			t.Errorf("Expected id[%d]=%d, got %d", i, expected[i], id)
+		}
+	}
+}
+
+func TestShardsInfo_Delete(t *testing.T) {
+	si := NewShardsInfo()
+
+	si.Set(ShardInfo{Id: 0, Size: 1000})
+	si.Set(ShardInfo{Id: 5, Size: 2000})
+	si.Set(ShardInfo{Id: 10, Size: 3000})
+
+	// Delete middle shard
+	si.Delete(5)
+
+	if si.Has(5) {
+		t.Error("Expected shard 5 to be deleted")
+	}
+	if !si.Has(0) || !si.Has(10) {
+		t.Error("Expected other shards to remain")
+	}
+	if got := si.Count(); got != 2 {
+		t.Errorf("Expected count 2, got %d", got)
+	}
+
+	// Verify slice is still sorted
+	ids := si.Ids()
+	if len(ids) != 2 || ids[0] != 0 || ids[1] != 10 {
+		t.Errorf("Expected ids [0, 10], got %v", ids)
+	}
+}
+
+func TestShardsInfo_Update(t *testing.T) {
+	si := NewShardsInfo()
+
+	si.Set(ShardInfo{Id: 5, Size: 1000})
+
+	// Update existing shard
+	si.Set(ShardInfo{Id: 5, Size: 2000})
+
+	if got := si.Size(5); got != 2000 {
+		t.Errorf("Expected updated size 2000, got %d", got)
+	}
+	if got := si.Count(); got != 1 {
+		t.Errorf("Expected count to remain 1, got %d", got)
+	}
+}
+
+func TestShardsInfo_TotalSize(t *testing.T) {
+	si := NewShardsInfo()
+
+	si.Set(ShardInfo{Id: 0, Size: 1000})
+	si.Set(ShardInfo{Id: 5, Size: 2000})
+	si.Set(ShardInfo{Id: 10, Size: 3000})
+
+	expected := ShardSize(6000)
+	if got := si.TotalSize(); got != expected {
+		t.Errorf("Expected total size %d, got %d", expected, got)
+	}
+}
+
+func TestShardsInfo_Sizes(t *testing.T) {
+	si := NewShardsInfo()
+
+	si.Set(ShardInfo{Id: 2, Size: 100})
+	si.Set(ShardInfo{Id: 5, Size: 200})
+	si.Set(ShardInfo{Id: 8, Size: 300})
+
+	sizes := si.Sizes()
+	expected := []ShardSize{100, 200, 300}
+
+	if len(sizes) != len(expected) {
+		t.Fatalf("Expected %d sizes, got %d", len(expected), len(sizes))
+	}
+	for i, size := range sizes {
+		if size != expected[i] {
+			t.Errorf("Expected size[%d]=%d, got %d", i, expected[i], size)
+		}
+	}
+}
+
+func TestShardsInfo_Copy(t *testing.T) {
+	si := NewShardsInfo()
+	si.Set(ShardInfo{Id: 0, Size: 1000})
+	si.Set(ShardInfo{Id: 5, Size: 2000})
+
+	siCopy := si.Copy()
+
+	// Verify copy has same data
+	if !siCopy.Has(0) || !siCopy.Has(5) {
+		t.Error("Copy should have same shards")
+	}
+	if siCopy.Size(0) != 1000 || siCopy.Size(5) != 2000 {
+		t.Error("Copy should have same sizes")
+	}
+
+	// Modify original
+	si.Set(ShardInfo{Id: 10, Size: 3000})
+
+	// Verify copy is independent
+	if siCopy.Has(10) {
+		t.Error("Copy should be independent of original")
+	}
+}
+
+func TestShardsInfo_AddSubtract(t *testing.T) {
+	si1 := NewShardsInfo()
+	si1.Set(ShardInfo{Id: 0, Size: 1000})
+	si1.Set(ShardInfo{Id: 2, Size: 2000})
+
+	si2 := NewShardsInfo()
+	si2.Set(ShardInfo{Id: 2, Size: 9999}) // Different size
+	si2.Set(ShardInfo{Id: 5, Size: 3000})
+
+	// Test Add
+	si1.Add(si2)
+	if !si1.Has(0) || !si1.Has(2) || !si1.Has(5) {
+		t.Error("Add should merge shards")
+	}
+	if si1.Size(2) != 9999 {
+		t.Error("Add should update existing shard size")
+	}
+
+	// Test Subtract
+	si1.Subtract(si2)
+	if si1.Has(2) || si1.Has(5) {
+		t.Error("Subtract should remove shards")
+	}
+	if !si1.Has(0) {
+		t.Error("Subtract should keep non-matching shards")
+	}
+}
+
+func TestShardsInfo_PlusMinus(t *testing.T) {
+	si1 := NewShardsInfo()
+	si1.Set(ShardInfo{Id: 0, Size: 1000})
+	si1.Set(ShardInfo{Id: 2, Size: 2000})
+
+	si2 := NewShardsInfo()
+	si2.Set(ShardInfo{Id: 2, Size: 2000})
+	si2.Set(ShardInfo{Id: 5, Size: 3000})
+
+	// Test Plus
+	result := si1.Plus(si2)
+	if !result.Has(0) || !result.Has(2) || !result.Has(5) {
+		t.Error("Plus should merge into new instance")
+	}
+	if si1.Has(5) {
+		t.Error("Plus should not modify original")
+	}
+
+	// Test Minus
+	result = si1.Minus(si2)
+	if !result.Has(0) || result.Has(2) {
+		t.Error("Minus should subtract into new instance")
+	}
+	if !si1.Has(2) {
+		t.Error("Minus should not modify original")
+	}
+}
+
+func TestShardsInfo_DeleteParityShards(t *testing.T) {
+	si := NewShardsInfo()
+
+	// Add data shards (0-9)
+	for i := 0; i < DataShardsCount; i++ {
+		si.Set(ShardInfo{Id: ShardId(i), Size: ShardSize((i + 1) * 1000)})
+	}
+
+	// Add parity shards (10-13)
+	for i := DataShardsCount; i < TotalShardsCount; i++ {
+		si.Set(ShardInfo{Id: ShardId(i), Size: ShardSize((i + 1) * 1000)})
+	}
+
+	si.DeleteParityShards(DataShardsCount)
+
+	// Verify only data shards remain
+	for i := 0; i < DataShardsCount; i++ {
+		if !si.Has(ShardId(i)) {
+			t.Errorf("Expected data shard %d to remain", i)
+		}
+	}
+	for i := DataShardsCount; i < TotalShardsCount; i++ {
+		if si.Has(ShardId(i)) {
+			t.Errorf("Expected parity shard %d to be deleted", i)
+		}
+	}
+}
+
+func TestShardsInfo_FromVolumeEcShardInformationMessage(t *testing.T) {
+	tests := []struct {
+		name      string
+		msg       *master_pb.VolumeEcShardInformationMessage
+		wantBits  uint32
+		wantSizes []int64
+	}{
+		{
+			name:      "nil message",
+			msg:       nil,
+			wantBits:  0,
+			wantSizes: []int64{},
+		},
+		{
+			name: "single shard",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: 1 << 5,
+				ShardSizes:  []int64{12345},
+			},
+			wantBits:  1 << 5,
+			wantSizes: []int64{12345},
+		},
+		{
+			name: "multiple shards",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: (1 << 0) | (1 << 3) | (1 << 7),
+				ShardSizes:  []int64{1000, 2000, 3000},
+			},
+			wantBits:  (1 << 0) | (1 << 3) | (1 << 7),
+			wantSizes: []int64{1000, 2000, 3000},
+		},
+		{
+			name: "missing sizes",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: (1 << 0) | (1 << 3),
+				ShardSizes:  []int64{1000},
+			},
+			wantBits:  (1 << 0) | (1 << 3),
+			wantSizes: []int64{1000, 0},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			si := ShardsInfoFromVolumeEcShardInformationMessage(tt.msg)
+
+			if got := si.Bitmap(); got != tt.wantBits {
+				t.Errorf("Bitmap() = %b, want %b", got, tt.wantBits)
+			}
+
+			if got := si.SizesInt64(); len(got) != len(tt.wantSizes) {
+				t.Errorf("SizesInt64() length = %d, want %d", len(got), len(tt.wantSizes))
+			} else {
+				for i, size := range got {
+					if size != tt.wantSizes[i] {
+						t.Errorf("SizesInt64()[%d] = %d, want %d", i, size, tt.wantSizes[i])
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestEcShardsTotalSize(t *testing.T) {
+	tests := []struct {
+		name string
+		msg  *master_pb.VolumeEcShardInformationMessage
+		want int64
+	}{
+		{name: "nil message", msg: nil, want: 0},
+		{
+			name: "all shards present",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: (1 << 0) | (1 << 1) | (1 << 10),
+				ShardSizes:  []int64{1000, 2000, 3000},
+			},
+			want: 6000,
+		},
+		{
+			name: "only parity shards",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: (1 << 10) | (1 << 11),
+				ShardSizes:  []int64{500, 500},
+			},
+			want: 1000,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := EcShardsTotalSize(tt.msg); got != tt.want {
+				t.Errorf("EcShardsTotalSize() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestEcShardsDataSize(t *testing.T) {
+	tests := []struct {
+		name       string
+		msg        *master_pb.VolumeEcShardInformationMessage
+		dataShards int
+		want       int64
+	}{
+		{name: "nil message", msg: nil, dataShards: 0, want: 0},
+		{
+			name: "data shards only",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: (1 << 0) | (1 << 1) | (1 << 9),
+				ShardSizes:  []int64{1000, 2000, 500},
+			},
+			dataShards: 0, // default 10+4
+			want:       3500,
+		},
+		{
+			name: "mixed data and parity (default 10+4)",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				// shards 7, 8, 9 are data; 10..13 are parity.
+				EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13),
+				ShardSizes:  []int64{1000, 1000, 500, 2000, 2000, 2000, 2000},
+			},
+			dataShards: 0,
+			want:       2500,
+		},
+		{
+			name: "only parity shards (excluded)",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: (1 << 10) | (1 << 11),
+				ShardSizes:  []int64{500, 500},
+			},
+			dataShards: 0,
+			want:       0,
+		},
+		{
+			name: "missing sizes tolerated",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: (1 << 0) | (1 << 3),
+				ShardSizes:  []int64{1000},
+			},
+			dataShards: 0,
+			want:       1000,
+		},
+		{
+			name: "custom 6+3 ratio: shards 0..5 are data, 6..8 are parity",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				// shards 0..8 all present. With dataShards=6, only 0..5
+				// are data (6*1000 = 6000); 6..8 are parity and excluded.
+				// Under the default 10+4, 0..8 would all count as data and
+				// return 9000 instead — this test pins the custom-ratio
+				// path.
+				EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6) | (1 << 7) | (1 << 8),
+				ShardSizes:  []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000},
+			},
+			dataShards: 6,
+			want:       6000,
+		},
+		{
+			name: "custom 16+6 ratio: shard id 12 is data, not parity",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				// Shards 10..12 present. Under default 10+4 these would
+				// all be parity (size 0). Under 16+6, all three are data.
+				EcIndexBits: (1 << 10) | (1 << 11) | (1 << 12),
+				ShardSizes:  []int64{2000, 2000, 2000},
+			},
+			dataShards: 16,
+			want:       6000,
+		},
+		{
+			name: "negative dataShards falls back to default",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: (1 << 0) | (1 << 1) | (1 << 10),
+				ShardSizes:  []int64{1000, 1000, 1000},
+			},
+			dataShards: -1,
+			want:       2000,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := EcShardsDataSize(tt.msg, tt.dataShards); got != tt.want {
+				t.Errorf("EcShardsDataSize(%d) = %d, want %d", tt.dataShards, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestShardsInfo_String(t *testing.T) {
+	si := NewShardsInfo()
+	si.Set(ShardInfo{Id: 0, Size: 1024})
+	si.Set(ShardInfo{Id: 5, Size: 2048})
+
+	str := si.String()
+	if str == "" {
+		t.Error("String() should not be empty")
+	}
+	// Basic validation - should contain shard IDs
+	if len(str) < 3 {
+		t.Errorf("String() too short: %s", str)
+	}
+}
+
+func BenchmarkShardsInfo_Set(b *testing.B) {
+	si := NewShardsInfo()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		si.Set(ShardInfo{Id: ShardId(i % TotalShardsCount), Size: ShardSize(i * 1000)})
+	}
+}
+
+func BenchmarkShardsInfo_Has(b *testing.B) {
+	si := NewShardsInfo()
+	for i := 0; i < TotalShardsCount; i++ {
+		si.Set(ShardInfo{Id: ShardId(i), Size: ShardSize(i * 1000)})
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		si.Has(ShardId(i % TotalShardsCount))
+	}
+}
+
+func BenchmarkShardsInfo_Size(b *testing.B) {
+	si := NewShardsInfo()
+	for i := 0; i < TotalShardsCount; i++ {
+		si.Set(ShardInfo{Id: ShardId(i), Size: ShardSize(i * 1000)})
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		si.Size(ShardId(i % TotalShardsCount))
+	}
+}
+
+func TestShardsInfo_ConcurrentAccess(t *testing.T) {
+	si := NewShardsInfo()
+
+	var wg sync.WaitGroup
+	wg.Add(3)
+
+	// Goroutine 1: Continuously Set/Delete shards
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 1000; i++ {
+			si.Set(ShardInfo{Id: ShardId(i % TotalShardsCount), Size: 100})
+			if i%10 == 0 {
+				si.Delete(ShardId((i / 10) % TotalShardsCount))
+			}
+		}
+	}()
+
+	// Goroutine 2: Continuously read Info (Sizes, Bitmap, Count)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 1000; i++ {
+			si.Sizes()
+			si.Bitmap()
+			si.Count()
+			si.TotalSize()
+		}
+	}()
+
+	// Goroutine 3: Continuously Add/Subtract from another ShardsInfo
+	go func() {
+		defer wg.Done()
+		other := NewShardsInfo()
+		other.Set(ShardInfo{Id: 1, Size: 100})
+		other.Set(ShardInfo{Id: 2, Size: 200})
+		for i := 0; i < 1000; i++ {
+			si.Add(other)
+			si.Subtract(other)
+		}
+	}()
+
+	wg.Wait()
+}

weed/storage/erasure_coding/ec_teardown.go

@@ -0,0 +1,76 @@
+package erasure_coding
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"google.golang.org/grpc"
+)
+
+// ErrFullTeardownNotAcked marks a reachable server that completed the delete
+// RPC but did not report a full teardown (e.g. a pre-upgrade volume server), so
+// a stale EC generation may remain on it. Callers distinguish this from an
+// unreachable node (which may recover and be re-swept) with errors.Is.
+var ErrFullTeardownNotAcked = errors.New("delete did not perform full teardown (pre-upgrade volume server?); a stale EC generation may remain")
+
+// UnmountAndDeleteEcShards unmounts then tears down the named EC shards for a
+// volume on one server. Unmount must precede delete (delete requires the shard
+// be unmounted); both RPCs are idempotent against missing shards.
+//
+// encodeTsNs fences both RPCs:
+//   - 0 selects the server's blanket, generation-independent teardown. This is
+//     the correct choice for a pre-encode or rollback wipe: it clears same-
+//     generation shards (a retried encode's prior attempt shares the job's
+//     generation) and shards whose .vif generation is unreadable (an
+//     interrupted distribute never landed the sidecar) — both of which a fenced
+//     teardown preserves. The blanket path aborts rather than clobber a live
+//     newer mount, and the caller must guarantee no concurrent newer encode of
+//     this volume (e.g. the admin dedupe key, or an operator lock).
+//   - a non-zero value fences the teardown to strictly-older generations,
+//     preserving same-or-newer, generation 0, and an unreadable .vif — for a
+//     stale-worker cleanup that must never wipe a newer run's live shards.
+//
+// Returns ErrFullTeardownNotAcked (wrapped, so errors.Is matches) when a
+// reachable server does not ack the full teardown.
+//
+// This is the single teardown primitive shared by the plugin-worker EC task
+// and the shell ec.encode pre-cleanup, so the fence semantics cannot drift
+// between the two paths.
+func UnmountAndDeleteEcShards(
+	ctx context.Context,
+	dialOption grpc.DialOption,
+	server pb.ServerAddress,
+	collection string,
+	volumeID uint32,
+	shardIds []uint32,
+	encodeTsNs int64,
+) error {
+	return operation.WithVolumeServerClient(false, server, dialOption,
+		func(client volume_server_pb.VolumeServerClient) error {
+			if _, err := client.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
+				VolumeId:   volumeID,
+				ShardIds:   shardIds,
+				EncodeTsNs: encodeTsNs,
+			}); err != nil {
+				return fmt.Errorf("unmount: %w", err)
+			}
+			resp, err := client.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+				VolumeId:     volumeID,
+				Collection:   collection,
+				ShardIds:     shardIds,
+				FullTeardown: true,
+				EncodeTsNs:   encodeTsNs,
+			})
+			if err != nil {
+				return fmt.Errorf("delete: %w", err)
+			}
+			if !resp.GetFullTeardownDone() {
+				return fmt.Errorf("delete on %s: %w", server, ErrFullTeardownNotAcked)
+			}
+			return nil
+		})
+}

weed/storage/erasure_coding/ec_test.go

@@ -0,0 +1,275 @@
+package erasure_coding
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/klauspost/reedsolomon"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+const (
+	largeBlockSize = 10000
+	smallBlockSize = 100
+)
+
+func TestEncodingDecoding(t *testing.T) {
+	bufferSize := 50
+	baseFileName := "1"
+
+	// Create default EC context for testing
+	ctx := NewDefaultECContext("", 0)
+
+	_, err := generateEcFiles(baseFileName, bufferSize, largeBlockSize, smallBlockSize, ctx)
+	if err != nil {
+		t.Logf("generateEcFiles: %v", err)
+	}
+
+	err = WriteSortedFileFromIdx(baseFileName, ".ecx")
+	if err != nil {
+		t.Logf("WriteSortedFileFromIdx: %v", err)
+	}
+
+	err = validateFiles(baseFileName, ctx)
+	if err != nil {
+		t.Logf("WriteSortedFileFromIdx: %v", err)
+	}
+
+	removeGeneratedFiles(baseFileName, ctx)
+
+}
+
+func validateFiles(baseFileName string, ctx *ECContext) error {
+	nm, err := readNeedleMap(baseFileName)
+	if err != nil {
+		return fmt.Errorf("readNeedleMap: %v", err)
+	}
+	defer nm.Close()
+
+	datFile, err := os.OpenFile(baseFileName+".dat", os.O_RDONLY, 0)
+	if err != nil {
+		return fmt.Errorf("failed to open dat file: %v", err)
+	}
+	defer datFile.Close()
+
+	fi, err := datFile.Stat()
+	if err != nil {
+		return fmt.Errorf("failed to stat dat file: %v", err)
+	}
+
+	ecFiles, err := openEcFiles(baseFileName, true, ctx)
+	if err != nil {
+		return fmt.Errorf("error opening ec files: %w", err)
+	}
+	defer closeEcFiles(ecFiles)
+
+	err = nm.AscendingVisit(func(value needle_map.NeedleValue) error {
+		return assertSame(datFile, fi.Size(), ecFiles, value.Offset, value.Size)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to check ec files: %v", err)
+	}
+	return nil
+}
+
+func assertSame(datFile *os.File, datSize int64, ecFiles []*os.File, offset types.Offset, size types.Size) error {
+
+	data, err := readDatFile(datFile, offset, size)
+	if err != nil {
+		return fmt.Errorf("failed to read dat file: %v", err)
+	}
+
+	ecFileStat, _ := ecFiles[0].Stat()
+
+	ecData, err := readEcFile(ecFileStat.Size(), ecFiles, offset, size)
+	if err != nil {
+		return fmt.Errorf("failed to read ec file: %v", err)
+	}
+
+	if bytes.Compare(data, ecData) != 0 {
+		return fmt.Errorf("unexpected data read")
+	}
+
+	return nil
+}
+
+func readDatFile(datFile *os.File, offset types.Offset, size types.Size) ([]byte, error) {
+
+	data := make([]byte, size)
+	n, err := datFile.ReadAt(data, offset.ToActualOffset())
+	if err != nil {
+		return nil, fmt.Errorf("failed to ReadAt dat file: %v", err)
+	}
+	if n != int(size) {
+		return nil, fmt.Errorf("unexpected read size %d, expected %d", n, size)
+	}
+	return data, nil
+}
+
+func readEcFile(shardDatSize int64, ecFiles []*os.File, offset types.Offset, size types.Size) (data []byte, err error) {
+
+	intervals := LocateData(largeBlockSize, smallBlockSize, shardDatSize, offset.ToActualOffset(), size)
+
+	for i, interval := range intervals {
+		if d, e := readOneInterval(interval, ecFiles); e != nil {
+			return nil, e
+		} else {
+			if i == 0 {
+				data = d
+			} else {
+				data = append(data, d...)
+			}
+		}
+	}
+
+	return data, nil
+}
+
+func readOneInterval(interval Interval, ecFiles []*os.File) (data []byte, err error) {
+
+	ecFileIndex, ecFileOffset := interval.ToShardIdAndOffset(largeBlockSize, smallBlockSize)
+
+	data = make([]byte, interval.Size)
+	err = readFromFile(ecFiles[ecFileIndex], data, ecFileOffset)
+	if false { // do some ec testing
+		ecData, err := readFromOtherEcFiles(ecFiles, int(ecFileIndex), ecFileOffset, interval.Size)
+		if err != nil {
+			return nil, fmt.Errorf("ec reconstruct error: %v", err)
+		}
+		if bytes.Compare(data, ecData) != 0 {
+			return nil, fmt.Errorf("ec compare error")
+		}
+	}
+	return
+}
+
+func readFromOtherEcFiles(ecFiles []*os.File, ecFileIndex int, ecFileOffset int64, size types.Size) (data []byte, err error) {
+	enc, err := reedsolomon.New(DataShardsCount, ParityShardsCount)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create encoder: %v", err)
+	}
+
+	bufs := make([][]byte, TotalShardsCount)
+	for i := 0; i < DataShardsCount; {
+		n := int(rand.Int31n(TotalShardsCount))
+		if n == ecFileIndex || bufs[n] != nil {
+			continue
+		}
+		bufs[n] = make([]byte, size)
+		i++
+	}
+
+	for i, buf := range bufs {
+		if buf == nil {
+			continue
+		}
+		err = readFromFile(ecFiles[i], buf, ecFileOffset)
+		if err != nil {
+			return
+		}
+	}
+
+	if err = enc.ReconstructData(bufs); err != nil {
+		return nil, err
+	}
+
+	return bufs[ecFileIndex], nil
+}
+
+func readFromFile(file *os.File, data []byte, ecFileOffset int64) (err error) {
+	_, err = file.ReadAt(data, ecFileOffset)
+	return
+}
+
+func removeGeneratedFiles(baseFileName string, ctx *ECContext) {
+	for i := 0; i < ctx.Total(); i++ {
+		fname := baseFileName + ctx.ToExt(i)
+		os.Remove(fname)
+	}
+	os.Remove(baseFileName + ".ecx")
+	RemoveBitrotSidecars(baseFileName)
+}
+
+func TestLocateData(t *testing.T) {
+	intervals := LocateData(largeBlockSize, smallBlockSize, largeBlockSize+1, DataShardsCount*largeBlockSize, 1)
+	if len(intervals) != 1 {
+		t.Errorf("unexpected interval size %d", len(intervals))
+	}
+	if !intervals[0].sameAs(Interval{0, 0, 1, false, 1}) {
+		t.Errorf("unexpected interval %+v", intervals[0])
+	}
+
+	intervals = LocateData(largeBlockSize, smallBlockSize, largeBlockSize+1, DataShardsCount*largeBlockSize/2+100, DataShardsCount*largeBlockSize+1-DataShardsCount*largeBlockSize/2-100)
+	fmt.Printf("%+v\n", intervals)
+}
+
+func (this Interval) sameAs(that Interval) bool {
+	return this.IsLargeBlock == that.IsLargeBlock &&
+		this.InnerBlockOffset == that.InnerBlockOffset &&
+		this.BlockIndex == that.BlockIndex &&
+		this.Size == that.Size
+}
+
+func TestLocateData2(t *testing.T) {
+	// Use ecdFileSize-1 to simulate the fallback path in LocateEcShardNeedleInterval
+	// when datFileSize is not available (old EC volumes without .vif datFileSize).
+	intervals := LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, 3221225472-1, 21479557912, 4194339)
+	assert.Equal(t, intervals, []Interval{
+		{BlockIndex: 4, InnerBlockOffset: 527128, Size: 521448, IsLargeBlock: false, LargeBlockRowsCount: 2},
+		{BlockIndex: 5, InnerBlockOffset: 0, Size: 1048576, IsLargeBlock: false, LargeBlockRowsCount: 2},
+		{BlockIndex: 6, InnerBlockOffset: 0, Size: 1048576, IsLargeBlock: false, LargeBlockRowsCount: 2},
+		{BlockIndex: 7, InnerBlockOffset: 0, Size: 1048576, IsLargeBlock: false, LargeBlockRowsCount: 2},
+		{BlockIndex: 8, InnerBlockOffset: 0, Size: 527163, IsLargeBlock: false, LargeBlockRowsCount: 2},
+	})
+}
+
+func TestLocateData3(t *testing.T) {
+	// Use ecdFileSize-1 to simulate the fallback path in LocateEcShardNeedleInterval
+	intervals := LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, 3221225472-1, 30782909808, 112568)
+	for _, interval := range intervals {
+		fmt.Printf("%+v\n", interval)
+	}
+	assert.Equal(t, intervals, []Interval{
+		{BlockIndex: 8876, InnerBlockOffset: 912752, Size: 112568, IsLargeBlock: false, LargeBlockRowsCount: 2},
+	})
+}
+
+func TestLocateData_ExactMultiple_Issue8947(t *testing.T) {
+	// When datFileSize is available, shardDatSize = datFileSize / DataShards.
+	// For a 30GB volume with 10 data shards, shardDatSize = 3GB = 3 * ErasureCodingLargeBlockSize.
+	// The encoder produces 3 large block rows, 0 small block rows.
+	// nLargeBlockRows must be 3, not 2.
+	shardDatSize := int64(3) * ErasureCodingLargeBlockSize // 3GB per shard from datFileSize/DataShards
+
+	// Reading from the 3rd large block row (offsets 20GB-30GB) should work
+	offset := int64(2) * ErasureCodingLargeBlockSize * DataShardsCount // 20GB
+	intervals := LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shardDatSize, offset, 1024)
+	assert.Equal(t, 1, len(intervals))
+	assert.True(t, intervals[0].IsLargeBlock, "data in 3rd large row should be in large blocks")
+	assert.Equal(t, 3, intervals[0].LargeBlockRowsCount)
+	assert.Equal(t, 20, intervals[0].BlockIndex) // block 20 = shard 0 of 3rd row
+}
+
+func TestLocateData_Issue8179(t *testing.T) {
+	large := int64(10000)
+	small := int64(100)
+	shardSize := int64(259092) // Resulting in nLargeBlockRows = 25 as seen in panic log
+
+	// Testing range through the large-to-small transition boundary
+	nLargeBlockRows := shardSize / large
+	largeAreaSize := nLargeBlockRows * int64(DataShardsCount) * large
+
+	for offset := largeAreaSize - 500; offset < largeAreaSize+500; offset++ {
+		intervals := LocateData(large, small, shardSize, offset, 200)
+		for _, interval := range intervals {
+			assert.True(t, interval.Size > 0, "Interval size must be positive at offset %d, got %+v", offset, interval)
+		}
+	}
+}

weed/storage/erasure_coding/ec_uniform_layout_test.go

@@ -0,0 +1,252 @@
+package erasure_coding
+
+import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"os"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+)
+
+// legacyShardSize is the padded shard length the two-tier layout produces:
+// whole 1GiB rows, then the remainder in 1MiB rows.
+func legacyShardSize(datFileSize int64, dataShards int) int64 {
+	largeRow := int64(ErasureCodingLargeBlockSize) * int64(dataShards)
+	nLarge := datFileSize / largeRow
+	size := nLarge * ErasureCodingLargeBlockSize
+	if rem := datFileSize - nLarge*largeRow; rem > 0 {
+		smallRow := int64(ErasureCodingSmallBlockSize) * int64(dataShards)
+		size += (rem + smallRow - 1) / smallRow * ErasureCodingSmallBlockSize
+	}
+	return size
+}
+
+// The uniform layout must never change a shard's length, only the byte
+// placement — capacity math and shard-size credibility checks depend on it.
+func TestUniformBlockSizeMatchesLegacyShardSize(t *testing.T) {
+	sizes := []int64{
+		8, 1000, ErasureCodingSmallBlockSize,
+		10 * ErasureCodingSmallBlockSize,
+		10*ErasureCodingSmallBlockSize + 1,
+		25 * 1024 * 1024,
+		10 * ErasureCodingLargeBlockSize,
+		10*ErasureCodingLargeBlockSize + 1,
+		10*ErasureCodingLargeBlockSize - ErasureCodingSmallBlockSize,
+		30 * 1024 * 1024 * 1024,
+		30*1024*1024*1024 + 12345,
+	}
+	for _, datSize := range sizes {
+		for _, ds := range []int{10, 9, 5, 1} {
+			if got, want := UniformBlockSize(datSize, ds), legacyShardSize(datSize, ds); got != want {
+				t.Errorf("UniformBlockSize(%d, %d) = %d, legacy shard size %d", datSize, ds, got, want)
+			}
+		}
+	}
+	if got := UniformBlockSize(0, 10); got != ErasureCodingSmallBlockSize {
+		t.Errorf("UniformBlockSize(0, 10) = %d, want one small block", got)
+	}
+}
+
+// encodeLayoutFixture writes a random .dat and encodes it with the given block
+// sizes, returning the .dat content.
+func encodeLayoutFixture(t *testing.T, baseFileName string, datSize int64, large, small int64, ctx *ECContext) []byte {
+	t.Helper()
+	data := make([]byte, datSize)
+	rand.New(rand.NewSource(datSize)).Read(data)
+	if err := os.WriteFile(baseFileName+".dat", data, 0o644); err != nil {
+		t.Fatalf("write .dat: %v", err)
+	}
+	if _, err := generateEcFiles(baseFileName, 256*1024, large, small, ctx); err != nil {
+		t.Fatalf("generateEcFiles: %v", err)
+	}
+	return data
+}
+
+// readIntervals assembles a byte range from the shard files the way the read
+// path does.
+func readIntervals(t *testing.T, baseFileName string, ctx *ECContext, intervals []Interval) []byte {
+	t.Helper()
+	var assembled []byte
+	for _, iv := range intervals {
+		shardId, shardOffset := iv.ToShardIdAndOffset(ctx.LargeBlockSize(), ctx.SmallBlockSize())
+		f, err := os.Open(baseFileName + ctx.ToExt(int(shardId)))
+		if err != nil {
+			t.Fatalf("open shard %d: %v", shardId, err)
+		}
+		buf := make([]byte, iv.Size)
+		if _, err := f.ReadAt(buf, shardOffset); err != nil {
+			t.Fatalf("read shard %d at %d: %v", shardId, shardOffset, err)
+		}
+		f.Close()
+		assembled = append(assembled, buf...)
+	}
+	return assembled
+}
+
+// Encode 25MB — large enough that the uniform and legacy layouts place bytes
+// differently — under both layouts and verify LocateData maps every probed
+// range back to the original bytes.
+func TestLocateDataMatchesEncoderForBothLayouts(t *testing.T) {
+	const datSize = 25*1024*1024 + 12345
+	dir := t.TempDir()
+
+	uniformCtx := NewDefaultECContext("", 1)
+	uniformCtx.BlockSize = UniformBlockSize(datSize, uniformCtx.DataShards)
+	legacyCtx := NewDefaultECContext("", 2)
+
+	for name, ctx := range map[string]*ECContext{"uniform": uniformCtx, "legacy": legacyCtx} {
+		t.Run(name, func(t *testing.T) {
+			base := fmt.Sprintf("%s/%s", dir, name)
+			data := encodeLayoutFixture(t, base, datSize, ctx.LargeBlockSize(), ctx.SmallBlockSize(), ctx)
+
+			shardSize := datSize / int64(ctx.DataShards)
+			for _, probe := range []struct{ offset, size int64 }{
+				{0, 1000},
+				{ctx.SmallBlockSize() - 100, 200}, // straddles a block boundary
+				{5 * 1024 * 1024, 4 * 1024 * 1024},
+				{datSize - 2000, 2000},
+			} {
+				intervals := LocateData(ctx.LargeBlockSize(), ctx.SmallBlockSize(), shardSize, probe.offset, types.Size(probe.size))
+				got := readIntervals(t, base, ctx, intervals)
+				if !bytes.Equal(got, data[probe.offset:probe.offset+probe.size]) {
+					t.Fatalf("%s layout: bytes at [%d,+%d) do not round-trip", name, probe.offset, probe.size)
+				}
+			}
+
+			// The point of the uniform layout: a 2MB range inside one 3MB
+			// block is a single interval instead of three 1MB stripes.
+			intervals := LocateData(ctx.LargeBlockSize(), ctx.SmallBlockSize(), shardSize, 6*1024*1024+512*1024, types.Size(2*1024*1024))
+			if name == "uniform" && len(intervals) != 1 {
+				t.Errorf("uniform layout reads 2MB in %d intervals, want 1", len(intervals))
+			}
+			if name == "legacy" && len(intervals) != 3 {
+				t.Errorf("legacy layout reads 2MB in %d intervals, want 3", len(intervals))
+			}
+		})
+	}
+}
+
+// Encode under both layouts and verify WriteDatFile reconstructs the original
+// .dat byte for byte — and that decoding a uniform volume with the legacy
+// geometry does NOT, i.e. the recorded block size is load-bearing.
+func TestUniformDecodeRoundTrip(t *testing.T) {
+	const datSize = 25 * 1024 * 1024
+	dir := t.TempDir()
+	base := dir + "/1"
+
+	ctx := NewDefaultECContext("", 1)
+	ctx.BlockSize = UniformBlockSize(datSize, ctx.DataShards)
+	data := encodeLayoutFixture(t, base, datSize, ctx.BlockSize, ctx.BlockSize, ctx)
+
+	shardFileNames := make([]string, ctx.DataShards)
+	for i := range shardFileNames {
+		shardFileNames[i] = base + ctx.ToExt(i)
+	}
+
+	if err := WriteDatFile(dir+"/decoded", datSize, datSize, shardFileNames, ctx.LargeBlockSize(), ctx.SmallBlockSize()); err != nil {
+		t.Fatalf("WriteDatFile: %v", err)
+	}
+	decoded, err := os.ReadFile(dir + "/decoded.dat")
+	if err != nil {
+		t.Fatalf("read decoded: %v", err)
+	}
+	if !bytes.Equal(decoded, data) {
+		t.Fatal("uniform decode with recorded geometry does not round-trip")
+	}
+
+	if err := WriteDatFile(dir+"/wrong", datSize, datSize, shardFileNames, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize); err != nil {
+		t.Fatalf("WriteDatFile legacy geometry: %v", err)
+	}
+	wrong, err := os.ReadFile(dir + "/wrong.dat")
+	if err != nil {
+		t.Fatalf("read wrong: %v", err)
+	}
+	if bytes.Equal(wrong, data) {
+		t.Fatal("decoding a uniform volume with legacy geometry should scramble it; the layouts did not diverge")
+	}
+}
+
+// Mount fixtures through NewEcVolume and read a large extent back through
+// LocateEcShardNeedleInterval, proving the .vif block size steers the read
+// path: a legacy .vif (no block size) keeps the legacy interpretation, a
+// uniform .vif reads uniform shards.
+func TestEcVolumeGeometryFromVif(t *testing.T) {
+	const datSize = 25 * 1024 * 1024
+
+	for _, layout := range []string{"legacy", "uniform"} {
+		t.Run(layout, func(t *testing.T) {
+			dir := t.TempDir()
+			vid := needle.VolumeId(7)
+			base := EcShardFileName("", dir, int(vid))
+
+			ctx := NewDefaultECContext("", vid)
+			if layout == "uniform" {
+				ctx.BlockSize = UniformBlockSize(datSize, ctx.DataShards)
+			}
+			data := encodeLayoutFixture(t, base, datSize, ctx.LargeBlockSize(), ctx.SmallBlockSize(), ctx)
+
+			if err := os.WriteFile(base+".ecx", nil, 0o644); err != nil {
+				t.Fatalf("write .ecx: %v", err)
+			}
+			if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+				Version:     uint32(needle.GetCurrentVersion()),
+				DatFileSize: datSize,
+				EcShardConfig: &volume_server_pb.EcShardConfig{
+					DataShards:   uint32(ctx.DataShards),
+					ParityShards: uint32(ctx.ParityShards),
+					BlockSize:    ctx.BlockSize,
+				},
+			}); err != nil {
+				t.Fatalf("save .vif: %v", err)
+			}
+
+			ev, err := NewEcVolume(types.HardDriveType, dir, dir, "", vid)
+			if err != nil {
+				t.Fatalf("NewEcVolume: %v", err)
+			}
+			defer ev.Close()
+			if ev.ECContext.BlockSize != ctx.BlockSize {
+				t.Fatalf("loaded BlockSize = %d, want %d", ev.ECContext.BlockSize, ctx.BlockSize)
+			}
+			for i := 0; i < ctx.DataShards; i++ {
+				shard, err := NewEcVolumeShard(types.HardDriveType, dir, "", vid, ShardId(i))
+				if err != nil {
+					t.Fatalf("NewEcVolumeShard %d: %v", i, err)
+				}
+				if _, err := ev.AddEcVolumeShard(shard); err != nil {
+					t.Fatalf("AddEcVolumeShard: %v", err)
+				}
+			}
+
+			// Sweep a large extent through the volume's own interval mapping.
+			// LocateEcShardNeedleInterval expands the size by the needle
+			// overhead, so compare only the probed prefix.
+			const probeSize = datSize - 64*1024
+			intervals := ev.LocateEcShardNeedleInterval(needle.GetCurrentVersion(), 0, types.Size(probeSize))
+			var assembled []byte
+			for _, iv := range intervals {
+				shardId, shardOffset := ev.IntervalToShardIdAndOffset(iv)
+				shard, found := ev.FindEcVolumeShard(shardId)
+				if !found {
+					t.Fatalf("shard %d not mounted", shardId)
+				}
+				buf := make([]byte, iv.Size)
+				if _, err := shard.ReadAt(buf, shardOffset); err != nil {
+					t.Fatalf("read shard %d at %d: %v", shardId, shardOffset, err)
+				}
+				assembled = append(assembled, buf...)
+			}
+			if len(assembled) < probeSize {
+				t.Fatalf("assembled %d bytes, want at least %d", len(assembled), probeSize)
+			}
+			if !bytes.Equal(assembled[:probeSize], data[:probeSize]) {
+				t.Fatalf("%s layout: full-extent read through the %s .vif does not match the .dat", layout, layout)
+			}
+		})
+	}
+}

weed/storage/erasure_coding/ec_volume.go

@@ -0,0 +1,739 @@
+package erasure_coding
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"slices"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+)
+
+var (
+	NotFoundError             = errors.New("needle not found")
+	destroyDelaySeconds int64 = 0
+)
+
+type EcVolume struct {
+	VolumeId                  needle.VolumeId
+	Collection                string
+	dir                       string
+	dirIdx                    string
+	ecxActualDir              string // directory where .ecx/.ecj were actually found (may differ from dirIdx after fallback)
+	ecxFile                   *os.File
+	ecxFileSize               int64
+	ecxCreatedAt              time.Time
+	Shards                    []*EcVolumeShard
+	ShardLocations            map[ShardId][]pb.ServerAddress
+	ShardLocationsRefreshTime time.Time
+	// ShardLocationsStale marks the map for a prompt re-check: a read that failed
+	// against a cached location has disproved what the map claims, and the normal
+	// freshness window is far too long to serve from a map known to be wrong.
+	ShardLocationsStale bool
+	ShardLocationsLock  sync.RWMutex
+	Version             needle.Version
+	ecjFile             *os.File
+	ecjFileAccessLock   sync.Mutex
+	diskType            types.DiskType
+	datFileSize         int64
+	ExpireAtSec         uint64     //ec volume destroy time, calculated from the ec volume was created
+	ECContext           *ECContext // EC encoding parameters
+
+	// EncodeTsNs is the encode time (unix nanos) loaded from .vif; reads carry it
+	// so a shard from a different encode run is rejected. 0 for pre-upgrade volumes.
+	EncodeTsNs int64
+
+	// ecjFileSize mirrors the on-disk size of the .ecj deletion journal and
+	// is maintained under ecjFileAccessLock. It is only used by IO helpers
+	// (seek/truncate) — the authoritative runtime delete count comes from
+	// deletedNeedles.
+	ecjFileSize int64
+
+	// deletedNeedles is the in-memory set of needle ids that have been
+	// deleted since the volume was encoded. .ecx is immutable at runtime —
+	// it only stores the sorted (id, offset, size) index written at encode
+	// time — and runtime deletes are journaled to .ecj + tracked here.
+	// Reads consult this set to mask out deleted needles on top of the
+	// sealed .ecx lookup. Heartbeat delete_count is derived from len(set).
+	// Seeded from .ecj in NewEcVolume and updated under deletedNeedlesLock.
+	deletedNeedlesLock sync.RWMutex
+	deletedNeedles     map[types.NeedleId]struct{}
+
+	// Bitrot checksum sidecar for the active generation (optional). bitrot is
+	// nil unless bitrotStatus == BitrotOn, and is loaded at mount. Guarded by
+	// bitrotLock.
+	bitrotLock   sync.RWMutex
+	bitrot       *volume_server_pb.EcBitrotProtection
+	bitrotStatus BitrotStatus
+
+	lastIoError        error
+	lastIoErrorCount   int32
+	ioErrorQuarantined bool
+	lastIoErrorLock    sync.RWMutex
+}
+
+func (ev *EcVolume) CheckReadWriteError(err error) {
+	if err == nil {
+		ev.clearIoError()
+		return
+	}
+	if errors.Is(err, syscall.EIO) {
+		ev.noteIoError(err)
+		return
+	}
+	ev.clearIoError()
+}
+
+func (ev *EcVolume) noteIoError(err error) {
+	ev.lastIoErrorLock.Lock()
+	defer ev.lastIoErrorLock.Unlock()
+	ev.lastIoError = err
+	ev.lastIoErrorCount++
+	stats.VolumeServerStorageIoErrorCounter.Inc()
+}
+
+func (ev *EcVolume) clearIoError() {
+	ev.lastIoErrorLock.Lock()
+	defer ev.lastIoErrorLock.Unlock()
+	ev.lastIoError = nil
+	ev.lastIoErrorCount = 0
+}
+
+func (ev *EcVolume) ResetIoErrorState() {
+	ev.lastIoErrorLock.Lock()
+	defer ev.lastIoErrorLock.Unlock()
+	ev.lastIoError = nil
+	ev.lastIoErrorCount = 0
+	ev.ioErrorQuarantined = false
+}
+
+func (ev *EcVolume) MarkIoQuarantined() {
+	ev.lastIoErrorLock.Lock()
+	defer ev.lastIoErrorLock.Unlock()
+	ev.ioErrorQuarantined = true
+}
+
+func (ev *EcVolume) GetIoErrorState() (error, int32, bool) {
+	ev.lastIoErrorLock.RLock()
+	defer ev.lastIoErrorLock.RUnlock()
+	return ev.lastIoError, ev.lastIoErrorCount, ev.ioErrorQuarantined
+}
+
+// statEcxSize returns the size of an .ecx file, os.ErrNotExist when it is absent
+// (or a directory), so the resolver can prefer a non-empty copy.
+func statEcxSize(path string) (int64, error) {
+	info, statErr := os.Stat(path)
+	if statErr != nil {
+		return 0, statErr
+	}
+	if info.IsDir() {
+		return 0, os.ErrNotExist
+	}
+	return info.Size(), nil
+}
+
+func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection string, vid needle.VolumeId) (ev *EcVolume, err error) {
+	ev = &EcVolume{dir: dir, dirIdx: dirIdx, Collection: collection, VolumeId: vid, diskType: diskType}
+
+	dataBaseFileName := EcShardFileName(collection, dir, int(vid))
+	indexBaseFileName := EcShardFileName(collection, dirIdx, int(vid))
+
+	// open ecx file. Wrap errors with %w so callers walking up the stack
+	// (notably Store.MountEcShards) can use errors.Is(err, os.ErrNotExist)
+	// to decide whether to try the next local disk vs. bail. A 0-byte .ecx
+	// is a legitimate index for a volume that had no live needles at encode
+	// time (e.g. all needles deleted before WriteSortedFileFromIdx) and
+	// must mount successfully here. A 0-byte stub left by a failed copy
+	// stream is indistinguishable from that empty case by file size alone;
+	// preventing such stubs is the receiver-side cleanup in writeToFile's
+	// job, not this open path.
+	// Resolve the .ecx, preferring the copy co-located with the shard data on
+	// this disk — where a move or reconstruct leaves it — then the caller's
+	// index directory. That directory is either the shared -dir.idx dir or a
+	// sibling disk that owns the .ecx when this disk holds only a 0-byte stub
+	// left by an interrupted copy (#9212). A 0-byte .ecx is a legitimate empty
+	// index, so the local copy yields only to a *non-empty* copy elsewhere,
+	// never to a mere absence: prefer a non-empty .ecx local-first, then fall
+	// back to whichever exists at all.
+	localBaseFileName := dataBaseFileName
+	sharedBaseFileName := indexBaseFileName
+	localSize, localErr := statEcxSize(localBaseFileName + ".ecx")
+	sharedSize, sharedErr := int64(0), os.ErrNotExist
+	if dirIdx != dir {
+		sharedSize, sharedErr = statEcxSize(sharedBaseFileName + ".ecx")
+	}
+	switch {
+	case localErr == nil && localSize > 0:
+		indexBaseFileName, ev.ecxActualDir = localBaseFileName, dir
+	case sharedErr == nil && sharedSize > 0:
+		indexBaseFileName, ev.ecxActualDir = sharedBaseFileName, dirIdx
+		glog.V(1).Infof("ecx not local at %s.ecx, using %s.ecx", localBaseFileName, sharedBaseFileName)
+	case localErr == nil: // local exists but is a 0-byte empty index
+		indexBaseFileName, ev.ecxActualDir = localBaseFileName, dir
+	case sharedErr == nil: // only a 0-byte copy in the index dir
+		indexBaseFileName, ev.ecxActualDir = sharedBaseFileName, dirIdx
+	default:
+		return nil, fmt.Errorf("cannot open ec volume index %s.ecx (or %s.ecx): %w", localBaseFileName, sharedBaseFileName, os.ErrNotExist)
+	}
+	if ev.ecxFile, err = backend.OpenVolumeFile(indexBaseFileName+".ecx", os.O_RDWR); err != nil {
+		return nil, fmt.Errorf("cannot open ec volume index %s.ecx: %w", indexBaseFileName, err)
+	}
+	ecxFi, statErr := ev.ecxFile.Stat()
+	if statErr != nil {
+		_ = ev.ecxFile.Close()
+		return nil, fmt.Errorf("can not stat ec volume index %s.ecx: %w", indexBaseFileName, statErr)
+	}
+	ev.ecxFileSize = ecxFi.Size()
+	ev.ecxCreatedAt = ecxFi.ModTime()
+
+	// open ecj file and seed the in-memory deleted set from it.
+	if ev.ecjFile, err = backend.OpenVolumeFile(indexBaseFileName+".ecj", os.O_RDWR|os.O_CREATE); err != nil {
+		return nil, fmt.Errorf("cannot open ec volume journal %s.ecj: %v", indexBaseFileName, err)
+	}
+	if ecjFi, statErr := ev.ecjFile.Stat(); statErr == nil {
+		ev.ecjFileSize = ecjFi.Size()
+	} else {
+		glog.Warningf("stat ec volume journal %s.ecj: %v", indexBaseFileName, statErr)
+	}
+	ev.deletedNeedles = make(map[types.NeedleId]struct{})
+	if loadErr := ev.loadDeletedNeedlesFromEcj(); loadErr != nil {
+		glog.Warningf("ec volume %d: load deleted needles from .ecj: %v", vid, loadErr)
+	}
+
+	// read volume info. Prefer .vif at the data dir (where shards live), but
+	// fall back to the index dir when the data dir does not have one — the
+	// orphan-shard reconciliation in Store loads shards on a disk whose only
+	// EC artefacts are .ec?? files, with .ecx / .ecj / .vif on a sibling disk
+	// (issue #9212). Without this fallback we'd write a stub .vif on the
+	// shard disk and lose the real EC config + datFileSize.
+	vifFileName := dataBaseFileName + ".vif"
+	if dirIdx != dir {
+		if _, statErr := os.Stat(vifFileName); statErr != nil && os.IsNotExist(statErr) {
+			altVif := EcShardFileName(collection, dirIdx, int(vid)) + ".vif"
+			if _, altStatErr := os.Stat(altVif); altStatErr == nil {
+				vifFileName = altVif
+			}
+		}
+	}
+	ev.Version = needle.Version3
+	// A present-but-unreadable or malformed .vif FAILS the mount: every new
+	// encode records a positive uniform block size there, and defaulting to
+	// the legacy layout would serve those shards with the wrong offset math.
+	// Absent stays legal — legacy volumes predate the sidecar.
+	volumeInfo, _, found, vifErr := volume_info.MaybeLoadVolumeInfo(vifFileName)
+	if vifErr != nil {
+		ev.Close()
+		return nil, fmt.Errorf("ec volume %d: load %s: %w", vid, vifFileName, vifErr)
+	}
+	if found {
+		ev.Version = needle.Version(volumeInfo.Version)
+		ev.datFileSize = volumeInfo.DatFileSize
+		ev.ExpireAtSec = volumeInfo.ExpireAtSec
+
+		// Initialize EC context from .vif if present; fallback to defaults
+		if volumeInfo.EcShardConfig != nil {
+			ds := int(volumeInfo.EcShardConfig.DataShards)
+			ps := int(volumeInfo.EcShardConfig.ParityShards)
+			ev.EncodeTsNs = volumeInfo.EcShardConfig.GetEncodeTsNs()
+
+			// A config that is PRESENT but records an impossible ratio is not
+			// a volume to fall back on: substituting the default 10+4 with the
+			// legacy layout would read uniform shards with the wrong offset
+			// math and answer with the wrong bytes. Only an ENTIRELY absent
+			// config means "this predates the record", which the else-branch
+			// below serves with the legacy defaults.
+			if !ValidEcShardCounts(volumeInfo.EcShardConfig.DataShards, volumeInfo.EcShardConfig.ParityShards) {
+				ev.Close()
+				return nil, fmt.Errorf("ec volume %d: %s records invalid shard counts %d+%d",
+					vid, vifFileName, volumeInfo.EcShardConfig.DataShards, volumeInfo.EcShardConfig.ParityShards)
+			} else if blockErr := ValidateBlockSize(volumeInfo.EcShardConfig.GetBlockSize()); blockErr != nil {
+				// A recorded block size that no encoder could have produced maps
+				// every read to the wrong shard offset. Refuse the mount rather
+				// than serve those bytes or silently pick a layout.
+				ev.Close()
+				return nil, fmt.Errorf("ec volume %d: %s: %w", vid, vifFileName, blockErr)
+			} else {
+				ev.ECContext = &ECContext{
+					Collection:   collection,
+					VolumeId:     vid,
+					DataShards:   ds,
+					ParityShards: ps,
+					BlockSize:    volumeInfo.EcShardConfig.GetBlockSize(),
+				}
+				glog.V(1).Infof("Loaded EC config from VolumeInfo for volume %d: %s", vid, ev.ECContext.String())
+			}
+		} else {
+			// A vif that carries no ecShardConfig answers nothing about the
+			// layout — it is no more informative than an absent one, so it
+			// must not skip the sidecar. Going straight to the defaults here
+			// read a uniform volume with the legacy offset math.
+			cfg, sidecarFound, sidecarErr := layoutFromSidecar(dataBaseFileName, indexBaseFileName)
+			if sidecarErr != nil {
+				ev.Close()
+				return nil, fmt.Errorf("ec volume %d: %s records no EC config and the bitrot sidecar cannot establish the layout: %w", vid, vifFileName, sidecarErr)
+			}
+			if sidecarFound {
+				ev.ECContext = &ECContext{
+					Collection:   collection,
+					VolumeId:     vid,
+					DataShards:   int(cfg.GetDataShards()),
+					ParityShards: int(cfg.GetParityShards()),
+					BlockSize:    cfg.GetBlockSize(),
+				}
+				ev.EncodeTsNs = cfg.GetEncodeTsNs()
+				glog.V(0).Infof("ec volume %d: .vif records no EC config; took it from the bitrot sidecar: %s",
+					vid, ev.ECContext.String())
+			} else {
+				ev.ECContext = NewDefaultECContext(collection, vid)
+			}
+		}
+	} else {
+		// Don't fabricate a stub .vif here: a version-only stub implies the
+		// default 10+4 ratio with DatFileSize=0 and no encode identity, which
+		// the custom-ratio resolver and the startup credibility checks must not
+		// mistake for an authoritative config. Mount with in-memory defaults and
+		// leave the real .vif to the encoder or a recovery tool (the Rust volume
+		// server already behaves this way).
+		//
+		// The bitrot sidecar records the same EC config at encode time, so when
+		// it is present it answers the layout question the missing .vif cannot:
+		// defaulting a uniform-layout volume to the legacy block sizes maps
+		// every read to the wrong shard offset. `weed fix -ecx` reads the
+		// sidecar for the same reason.
+		ev.ECContext = NewDefaultECContext(collection, vid)
+		cfg, sidecarFound, sidecarErr := layoutFromSidecar(dataBaseFileName, indexBaseFileName)
+		if sidecarErr != nil {
+			// With no .vif the sidecar is the ONLY record of this volume's
+			// layout. Present but unusable is not "assume legacy" — that
+			// answers reads with the wrong shard offsets, which is worse than
+			// not answering at all.
+			ev.Close()
+			return nil, fmt.Errorf("ec volume %d: no .vif and the bitrot sidecar cannot establish the layout: %w", vid, sidecarErr)
+		}
+		if sidecarFound {
+			ev.ECContext = &ECContext{
+				Collection:   collection,
+				VolumeId:     vid,
+				DataShards:   int(cfg.GetDataShards()),
+				ParityShards: int(cfg.GetParityShards()),
+				BlockSize:    cfg.GetBlockSize(),
+			}
+			ev.EncodeTsNs = cfg.GetEncodeTsNs()
+			glog.V(0).Infof("ec volume %d: .vif missing; took EC config from the bitrot sidecar: %s",
+				vid, ev.ECContext.String())
+		} else {
+			// Only now are the defaults what the volume actually mounted on;
+			// logging this after the sidecar answered would send an operator
+			// triaging wrong bytes after the legacy layout instead.
+			glog.Warningf("vif file not found, using defaults, volumeId:%d, filename:%s", vid, vifFileName)
+		}
+	}
+
+	ev.ShardLocations = make(map[ShardId][]pb.ServerAddress)
+
+	// Load the active-generation bitrot checksum sidecar (optional).
+	if err := ev.loadActiveBitrotSidecar(); err != nil {
+		ev.Close()
+		return nil, err
+	}
+
+	return
+}
+
+func (ev *EcVolume) AddEcVolumeShard(ecVolumeShard *EcVolumeShard) (bool, error) {
+	for _, s := range ev.Shards {
+		if s.ShardId == ecVolumeShard.ShardId {
+			return false, nil
+		}
+	}
+	// A 0-byte shard file beside an index with entries is residue of a
+	// failed copy or a truncation, not a mountable shard: registering it
+	// would advertise a size-0 claim that serves nothing and, since
+	// placement pins re-copies to the owning disk, would keep attracting
+	// repairs to a file that was never valid. A 0-byte shard beside a
+	// 0-byte index is different — that is the legitimate layout of a
+	// volume encoded with no live needles, and it must keep mounting.
+	// The startup scan already skips 0-byte shard files; this covers the
+	// mount RPC path, which opens the file directly.
+	if ecVolumeShard.Size() == 0 && ev.ecxFileSize > 0 {
+		return false, fmt.Errorf("ec volume %d shard %d: shard file is empty (0 bytes) but the index has %d entries: residue of a failed copy, not a mountable shard",
+			ev.VolumeId, ecVolumeShard.ShardId, ev.ecxFileSize/types.NeedleMapEntrySize)
+	}
+	ev.Shards = append(ev.Shards, ecVolumeShard)
+	slices.SortFunc(ev.Shards, func(a, b *EcVolumeShard) int {
+		if a.VolumeId != b.VolumeId {
+			return int(a.VolumeId - b.VolumeId)
+		}
+		return int(a.ShardId - b.ShardId)
+	})
+	return true, nil
+}
+
+func (ev *EcVolume) DeleteEcVolumeShard(shardId ShardId) (ecVolumeShard *EcVolumeShard, deleted bool) {
+	foundPosition := -1
+	for i, s := range ev.Shards {
+		if s.ShardId == shardId {
+			foundPosition = i
+		}
+	}
+	if foundPosition < 0 {
+		return nil, false
+	}
+
+	ecVolumeShard = ev.Shards[foundPosition]
+	ecVolumeShard.Unmount()
+	ev.Shards = append(ev.Shards[:foundPosition], ev.Shards[foundPosition+1:]...)
+	return ecVolumeShard, true
+}
+
+func (ev *EcVolume) FindEcVolumeShard(shardId ShardId) (ecVolumeShard *EcVolumeShard, found bool) {
+	for _, s := range ev.Shards {
+		if s.ShardId == shardId {
+			return s, true
+		}
+	}
+	return nil, false
+}
+
+func (ev *EcVolume) Close() {
+	for _, s := range ev.Shards {
+		s.Close()
+	}
+	ev.ecjFileAccessLock.Lock()
+	if ev.ecjFile != nil {
+		_ = ev.ecjFile.Close()
+		ev.ecjFile = nil
+	}
+	ev.ecjFileAccessLock.Unlock()
+	if ev.ecxFile != nil {
+		_ = ev.ecxFile.Sync()
+		// Do NOT nil ecxFile: LocateEcShardNeedle reads it without the
+		// ecVolumesLock after the resolving lookup released it, so a concurrent
+		// eviction that nils the field would race that read. A closed-but-set fd
+		// yields a clean read error (recovered from parity) and no data race.
+		_ = ev.ecxFile.Close()
+	}
+}
+
+// Sync flushes the .ecx and .ecj files to disk without closing them.
+// This ensures that deletions made via DeleteNeedleFromEcx are visible
+// to other processes/file handles that may read these files.
+func (ev *EcVolume) Sync() {
+	ev.ecjFileAccessLock.Lock()
+	if ev.ecjFile != nil {
+		if err := ev.ecjFile.Sync(); err != nil {
+			glog.Warningf("failed to sync ecj file for volume %d: %v", ev.VolumeId, err)
+		}
+	}
+	ev.ecjFileAccessLock.Unlock()
+	if ev.ecxFile != nil {
+		if err := ev.ecxFile.Sync(); err != nil {
+			glog.Warningf("failed to sync ecx file for volume %d: %v", ev.VolumeId, err)
+		}
+	}
+}
+
+func (ev *EcVolume) Destroy() {
+	ev.Close()
+
+	for _, s := range ev.Shards {
+		s.Destroy()
+	}
+	// Sweep the EC-only index files from BOTH the data directory and the shared
+	// index directory. A move or reconstruct can leave a copy in whichever
+	// directory is not ecxActualDir; removing only the active one leaves a stale
+	// index that a later reload could pick up and re-mount as a phantom EC
+	// volume. .ecx/.ecj are EC-specific, so removing both copies is safe.
+	for _, base := range ev.ecIndexBaseNames() {
+		os.Remove(base + ".ecx")
+		os.Remove(base + ".ecj")
+	}
+	// The .vif is shared with a coexisting normal volume (e.g. mid-decode), so
+	// only remove the active copy, not both.
+	os.Remove(ev.FileName(".vif"))
+	// Remove the bitrot checksum sidecar(s) so a later volume reuse cannot load
+	// stale protection. Search both the data and index bases.
+	RemoveBitrotSidecars(ev.DataBaseFileName())
+	if ev.IndexBaseFileName() != ev.DataBaseFileName() {
+		RemoveBitrotSidecars(ev.IndexBaseFileName())
+	}
+}
+
+// ecIndexBaseNames returns the base paths for the volume's EC index files in
+// both the data and index directories, deduplicated when they coincide.
+func (ev *EcVolume) ecIndexBaseNames() []string {
+	bases := []string{ev.DataBaseFileName()}
+	if ev.IndexBaseFileName() != ev.DataBaseFileName() {
+		bases = append(bases, ev.IndexBaseFileName())
+	}
+	return bases
+}
+
+// DiskType returns the disk type the EC volume currently reports under.
+// Defaults to the physical location's disk type; orchestrators can override
+// it via SetDiskType so the volume keeps reporting under the source
+// volume's disk type after encoding (#9423).
+func (ev *EcVolume) DiskType() types.DiskType {
+	return ev.diskType
+}
+
+// SetDiskType overrides the EC volume's reported disk type and propagates
+// to its mounted shards. Intended for the orchestrator-driven mount path
+// (VolumeEcShardsMount); not persisted across restarts.
+func (ev *EcVolume) SetDiskType(d types.DiskType) {
+	ev.diskType = d
+	for _, s := range ev.Shards {
+		s.DiskType = d
+	}
+}
+
+func (ev *EcVolume) FileName(ext string) string {
+	switch ext {
+	case ".ecx", ".ecj":
+		return EcShardFileName(ev.Collection, ev.ecxActualDir, int(ev.VolumeId)) + ext
+	}
+	// .vif
+	return ev.DataBaseFileName() + ext
+}
+
+func (ev *EcVolume) DataBaseFileName() string {
+	return EcShardFileName(ev.Collection, ev.dir, int(ev.VolumeId))
+}
+
+func (ev *EcVolume) IndexBaseFileName() string {
+	return EcShardFileName(ev.Collection, ev.dirIdx, int(ev.VolumeId))
+}
+
+func (ev *EcVolume) ShardSize() uint64 {
+	if len(ev.Shards) > 0 {
+		return uint64(ev.Shards[0].Size())
+	}
+	return 0
+}
+
+// DatFileSize returns the source .dat file size as recorded in .vif at
+// EC encoding time. Zero for old EC volumes whose .vif predates the
+// field, or for .vif files we failed to parse. Used by the Store-level
+// prune in store_ec_reconcile.go to validate that a sibling-disk .dat
+// is plausibly the encoding source before deleting the partial EC.
+func (ev *EcVolume) DatFileSize() int64 {
+	return ev.datFileSize
+}
+
+func (ev *EcVolume) Size() (size uint64) {
+	for _, shard := range ev.Shards {
+		if shardSize := shard.Size(); shardSize > 0 {
+			size += uint64(shardSize)
+		}
+	}
+	return
+}
+
+func (ev *EcVolume) CreatedAt() time.Time {
+	return ev.ecxCreatedAt
+}
+
+func (ev *EcVolume) ShardIdList() (shardIds []ShardId) {
+	for _, s := range ev.Shards {
+		shardIds = append(shardIds, s.ShardId)
+	}
+	return
+}
+
+func (ev *EcVolume) ToVolumeEcShardInformationMessage(diskId uint32) (messages []*master_pb.VolumeEcShardInformationMessage) {
+	ecInfoPerVolume := map[needle.VolumeId]*master_pb.VolumeEcShardInformationMessage{}
+
+	fileCount, deleteCount := ev.FileAndDeleteCount()
+
+	for _, s := range ev.Shards {
+		m, ok := ecInfoPerVolume[s.VolumeId]
+		if !ok {
+			m = &master_pb.VolumeEcShardInformationMessage{
+				Id:          uint32(s.VolumeId),
+				Collection:  s.Collection,
+				DiskType:    string(ev.diskType),
+				ExpireAtSec: ev.ExpireAtSec,
+				DiskId:      diskId,
+				FileCount:   fileCount,
+				DeleteCount: deleteCount,
+				EncodeTsNs:  ev.EncodeTsNs,
+			}
+			ecInfoPerVolume[s.VolumeId] = m
+		}
+
+		// Update EC shard bits and sizes.
+		si := ShardsInfoFromVolumeEcShardInformationMessage(m)
+		si.Set(NewShardInfo(s.ShardId, ShardSize(s.Size())))
+		m.EcIndexBits = uint32(si.Bitmap())
+		m.ShardSizes = si.SizesInt64()
+	}
+
+	for _, m := range ecInfoPerVolume {
+		messages = append(messages, m)
+	}
+	return
+}
+
+// FileAndDeleteCount returns the current (fileCount, deleteCount) for this
+// EC volume.
+//
+//   - fileCount = .ecx size / NeedleMapEntrySize — the total number of
+//     needles recorded in the sealed sorted index. Because .ecx is written
+//     at encode time and only overwritten during decode/rebuild (which
+//     preserves record count), this matches the "cumulative put count"
+//     semantics of regular volume FileCount.
+//
+//   - deleteCount = len(deletedNeedles) — the number of unique runtime
+//     deletes tracked in memory. The set is seeded from .ecj on load and
+//     appended to on every successful DeleteNeedleFromEcx. Because a
+//     needle delete is applied on exactly one shard holder, the admin
+//     aggregation sums deleteCount across nodes to get the volume's true
+//     delete total.
+//
+// Both values are O(1) — no index walking.
+func (ev *EcVolume) FileAndDeleteCount() (fileCount, deleteCount uint64) {
+	fileCount = uint64(ev.ecxFileSize) / uint64(types.NeedleMapEntrySize)
+	ev.deletedNeedlesLock.RLock()
+	deleteCount = uint64(len(ev.deletedNeedles))
+	ev.deletedNeedlesLock.RUnlock()
+	return
+}
+
+// IsNeedleDeleted reports whether the given needle id is in the in-memory
+// deleted set. Callers that have already looked the needle up in .ecx
+// should consult this to apply runtime deletion state on top of the
+// sealed index.
+func (ev *EcVolume) IsNeedleDeleted(needleId types.NeedleId) bool {
+	ev.deletedNeedlesLock.RLock()
+	_, ok := ev.deletedNeedles[needleId]
+	ev.deletedNeedlesLock.RUnlock()
+	return ok
+}
+
+// markNeedleDeletedInMemory inserts a needle id into the deleted set.
+func (ev *EcVolume) markNeedleDeletedInMemory(needleId types.NeedleId) {
+	ev.deletedNeedlesLock.Lock()
+	ev.deletedNeedles[needleId] = struct{}{}
+	ev.deletedNeedlesLock.Unlock()
+}
+
+// loadDeletedNeedlesFromEcj walks the .ecj journal and populates the
+// in-memory deleted set. Called once from NewEcVolume under the exclusive
+// ownership of the just-constructed (and not yet shared) EcVolume.
+func (ev *EcVolume) loadDeletedNeedlesFromEcj() error {
+	if ev.ecjFile == nil || ev.ecjFileSize < int64(types.NeedleIdSize) {
+		return nil
+	}
+	buf := make([]byte, types.NeedleIdSize)
+	for off := int64(0); off+int64(types.NeedleIdSize) <= ev.ecjFileSize; off += int64(types.NeedleIdSize) {
+		if _, err := ev.ecjFile.ReadAt(buf, off); err != nil {
+			return fmt.Errorf("read ecj at %d: %w", off, err)
+		}
+		id := types.BytesToNeedleId(buf)
+		ev.deletedNeedles[id] = struct{}{}
+	}
+	return nil
+}
+
+func (ev *EcVolume) LocateEcShardNeedle(needleId types.NeedleId, version needle.Version) (offset types.Offset, size types.Size, intervals []Interval, err error) {
+
+	// find the needle from ecx file
+	offset, size, err = ev.FindNeedleFromEcx(needleId)
+	if err != nil {
+		return types.Offset{}, 0, nil, fmt.Errorf("FindNeedleFromEcx: %w", err)
+	}
+
+	intervals = ev.LocateEcShardNeedleInterval(version, offset.ToActualOffset(), types.Size(needle.GetActualSize(size, version)))
+	return
+}
+
+func (ev *EcVolume) LocateEcShardNeedleInterval(version needle.Version, offset int64, size types.Size) (intervals []Interval) {
+	shard := ev.Shards[0]
+	var shardSize int64
+	if ev.datFileSize > 0 {
+		// Use datFileSize to calculate the shardSize to match the EC encoding logic.
+		// This is the authoritative value stored in .vif during EC encoding.
+		shardSize = ev.datFileSize / int64(ev.ECContext.DataShards)
+	} else {
+		// Fallback for old EC volumes without datFileSize in .vif.
+		// Subtract 1 to handle the ambiguous case where ecdFileSize is an exact
+		// multiple of ErasureCodingLargeBlockSize but the data is actually in small
+		// blocks (e.g., datFileSize was just under DataShards*ErasureCodingLargeBlockSize).
+		shardSize = shard.ecdFileSize - 1
+	}
+	// calculate the locations in the ec shards
+	intervals = LocateData(ev.ECContext.LargeBlockSize(), ev.ECContext.SmallBlockSize(), shardSize, offset, types.Size(needle.GetActualSize(size, version)))
+
+	return
+}
+
+// IntervalToShardIdAndOffset resolves an interval against this volume's shard
+// block layout.
+func (ev *EcVolume) IntervalToShardIdAndOffset(interval Interval) (ShardId, int64) {
+	return interval.ToShardIdAndOffset(ev.ECContext.LargeBlockSize(), ev.ECContext.SmallBlockSize())
+}
+
+func (ev *EcVolume) FindNeedleFromEcx(needleId types.NeedleId) (offset types.Offset, size types.Size, err error) {
+	offset, size, err = SearchNeedleFromSortedIndex(ev.ecxFile, ev.ecxFileSize, needleId, nil)
+	if err != nil {
+		ev.CheckReadWriteError(err)
+		return
+	}
+	ev.CheckReadWriteError(nil)
+	if ev.IsNeedleDeleted(needleId) {
+		size = types.TombstoneFileSize
+	}
+	return
+}
+
+func SearchNeedleFromSortedIndex(ecxFile *os.File, ecxFileSize int64, needleId types.NeedleId, processNeedleFn func(file *os.File, offset int64) error) (offset types.Offset, size types.Size, err error) {
+	var key types.NeedleId
+	buf := make([]byte, types.NeedleMapEntrySize)
+	l, h := int64(0), ecxFileSize/types.NeedleMapEntrySize
+	for l < h {
+		m := (l + h) / 2
+		if n, err := ecxFile.ReadAt(buf, m*types.NeedleMapEntrySize); err != nil {
+			if n != types.NeedleMapEntrySize {
+				return types.Offset{}, types.TombstoneFileSize, fmt.Errorf("ecx file %d read at %d: %w", ecxFileSize, m*types.NeedleMapEntrySize, err)
+			}
+		}
+		key, offset, size = idx.IdxFileEntry(buf)
+		if key == needleId {
+			if processNeedleFn != nil {
+				err = processNeedleFn(ecxFile, m*types.NeedleMapEntrySize)
+			}
+			return
+		}
+		if key < needleId {
+			l = m + 1
+		} else {
+			h = m
+		}
+	}
+
+	err = NotFoundError
+	return
+}
+
+func (ev *EcVolume) IsTimeToDestroy() bool {
+	return ev.ExpireAtSec > 0 && time.Now().Unix() > (int64(ev.ExpireAtSec)+destroyDelaySeconds)
+}
+
+func (ev *EcVolume) WalkIndex(processNeedleFn func(key types.NeedleId, offset types.Offset, size types.Size) error) error {
+	if ev.ecxFile == nil {
+		return fmt.Errorf("no ECX file associated with EC volume %v", ev.VolumeId)
+	}
+	return idx.WalkIndexFile(ev.ecxFile, 0, processNeedleFn)
+}

weed/storage/erasure_coding/ec_volume_counts_test.go

@@ -0,0 +1,125 @@
+package erasure_coding_test
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// makeEntry builds one .ecx index entry (needle id, offset, size).
+func makeEntry(key types.NeedleId, offset types.Offset, size types.Size) []byte {
+	b := make([]byte, types.NeedleIdSize+types.OffsetSize+types.SizeSize)
+	types.NeedleIdToBytes(b[0:types.NeedleIdSize], key)
+	types.OffsetToBytes(b[types.NeedleIdSize:types.NeedleIdSize+types.OffsetSize], offset)
+	types.SizeToBytes(b[types.NeedleIdSize+types.OffsetSize:], size)
+	return b
+}
+
+// encodeEcjIds packs a slice of needle ids into the binary .ecj on-disk format.
+func encodeEcjIds(ids []types.NeedleId) []byte {
+	buf := make([]byte, 0, len(ids)*types.NeedleIdSize)
+	for _, id := range ids {
+		b := make([]byte, types.NeedleIdSize)
+		types.NeedleIdToBytes(b, id)
+		buf = append(buf, b...)
+	}
+	return buf
+}
+
+func writeFixture(t *testing.T, dir, collection string, vid int, ecxData []byte, ecjIds []types.NeedleId) *erasure_coding.EcVolume {
+	t.Helper()
+	base := filepath.Join(dir, fmt.Sprintf("%s_%d", collection, vid))
+
+	if err := os.WriteFile(base+".ecx", ecxData, 0644); err != nil {
+		t.Fatalf("write ecx: %v", err)
+	}
+	if err := os.WriteFile(base+".ecj", encodeEcjIds(ecjIds), 0644); err != nil {
+		t.Fatalf("write ecj: %v", err)
+	}
+	if err := os.WriteFile(base+".ec00", make([]byte, 8), 0644); err != nil {
+		t.Fatalf("write ec00: %v", err)
+	}
+	if err := os.WriteFile(base+".vif", []byte{}, 0644); err != nil {
+		t.Fatalf("write vif: %v", err)
+	}
+
+	ev, err := erasure_coding.NewEcVolume("hdd", dir, dir, collection, needle.VolumeId(vid))
+	if err != nil {
+		t.Fatalf("NewEcVolume: %v", err)
+	}
+	return ev
+}
+
+// TestEcVolumeFileAndDeleteCountInitial verifies that FileAndDeleteCount is
+// derived from file sizes at load: fileCount = .ecx size / NeedleMapEntrySize
+// and deleteCount = .ecj size / NeedleIdSize. No index walk is performed.
+func TestEcVolumeFileAndDeleteCountInitial(t *testing.T) {
+	dir := t.TempDir()
+
+	// 3 needles in .ecx, 2 deletions already recorded in .ecj.
+	ecx := []byte{}
+	ecx = append(ecx, makeEntry(1, types.ToOffset(64), 100)...)
+	ecx = append(ecx, makeEntry(2, types.ToOffset(128), 200)...)
+	ecx = append(ecx, makeEntry(3, types.ToOffset(256), 300)...)
+	ecj := []types.NeedleId{2, 3}
+
+	ev := writeFixture(t, dir, "test", 1, ecx, ecj)
+	defer ev.Close()
+
+	fileCount, deleteCount := ev.FileAndDeleteCount()
+	if fileCount != 3 {
+		t.Errorf("fileCount: got %d, want 3 (.ecx entries)", fileCount)
+	}
+	if deleteCount != 2 {
+		t.Errorf("deleteCount: got %d, want 2 (.ecj entries)", deleteCount)
+	}
+}
+
+// TestEcVolumeFileAndDeleteCountAfterDelete verifies that DeleteNeedleFromEcx
+// increments the derived delete count by appending to .ecj, while leaving
+// fileCount (derived from the sealed .ecx) unchanged. Idempotent re-deletes
+// and deletes of missing needles must not drift the count.
+func TestEcVolumeFileAndDeleteCountAfterDelete(t *testing.T) {
+	dir := t.TempDir()
+
+	ecx := []byte{}
+	ecx = append(ecx, makeEntry(1, types.ToOffset(64), 100)...)
+	ecx = append(ecx, makeEntry(2, types.ToOffset(128), 200)...)
+
+	ev := writeFixture(t, dir, "test", 1, ecx, nil)
+	defer ev.Close()
+
+	if fc, dc := ev.FileAndDeleteCount(); fc != 2 || dc != 0 {
+		t.Fatalf("initial: got (%d, %d), want (2, 0)", fc, dc)
+	}
+
+	if err := ev.DeleteNeedleFromEcx(2); err != nil {
+		t.Fatalf("DeleteNeedleFromEcx: %v", err)
+	}
+	if fc, dc := ev.FileAndDeleteCount(); fc != 2 || dc != 1 {
+		t.Errorf("after first delete: got (%d, %d), want (2, 1)", fc, dc)
+	}
+
+	// Re-deleting an already tombstoned needle is a no-op on .ecj, so
+	// deleteCount must stay at 1.
+	if err := ev.DeleteNeedleFromEcx(2); err != nil {
+		t.Fatalf("idempotent DeleteNeedleFromEcx: %v", err)
+	}
+	if fc, dc := ev.FileAndDeleteCount(); fc != 2 || dc != 1 {
+		t.Errorf("after idempotent delete: got (%d, %d), want (2, 1)", fc, dc)
+	}
+
+	// Deleting a non-existent needle is a no-op: search returns NotFound,
+	// no .ecj append.
+	if err := ev.DeleteNeedleFromEcx(99); err != nil {
+		t.Fatalf("missing DeleteNeedleFromEcx: %v", err)
+	}
+	if fc, dc := ev.FileAndDeleteCount(); fc != 2 || dc != 1 {
+		t.Errorf("after missing delete: got (%d, %d), want (2, 1)", fc, dc)
+	}
+}

weed/storage/erasure_coding/ec_volume_delete.go

@@ -0,0 +1,166 @@
+package erasure_coding
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var (
+	MarkNeedleDeleted = func(file *os.File, offset int64) error {
+		b := make([]byte, types.SizeSize)
+		types.SizeToBytes(b, types.TombstoneFileSize)
+		n, err := file.WriteAt(b, offset+types.NeedleIdSize+types.OffsetSize)
+		if err != nil {
+			return fmt.Errorf("sorted needle write error: %w", err)
+		}
+		if n != types.SizeSize {
+			return fmt.Errorf("sorted needle written %d bytes, expecting %d", n, types.SizeSize)
+		}
+		return nil
+	}
+)
+
+// DeleteNeedleFromEcx marks the given needle as deleted. .ecx is treated
+// as an immutable sealed sorted index; runtime deletes are recorded by
+// appending the needle id to the .ecj deletion journal and inserting it
+// into the in-memory deletedNeedles set. A subsequent FindNeedleFromEcx
+// masks the id out by returning TombstoneFileSize.
+//
+// The .ecj append is the durable commit point — only after it syncs do
+// we publish the id into the in-memory set. A partial write is truncated
+// back to the known-good size so the on-disk journal and the set cannot
+// drift.
+func (ev *EcVolume) DeleteNeedleFromEcx(needleId types.NeedleId) (err error) {
+
+	// Look the needle up read-only. A missing needle is not an error
+	// (already gone, e.g. from a race against encode); a pre-existing
+	// .ecx tombstone means a prior decode/rebuild folded it in, in
+	// which case there is nothing to journal but we still mirror it
+	// into the in-memory set so delete_count stays consistent.
+	_, oldSize, err := SearchNeedleFromSortedIndex(ev.ecxFile, ev.ecxFileSize, needleId, nil)
+	if err != nil {
+		if err == NotFoundError {
+			return nil
+		}
+		return err
+	}
+	if oldSize.IsDeleted() {
+		ev.markNeedleDeletedInMemory(needleId)
+		return nil
+	}
+
+	// Serialise runtime deletes on ecjFileAccessLock so the idempotence
+	// check, the journal append and the set insertion happen atomically
+	// with respect to one another.
+	ev.ecjFileAccessLock.Lock()
+	defer ev.ecjFileAccessLock.Unlock()
+
+	if ev.IsNeedleDeleted(needleId) {
+		return nil
+	}
+
+	// Close nils ecjFile under this same lock, so a delete that resolved its
+	// .ecx lookup before an eviction (e.g. the generate-time UnloadEcVolume)
+	// can reach here with no journal fd. Bail with a clear error rather than
+	// operating on the closed/nil handle.
+	if ev.ecjFile == nil {
+		return fmt.Errorf("ec volume %d closed", ev.VolumeId)
+	}
+
+	b := make([]byte, types.NeedleIdSize)
+	types.NeedleIdToBytes(b, needleId)
+
+	prevEcjSize := ev.ecjFileSize
+	if _, seekErr := ev.ecjFile.Seek(0, io.SeekEnd); seekErr != nil {
+		return fmt.Errorf("seek ecj: %w", seekErr)
+	}
+	n, writeErr := ev.ecjFile.Write(b)
+	if writeErr != nil {
+		if truncErr := ev.ecjFile.Truncate(prevEcjSize); truncErr != nil {
+			glog.Errorf("ec volume %d: failed to truncate ecj after write error: %v", ev.VolumeId, truncErr)
+		}
+		return fmt.Errorf("write ecj: %w", writeErr)
+	}
+	if syncErr := ev.ecjFile.Sync(); syncErr != nil {
+		if truncErr := ev.ecjFile.Truncate(prevEcjSize); truncErr != nil {
+			glog.Errorf("ec volume %d: failed to truncate ecj after sync error: %v", ev.VolumeId, truncErr)
+		}
+		return fmt.Errorf("sync ecj: %w", syncErr)
+	}
+	ev.ecjFileSize += int64(n)
+
+	// Publish into the in-memory set only after the journal is durable.
+	ev.markNeedleDeletedInMemory(needleId)
+
+	return nil
+}
+
+func RebuildEcxFile(baseFileName string) error {
+
+	if !util.FileExists(baseFileName + ".ecj") {
+		return nil
+	}
+
+	ecxFile, err := os.OpenFile(baseFileName+".ecx", os.O_RDWR, 0644)
+	if err != nil {
+		return fmt.Errorf("rebuild: failed to open ecx file: %w", err)
+	}
+	defer ecxFile.Close()
+
+	fstat, err := ecxFile.Stat()
+	if err != nil {
+		return err
+	}
+
+	ecxFileSize := fstat.Size()
+
+	ecjFile, err := os.OpenFile(baseFileName+".ecj", os.O_RDWR, 0644)
+	if err != nil {
+		return fmt.Errorf("rebuild: failed to open ecj file: %w", err)
+	}
+	defer ecjFile.Close()
+
+	buf := make([]byte, types.NeedleIdSize)
+	for {
+		// io.ReadFull distinguishes a clean end (io.EOF) from a torn tail
+		// (io.ErrUnexpectedEOF) and a transient short read; a bare n!=size
+		// break would silently drop the rest of the journal and then unlink it.
+		_, readErr := io.ReadFull(ecjFile, buf)
+		if readErr == io.EOF {
+			break
+		}
+		if readErr != nil {
+			// Torn or unreadable journal: abort and leave .ecj in place so a
+			// retry can re-apply the deletions rather than resurrect them.
+			return fmt.Errorf("rebuild: read ecj: %w", readErr)
+		}
+
+		needleId := types.BytesToNeedleId(buf)
+
+		_, _, err = SearchNeedleFromSortedIndex(ecxFile, ecxFileSize, needleId, MarkNeedleDeleted)
+
+		if err != nil && err != NotFoundError {
+			return err
+		}
+
+	}
+
+	// Flush the in-place tombstones before removing the journal; otherwise a
+	// crash can persist the .ecj unlink ahead of the .ecx writes and resurrect
+	// the deleted needles on the next load.
+	if err = ecxFile.Sync(); err != nil {
+		return fmt.Errorf("rebuild: sync ecx: %w", err)
+	}
+
+	// Close the journal before unlinking it (Windows cannot delete an open
+	// file); the deferred Close becomes a harmless no-op.
+	ecjFile.Close()
+	os.Remove(baseFileName + ".ecj")
+
+	return nil
+}

weed/storage/erasure_coding/ec_volume_info.go

@@ -0,0 +1,69 @@
+package erasure_coding
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// data structure used in master
+type EcVolumeInfo struct {
+	VolumeId    needle.VolumeId
+	Collection  string
+	DiskType    string
+	DiskId      uint32 // ID of the disk this EC volume is on
+	ExpireAtSec uint64 // ec volume destroy time, calculated from the ec volume was created
+	ShardsInfo  *ShardsInfo
+	FileCount   uint64 // live needle count for this EC volume (same on every node holding shards)
+	DeleteCount uint64 // tombstoned needle count for this EC volume
+	EncodeTsNs  int64  // encode-run identity (unix nanos); one value per (volume, disk)
+}
+
+// DataShardsOrDefault returns how many of this volume's shards hold data; shard
+// ids below it are data, the rest are parity. Open-source SeaweedFS always uses
+// the fixed 10+4 layout, so this returns DataShardsCount. It is a per-volume
+// accessor, like EcShardsVolumeDataShards, so callers stay correct on builds
+// that derive the ratio per volume.
+func (ecInfo *EcVolumeInfo) DataShardsOrDefault() int {
+	return DataShardsCount
+}
+
+func (ecInfo *EcVolumeInfo) Minus(other *EcVolumeInfo) *EcVolumeInfo {
+	return &EcVolumeInfo{
+		VolumeId:    ecInfo.VolumeId,
+		Collection:  ecInfo.Collection,
+		ShardsInfo:  ecInfo.ShardsInfo.Minus(other.ShardsInfo),
+		DiskType:    ecInfo.DiskType,
+		DiskId:      ecInfo.DiskId,
+		ExpireAtSec: ecInfo.ExpireAtSec,
+		FileCount:   ecInfo.FileCount,
+		DeleteCount: ecInfo.DeleteCount,
+		EncodeTsNs:  ecInfo.EncodeTsNs,
+	}
+}
+
+func (evi *EcVolumeInfo) ToVolumeEcShardInformationMessage() (ret *master_pb.VolumeEcShardInformationMessage) {
+	return &master_pb.VolumeEcShardInformationMessage{
+		Id:          uint32(evi.VolumeId),
+		EcIndexBits: evi.ShardsInfo.Bitmap(),
+		ShardSizes:  evi.ShardsInfo.SizesInt64(),
+		Collection:  evi.Collection,
+		DiskType:    evi.DiskType,
+		ExpireAtSec: evi.ExpireAtSec,
+		DiskId:      evi.DiskId,
+		FileCount:   evi.FileCount,
+		DeleteCount: evi.DeleteCount,
+		EncodeTsNs:  evi.EncodeTsNs,
+	}
+}
+
+func (evi *EcVolumeInfo) GetCollection() string {
+	return evi.Collection
+}
+
+func (evi *EcVolumeInfo) GetVolumeId() needle.VolumeId {
+	return evi.VolumeId
+}
+
+func (evi *EcVolumeInfo) GetRemoteStorageName() string {
+	return ""
+}

weed/storage/erasure_coding/ec_volume_info_test.go

@@ -0,0 +1,173 @@
+package erasure_coding_test
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	erasure_coding "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+func TestShardsInfoDeleteParityShards(t *testing.T) {
+	si := erasure_coding.NewShardsInfo()
+	for _, id := range erasure_coding.AllShardIds() {
+		si.Set(erasure_coding.ShardInfo{Id: id, Size: 123})
+	}
+	si.DeleteParityShards(erasure_coding.DataShardsCount)
+
+	if got, want := si.String(), "0:123 B 1:123 B 2:123 B 3:123 B 4:123 B 5:123 B 6:123 B 7:123 B 8:123 B 9:123 B"; got != want {
+		t.Errorf("expected %q, got %q", want, got)
+	}
+
+}
+
+func TestShardsInfoAsSlice(t *testing.T) {
+	si := erasure_coding.NewShardsInfo()
+	si.Set(erasure_coding.ShardInfo{Id: 5, Size: 555})
+	si.Set(erasure_coding.ShardInfo{Id: 2, Size: 222})
+	si.Set(erasure_coding.ShardInfo{Id: 7, Size: 777})
+	si.Set(erasure_coding.ShardInfo{Id: 1, Size: 111})
+
+	want := []erasure_coding.ShardInfo{
+		{Id: 1, Size: 111},
+		{Id: 2, Size: 222},
+		{Id: 5, Size: 555},
+		{Id: 7, Size: 777},
+	}
+	if got := si.AsSlice(); !reflect.DeepEqual(got, want) {
+		t.Errorf("expected %v, got %v", want, got)
+	}
+}
+
+func TestShardsInfoSerialize(t *testing.T) {
+	testCases := []struct {
+		name      string
+		shardIds  map[erasure_coding.ShardId]erasure_coding.ShardSize
+		wantBits  uint32
+		wantSizes []erasure_coding.ShardSize
+	}{
+		{
+			name:      "no bits",
+			shardIds:  nil,
+			wantBits:  0b0,
+			wantSizes: []erasure_coding.ShardSize{},
+		},
+		{
+			name: "single shard, first",
+			shardIds: map[erasure_coding.ShardId]erasure_coding.ShardSize{
+				0: 2345,
+			},
+			wantBits:  0b1,
+			wantSizes: []erasure_coding.ShardSize{2345},
+		},
+		{
+			name: "single shard, 5th",
+			shardIds: map[erasure_coding.ShardId]erasure_coding.ShardSize{
+				4: 6789,
+			},
+			wantBits:  0b10000,
+			wantSizes: []erasure_coding.ShardSize{6789},
+		},
+		{
+			name: "multiple shards",
+			shardIds: map[erasure_coding.ShardId]erasure_coding.ShardSize{
+				8: 800,
+				0: 5,
+				3: 300,
+				1: 100,
+			},
+			wantBits:  0b100001011,
+			wantSizes: []erasure_coding.ShardSize{5, 100, 300, 800},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			si := erasure_coding.NewShardsInfo()
+			for id, size := range tc.shardIds {
+				si.Set(erasure_coding.ShardInfo{Id: id, Size: size})
+			}
+
+			if got, want := si.Bitmap(), tc.wantBits; got != want {
+				t.Errorf("expected bits %v, got %v", want, got)
+			}
+			if got, want := si.Sizes(), tc.wantSizes; !reflect.DeepEqual(got, want) {
+				t.Errorf("expected sizes %v, got %v", want, got)
+			}
+		})
+	}
+}
+
+func TestShardsInfoFromVolumeEcShardInformationMessage(t *testing.T) {
+	testCases := []struct {
+		name    string
+		ecvInfo *master_pb.VolumeEcShardInformationMessage
+		want    string
+	}{
+		{
+			name:    "no msg",
+			ecvInfo: nil,
+			want:    "",
+		},
+		{
+			name:    "no shards",
+			ecvInfo: &master_pb.VolumeEcShardInformationMessage{},
+			want:    "",
+		},
+		{
+			name: "single shard",
+			ecvInfo: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: 0b100,
+				ShardSizes:  []int64{333},
+			},
+			want: "2:333 B",
+		},
+		{
+			name: "multiple shards",
+			ecvInfo: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: 0b1101,
+				ShardSizes:  []int64{111, 333, 444},
+			},
+			want: "0:111 B 2:333 B 3:444 B",
+		},
+		{
+			name: "multiple shards with missing sizes",
+			ecvInfo: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: 0b110110,
+				ShardSizes:  []int64{111, 333, 444},
+			},
+			want: "1:111 B 2:333 B 4:444 B 5:0 B",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(tc.ecvInfo)
+			if got, want := si.String(), tc.want; got != want {
+				t.Errorf("expected %q, got %q", want, got)
+			}
+		})
+	}
+}
+
+func TestShardsInfoCombine(t *testing.T) {
+	a := erasure_coding.NewShardsInfo()
+	a.Set(erasure_coding.ShardInfo{Id: 1, Size: 111})
+	a.Set(erasure_coding.ShardInfo{Id: 2, Size: 222})
+	a.Set(erasure_coding.ShardInfo{Id: 3, Size: 333})
+	a.Set(erasure_coding.ShardInfo{Id: 4, Size: 444})
+	a.Set(erasure_coding.ShardInfo{Id: 5, Size: 0})
+
+	b := erasure_coding.NewShardsInfo()
+	b.Set(erasure_coding.ShardInfo{Id: 1, Size: 555})
+	b.Set(erasure_coding.ShardInfo{Id: 4, Size: 666})
+	b.Set(erasure_coding.ShardInfo{Id: 5, Size: 777})
+	b.Set(erasure_coding.ShardInfo{Id: 6, Size: 888})
+
+	if got, want := a.Plus(b).String(), "1:555 B 2:222 B 3:333 B 4:666 B 5:777 B 6:888 B"; got != want {
+		t.Errorf("expected %q for plus, got %q", want, got)
+	}
+	if got, want := a.Minus(b).String(), "2:222 B 3:333 B"; got != want {
+		t.Errorf("expected %q for minus, got %q", want, got)
+	}
+}

weed/storage/erasure_coding/ec_volume_merge.go

@@ -0,0 +1,301 @@
+package erasure_coding
+
+import (
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+// MergedEcRuntimes resolves one volume id's per-disk runtimes into a single
+// scrubbable view. Every scrub mode builds from this, so there is exactly one
+// answer to "which disks count" per volume.
+//
+// Two fences admit a runtime into Merged: the anchor's EncodeTsNs (the
+// maximum, so 0 implies every runtime is 0 and nothing is excluded — legacy
+// leniency), and the anchor's geometry (DataShards, ParityShards, BlockSize).
+// Equal timestamps do not guarantee equal layouts, so a same-generation
+// runtime whose .vif disagrees is excluded and reported rather than merged
+// into a plan that would apply the anchor's offsets and checksums to
+// incompatible shards.
+type MergedEcRuntimes struct {
+	// Anchor is the volume-level metadata source: geometry, .ecx handles,
+	// version. It is NOT the bitrot protection source — the .ecsum sidecar is
+	// per-DISK state, so ChecksumScrubMerged sources it from the first Merged
+	// runtime that has any.
+	Anchor *EcVolume
+	// Merged holds the runtimes whose shards are safe to verify together.
+	Merged []*EcVolume
+	// Slots is indexed BY SHARD ID. A nil entry is a shard no merged runtime
+	// holds. The volume's shard-id range is the anchor's geometry
+	// (0..DataShards+ParityShards); consumers truncate to it.
+	Slots []*EcVolumeShard
+	// Skipped holds one line per runtime excluded by the identity or geometry
+	// fence. Reported, never dropped.
+	Skipped []string
+}
+
+// MergeEcRuntimes resolves runtimes (one vid's mounts, in location order) into
+// a merged view. Returns nil for an empty slice (the vanished-volume case).
+func MergeEcRuntimes(runtimes []*EcVolume) *MergedEcRuntimes {
+	if len(runtimes) == 0 {
+		return nil
+	}
+
+	anchorGen := int64(0)
+	for _, v := range runtimes {
+		if v.EncodeTsNs > anchorGen {
+			anchorGen = v.EncodeTsNs
+		}
+	}
+
+	var genMatches []*EcVolume
+	for _, v := range runtimes {
+		if v.EncodeTsNs == anchorGen {
+			genMatches = append(genMatches, v)
+		}
+	}
+
+	anchor := genMatches[0]
+	for _, v := range genMatches {
+		if len(v.Shards) > 0 {
+			anchor = v
+			break
+		}
+	}
+
+	var merged []*EcVolume
+	for _, v := range genMatches {
+		if geometryMatches(v, anchor) {
+			merged = append(merged, v)
+		}
+	}
+
+	total := ecDataShards(anchor) + ecParityShards(anchor)
+	width := total
+	for _, v := range merged {
+		if n := int(v.Shards[len(v.Shards)-1].ShardId) + 1; n > width {
+			width = n
+		}
+	}
+	slots := make([]*EcVolumeShard, width)
+	for _, v := range merged {
+		for _, shard := range v.Shards {
+			id := int(shard.ShardId)
+			if id >= len(slots) {
+				continue
+			}
+			if slots[id] == nil {
+				slots[id] = shard
+			}
+		}
+	}
+
+	var skipped []string
+	for pos, v := range runtimes {
+		if v.EncodeTsNs != anchorGen {
+			skipped = append(skipped, fmt.Sprintf(
+				"EC volume %d shards at %s (position %d) belong to encode run %d but the scrub anchors on %d; they were not verified",
+				v.VolumeId, v.dir, pos, v.EncodeTsNs, anchorGen))
+			continue
+		}
+		if !geometryMatches(v, anchor) {
+			skipped = append(skipped, fmt.Sprintf(
+				"EC volume %d shards at %s (position %d) share encode run %d but disagree on geometry (%d+%d bs %d vs %d+%d bs %d); they were not verified",
+				v.VolumeId, v.dir, pos, v.EncodeTsNs,
+				ecDataShards(v), ecParityShards(v), ecBlockSize(v),
+				ecDataShards(anchor), ecParityShards(anchor), ecBlockSize(anchor)))
+		}
+	}
+
+	return &MergedEcRuntimes{
+		Anchor:  anchor,
+		Merged:  merged,
+		Slots:   slots,
+		Skipped: skipped,
+	}
+}
+
+func geometryMatches(a, b *EcVolume) bool {
+	return ecDataShards(a) == ecDataShards(b) &&
+		ecParityShards(a) == ecParityShards(b) &&
+		ecBlockSize(a) == ecBlockSize(b)
+}
+
+func ecDataShards(v *EcVolume) int {
+	if v.ECContext != nil {
+		return v.ECContext.DataShards
+	}
+	return 0
+}
+
+func ecParityShards(v *EcVolume) int {
+	if v.ECContext != nil {
+		return v.ECContext.ParityShards
+	}
+	return 0
+}
+
+func ecBlockSize(v *EcVolume) int64 {
+	if v.ECContext != nil {
+		return v.ECContext.BlockSize
+	}
+	return 0
+}
+
+// asVolume returns a synthetic EcVolume that shares the anchor's volume-level
+// state (ecx handles, version, geometry, bitrot protection) but presents the
+// merged shard set. ScrubLocal/ChecksumScrub read Shards, FindEcVolumeShard,
+// ECContext, ecxFile, BitrotProtection and Version — all of which the anchor
+// supplies except Shards, which is rebuilt from the merged slots.
+//
+// For legacy volumes (no datFileSize in .vif), LocateEcShardNeedleInterval
+// derives the shard size from Shards[0].ecdFileSize. The merged shard set is
+// compacted in shard-ID order, so a truncated lowest-ID shard would shrink
+// every interval and misread intact sibling shards. To prevent that, asVolume
+// synthesizes a datFileSize from the maximum mounted shard size when the
+// anchor lacks one, so the datFileSize>0 path in LocateEcShardNeedleInterval
+// uses the largest shard's size across all merged runtimes.
+func (m *MergedEcRuntimes) asVolume() *EcVolume {
+	anchor := m.Anchor
+	shards := make([]*EcVolumeShard, 0, len(m.Slots))
+	for _, s := range m.Slots {
+		if s != nil {
+			shards = append(shards, s)
+		}
+	}
+	datFileSize := anchor.datFileSize
+	if datFileSize == 0 && anchor.ECContext != nil && anchor.ECContext.DataShards > 0 {
+		var maxShardSize int64
+		for _, s := range shards {
+			if s.ecdFileSize > maxShardSize {
+				maxShardSize = s.ecdFileSize
+			}
+		}
+		// Subtract 1 to match the legacy fallback in LocateEcShardNeedleInterval
+		// (ecdFileSize - 1): an exact large-block boundary is ambiguous, and
+		// the unadjusted size would select an extra large row.
+		if maxShardSize > 0 {
+			datFileSize = (maxShardSize - 1) * int64(anchor.ECContext.DataShards)
+		}
+	}
+	return &EcVolume{
+		VolumeId:     anchor.VolumeId,
+		Collection:   anchor.Collection,
+		dir:          anchor.dir,
+		dirIdx:       anchor.dirIdx,
+		ecxActualDir: anchor.ecxActualDir,
+		ecxFile:      anchor.ecxFile,
+		ecxFileSize:  anchor.ecxFileSize,
+		ecxCreatedAt: anchor.ecxCreatedAt,
+		Shards:       shards,
+		Version:      anchor.Version,
+		diskType:     anchor.diskType,
+		datFileSize:  datFileSize,
+		ECContext:    anchor.ECContext,
+		EncodeTsNs:   anchor.EncodeTsNs,
+		bitrot:       anchor.bitrot,
+		bitrotStatus: anchor.bitrotStatus,
+	}
+}
+
+// ScrubLocal checks the integrity of local shards across every merged
+// runtime, mirroring EcVolume.ScrubLocal over the merged shard set. Skipped
+// runtimes are reported alongside any scrub errors.
+func (m *MergedEcRuntimes) ScrubLocal() (int64, []*volume_server_pb.EcShardInfo, []error) {
+	files, shardInfos, errs := m.asVolume().ScrubLocal()
+	for _, s := range m.Skipped {
+		errs = append(errs, fmt.Errorf("%s", s))
+	}
+	return files, shardInfos, errs
+}
+
+// ChecksumScrub verifies every merged runtime's local shards against the
+// bitrot sidecar, mirroring EcVolume.ChecksumScrub. The sidecar is per-DISK
+// state: protection is taken from the first merged runtime that has any (On,
+// else Invalid, else the anchor's Off), and every merged runtime whose
+// sidecar resolved Invalid is reported even when protection is taken from a
+// sibling that is On. Skipped runtimes are reported alongside any scrub
+// errors.
+func (m *MergedEcRuntimes) ChecksumScrub() (int64, []*volume_server_pb.EcShardInfo, []error) {
+	// Pick the protection source: first On, else first Invalid, else anchor.
+	var protectionSource *EcVolume
+	for _, v := range m.Merged {
+		if _, status := v.BitrotProtection(); status == BitrotOn {
+			protectionSource = v
+			break
+		}
+	}
+	if protectionSource == nil {
+		for _, v := range m.Merged {
+			if _, status := v.BitrotProtection(); status == BitrotInvalid {
+				protectionSource = v
+				break
+			}
+		}
+	}
+	if protectionSource == nil {
+		protectionSource = m.Anchor
+	}
+
+	prot, status := protectionSource.BitrotProtection()
+
+	// Fence the sidecar's encode generation: a merged runtime can load a
+	// sidecar from a sibling metadata directory (ReloadBitrotSidecar), and
+	// the merge fence may then exclude the runtime owning that directory.
+	// Generation-0 sidecars do not identify the encode run, so geometry
+	// validation alone cannot prove the borrowed manifest describes the
+	// anchor's shards. If the sidecar records a non-zero EncodeTsNs that
+	// disagrees with the anchor's, scanning would apply stale checksums to
+	// current shards and report false corruption. Refuse instead.
+	if status == BitrotOn && prot != nil && prot.EcShardConfig != nil {
+		sidecarGen := prot.EcShardConfig.EncodeTsNs
+		if sidecarGen != 0 && m.Anchor.EncodeTsNs != 0 && sidecarGen != m.Anchor.EncodeTsNs {
+			errs := []error{fmt.Errorf(
+				"ec volume %d: bitrot sidecar at %s records encode run %d but the scrub anchors on %d; protection is unverifiable",
+				m.Anchor.VolumeId, protectionSource.dir, sidecarGen, m.Anchor.EncodeTsNs)}
+			for _, rt := range m.Merged {
+				if rt == protectionSource {
+					continue
+				}
+				if _, s := rt.BitrotProtection(); s == BitrotInvalid {
+					errs = append(errs, fmt.Errorf(
+						"ec volume %d bitrot sidecar at %s is malformed/unverifiable (sidecar integrity)",
+						rt.VolumeId, rt.dir))
+				}
+			}
+			for _, s := range m.Skipped {
+				errs = append(errs, fmt.Errorf("%s", s))
+			}
+			return 0, nil, errs
+		}
+	}
+
+	// Run the byte scan against the protection source's sidecar, but over
+	// the merged shard set. The synthetic volume inherits the protection
+	// source's bitrot state so ChecksumScrub's BitrotOff/Invalid arms fire.
+	v := m.asVolume()
+	v.bitrot = prot
+	v.bitrotStatus = status
+
+	blocks, broken, errs := v.ChecksumScrub()
+
+	// Collect Invalid-sidecar errors from every merged runtime except the
+	// protection source (whose error the Invalid arm already reports), so a
+	// malformed sidecar is never silently discarded when a sibling is On.
+	if status != BitrotInvalid {
+		for _, rt := range m.Merged {
+			if rt == protectionSource {
+				continue
+			}
+			if _, s := rt.BitrotProtection(); s == BitrotInvalid {
+				errs = append(errs, fmt.Errorf(
+					"ec volume %d bitrot sidecar at %s is malformed/unverifiable (sidecar integrity)",
+					rt.VolumeId, rt.dir))
+			}
+		}
+	}
+	for _, s := range m.Skipped {
+		errs = append(errs, fmt.Errorf("%s", s))
+	}
+	return blocks, broken, errs
+}

weed/storage/erasure_coding/ec_volume_scrub.go

@@ -0,0 +1,315 @@
+package erasure_coding
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"slices"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// ScrubIndex verifies index integrity of an EC volume.
+func (ev *EcVolume) ScrubIndex() (int64, []error) {
+	if ev.ecxFile == nil {
+		return 0, []error{fmt.Errorf("no ECX file associated with EC volume %v", ev.VolumeId)}
+	}
+	if ev.ecxFileSize == 0 {
+		return 0, []error{fmt.Errorf("zero-size ECX file for EC volume %v", ev.VolumeId)}
+	}
+
+	return idx.CheckIndexFile(ev.ecxFile, ev.ecxFileSize, ev.Version)
+}
+
+// ChecksumScrub verifies every locally-held EC shard's raw bytes against the
+// active-generation bitrot checksum sidecar. It is read-only: it detects and
+// reports corruption but never mutates or quarantines anything (the apply path
+// drives quarantine + rebuild). It is the only path that exercises cold parity
+// shards, which are never read during normal serving.
+//
+// Returns the number of blocks scanned, the broken shards (sidecar mismatch or
+// length drift), and errors. A wholesale mismatch (more than parity_shards
+// shards differing) is reported as a suspect/stale sidecar — an integrity error
+// — rather than flagging the shards as corrupt, so a bad sidecar can never be
+// mistaken for mass bitrot.
+func (ecv *EcVolume) ChecksumScrub() (blocksScanned int64, brokenShards []*volume_server_pb.EcShardInfo, errs []error) {
+	prot, status := ecv.BitrotProtection()
+	switch status {
+	case BitrotOff:
+		// Unprotected generation: nothing to verify. Not an error.
+		return 0, nil, nil
+	case BitrotInvalid:
+		return 0, nil, []error{fmt.Errorf("ec volume %d: bitrot sidecar is malformed/unverifiable (sidecar integrity)", ecv.VolumeId)}
+	}
+
+	blockSize := int64(prot.BlockSize)
+	var broken []*EcVolumeShard
+	for _, shard := range ecv.Shards {
+		entry := shardChecksums(prot, uint32(shard.ShardId))
+		if entry == nil {
+			errs = append(errs, fmt.Errorf("ec volume %d: no checksum entry for local shard %d", ecv.VolumeId, shard.ShardId))
+			continue
+		}
+		if shard.Size() != entry.CoveredSize {
+			errs = append(errs, fmt.Errorf("ec volume %d shard %d: size %d != covered_size %d (truncated or extended)",
+				ecv.VolumeId, shard.ShardId, shard.Size(), entry.CoveredSize))
+			broken = append(broken, shard)
+			continue
+		}
+		want := unpackUint32LE(entry.BlockCrc32C)
+		buf := make([]byte, blockSize)
+		var offset int64
+		shardBad := false
+		for i := 0; i < len(want); i++ {
+			toRead := blockSize
+			if rem := entry.CoveredSize - offset; rem < toRead {
+				toRead = rem
+			}
+			n, rerr := shard.ReadAt(buf[:toRead], offset)
+			if rerr != nil || int64(n) != toRead {
+				errs = append(errs, fmt.Errorf("ec volume %d shard %d: read block %d at %d: %v (got %d/%d)",
+					ecv.VolumeId, shard.ShardId, i, offset, rerr, n, toRead))
+				shardBad = true
+				break
+			}
+			if uint32(needle.NewCRC(buf[:n])) != want[i] {
+				errs = append(errs, fmt.Errorf("ec volume %d shard %d: checksum mismatch at block %d (offset %d)",
+					ecv.VolumeId, shard.ShardId, i, offset))
+				shardBad = true
+			}
+			offset += int64(n)
+			blocksScanned++
+		}
+		if shardBad {
+			broken = append(broken, shard)
+		}
+	}
+
+	// Wholesale-mismatch guard: localized bitrot touches a few shards; a
+	// stale/wrong sidecar mismatches more than parity_shards. Classify the
+	// latter as sidecar integrity, never as mass shard corruption.
+	if ecv.ECContext != nil && len(broken) > ecv.ECContext.ParityShards {
+		return blocksScanned, nil, []error{fmt.Errorf("ec volume %d: %d/%d local shards mismatch (> parity %d): suspect stale sidecar, not flagging shard corruption",
+			ecv.VolumeId, len(broken), len(ecv.Shards), ecv.ECContext.ParityShards)}
+	}
+
+	// Reed-Solomon arbitration: a sidecar mismatch could mean the shard rotted
+	// OR the sidecar block is stale. When enough OTHER shards are local to
+	// reconstruct, reconstruct each flagged shard from the verified-clean shards
+	// and compare to its on-disk bytes. Only shards RS *also* disagrees with are
+	// truly corrupt; the rest are stale-sidecar false positives we must not
+	// quarantine. This is the arbiter that makes auto-repair safe.
+	if ecv.ECContext != nil && len(broken) > 0 {
+		brokenSet := make(map[ShardId]bool, len(broken))
+		for _, s := range broken {
+			brokenSet[s.ShardId] = true
+		}
+		cleanCount := 0
+		for _, s := range ecv.Shards {
+			if !brokenSet[s.ShardId] {
+				cleanCount++
+			}
+		}
+		if cleanCount >= ecv.ECContext.DataShards {
+			confirmed := make([]*EcVolumeShard, 0, len(broken))
+			for _, s := range broken {
+				corrupt, aerr := ecv.rsConfirmsShardCorrupt(s.ShardId, brokenSet, blockSize)
+				if aerr != nil {
+					// Could not arbitrate this shard; stay conservative and keep it.
+					errs = append(errs, fmt.Errorf("ec volume %d shard %d: RS arbitration failed: %v", ecv.VolumeId, s.ShardId, aerr))
+					confirmed = append(confirmed, s)
+					continue
+				}
+				if corrupt {
+					confirmed = append(confirmed, s)
+				} else {
+					errs = append(errs, fmt.Errorf("ec volume %d shard %d: sidecar mismatch but Reed-Solomon confirms the bytes are correct: stale sidecar, not flagging shard",
+						ecv.VolumeId, s.ShardId))
+				}
+			}
+			broken = confirmed
+		}
+	}
+
+	for _, s := range broken {
+		brokenShards = append(brokenShards, s.ToEcShardInfo())
+	}
+	slices.SortFunc(brokenShards, func(a, b *volume_server_pb.EcShardInfo) int {
+		return int(a.ShardId) - int(b.ShardId)
+	})
+	return blocksScanned, brokenShards, errs
+}
+
+// rsConfirmsShardCorrupt reconstructs targetId from the verified-clean local
+// shards (all local shards except those in brokenSet) and compares the result
+// to targetId's on-disk bytes block-by-block. It returns true only if
+// Reed-Solomon disagrees with the disk (the shard is genuinely corrupt); a full
+// match means the shard is fine and its sidecar block is stale. The caller
+// guarantees at least DataShards clean shards are local before invoking this.
+func (ecv *EcVolume) rsConfirmsShardCorrupt(targetId ShardId, brokenSet map[ShardId]bool, blockSize int64) (bool, error) {
+	enc, err := ecv.ECContext.CreateEncoder()
+	if err != nil {
+		return false, err
+	}
+	total := ecv.ECContext.Total()
+	local := make(map[ShardId]*EcVolumeShard, len(ecv.Shards))
+	for _, s := range ecv.Shards {
+		local[s.ShardId] = s
+	}
+	target := local[targetId]
+	if target == nil {
+		return false, fmt.Errorf("target shard not local")
+	}
+	size := target.Size()
+
+	buffers := make([][]byte, total)
+	diskBuf := make([]byte, blockSize)
+	var offset int64
+	for offset < size {
+		n := blockSize
+		if rem := size - offset; rem < n {
+			n = rem
+		}
+		// Fill clean local shards; nil for the target and any broken/absent shard
+		// so Reconstruct regenerates the target from the trusted inputs.
+		for id := 0; id < total; id++ {
+			sid := ShardId(id)
+			if sid == targetId || brokenSet[sid] {
+				buffers[id] = nil
+				continue
+			}
+			s := local[sid]
+			if s == nil {
+				buffers[id] = nil
+				continue
+			}
+			buf := make([]byte, n)
+			got, rerr := s.ReadAt(buf, offset)
+			if rerr != nil || int64(got) != n {
+				return false, fmt.Errorf("read clean shard %d at %d: %v (got %d/%d)", sid, offset, rerr, got, n)
+			}
+			buffers[id] = buf
+		}
+		if err := enc.Reconstruct(buffers); err != nil {
+			return false, fmt.Errorf("reconstruct: %w", err)
+		}
+		got, rerr := target.ReadAt(diskBuf[:n], offset)
+		if rerr != nil || int64(got) != n {
+			return false, fmt.Errorf("read target shard %d at %d: %v", targetId, offset, rerr)
+		}
+		if !bytes.Equal(buffers[targetId][:n], diskBuf[:n]) {
+			return true, nil // Reed-Solomon disagrees with disk: genuinely corrupt.
+		}
+		offset += n
+	}
+	return false, nil // every block matches: shard is fine, sidecar block is stale.
+}
+
+// ScrubLocal checks the integrity of local shards for a EC volume. Notably, it cannot verify CRC on needles.
+// Returns a count of processed file entries, slice of found broken shards, and slice of found errors.
+func (ecv *EcVolume) ScrubLocal() (int64, []*volume_server_pb.EcShardInfo, []error) {
+	// local scan means verifying indexes as well
+	_, errs := ecv.ScrubIndex()
+
+	brokenShardsMap := map[ShardId]*EcVolumeShard{}
+	var count int64
+
+	flagShardBroken := func(ecs *EcVolumeShard, errFmt string, a ...any) {
+		// reads for EC chunks can hit the same shard multiple times, so dedupe upon read errors
+		brokenShardsMap[ecs.ShardId] = ecs
+		errs = append(errs, fmt.Errorf(errFmt, a...))
+	}
+
+	err := idx.WalkIndexFile(ecv.ecxFile, 0, func(id types.NeedleId, offset types.Offset, size types.Size) error {
+		count++
+		if size.IsTombstone() {
+			// nothing to do for tombstones...
+			return nil
+		}
+
+		var read int64
+		var hasRemoteChunks bool
+		var data []byte
+
+		locations := ecv.LocateEcShardNeedleInterval(ecv.Version, offset.ToActualOffset(), size)
+		localShardIds := []ShardId{}
+
+		for i, iv := range locations {
+			sid, soffset := ecv.IntervalToShardIdAndOffset(iv)
+			ssize := int64(iv.Size.Raw())
+			shard, found := ecv.FindEcVolumeShard(sid)
+
+			if !found {
+				// shard is not local :( skip it
+				hasRemoteChunks = true
+				read += ssize
+				continue
+			}
+			localShardIds = append(localShardIds, sid)
+
+			if soffset+int64(ssize) > shard.Size() {
+				flagShardBroken(shard, "local shard %d for needle %d is too short (%d), cannot read chunk %d/%d", sid, id, shard.Size(), i+1, len(locations))
+				continue
+			}
+
+			chunk := make([]byte, ssize)
+			got, err := shard.ReadAt(chunk, soffset)
+			if err != nil {
+				flagShardBroken(shard, "failed to read chunk %d/%d for needle %d from local shard %d at offset %d: %v", i+1, len(locations), id, sid, soffset, err)
+				continue
+			}
+			if int64(got) != ssize {
+				flagShardBroken(shard, "expected %d bytes for chunk %d/%d for needle %d from local shard %d, got %d", ssize, i+1, len(locations), id, sid, got)
+				continue
+			}
+
+			if !hasRemoteChunks {
+				data = append(data, chunk...)
+			}
+			read += int64(got)
+		}
+
+		slices.Sort(localShardIds)
+
+		if got, want := read, needle.GetActualSize(size, ecv.Version); got != want {
+			return fmt.Errorf("expected %d bytes for needle %d on volume %d, got %d", want, id, ecv.VolumeId, got)
+		}
+		if !hasRemoteChunks {
+			// needle was fully recovered from local shards \o/ let's check it
+			n := needle.Needle{}
+			if err := n.ReadBytes(data, 0, size, ecv.Version); err != nil {
+				// A delete-state disagreement between the .ecx index and the reassembled
+				// on-disk header (live index vs zero header size) is not corruption.
+				deleteStateDisagrees := errors.Is(err, needle.ErrorSizeMismatch) && size.IsDeleted() != (n.Size == 0)
+				if !deleteStateDisagrees {
+					errs = append(errs, fmt.Errorf("needle %d on volume %d, shards %v: %v", id, ecv.VolumeId, localShardIds, err))
+				}
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		errs = append(errs, err)
+	}
+
+	// collect broken shard infos for reporting
+	brokenShards := make([]*volume_server_pb.EcShardInfo, 0, len(brokenShardsMap))
+	for _, s := range brokenShardsMap {
+		brokenShards = append(brokenShards, s.ToEcShardInfo())
+	}
+	slices.SortFunc(brokenShards, func(a, b *volume_server_pb.EcShardInfo) int {
+		if a.ShardId < b.ShardId {
+			return -1
+		}
+		if a.ShardId > b.ShardId {
+			return 1
+		}
+		return 0
+	})
+
+	return count, brokenShards, errs
+}

weed/storage/erasure_coding/ec_volume_sidecar_layout_test.go

@@ -0,0 +1,246 @@
+package erasure_coding
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+)
+
+func uniformSidecar(t *testing.T, base string, ds, ps uint32, blockSize int64) {
+	t.Helper()
+	prot := &volume_server_pb.EcBitrotProtection{
+		Algorithm:  volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+		BlockSize:  uint32(BitrotBlockSize),
+		Generation: 0,
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards: ds, ParityShards: ps, BlockSize: blockSize,
+		},
+	}
+	if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), prot); err != nil {
+		t.Fatalf("seed .ecsum: %v", err)
+	}
+}
+
+// The sidecar records the layout at encode time, so it answers what a .vif
+// cannot. A .vif that merely omits ecShardConfig is no more informative than an
+// absent one — going straight to the defaults reads a uniform volume with the
+// legacy offset math and returns the wrong bytes.
+func TestNewEcVolumeTakesLayoutFromSidecarWhenVifHasNoConfig(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "1")
+	if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+		t.Fatalf("seed .ecx: %v", err)
+	}
+	// A config-free .vif: version and dat size only, as a legacy encode left it.
+	if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+		Version: uint32(needle.Version3),
+	}); err != nil {
+		t.Fatalf("seed .vif: %v", err)
+	}
+	uniformSidecar(t, base, 12, 4, 3*1024*1024)
+
+	ev, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(1))
+	if err != nil {
+		t.Fatalf("mount: %v", err)
+	}
+	defer ev.Close()
+	if got := ev.ECContext; got.DataShards != 12 || got.ParityShards != 4 || got.BlockSize != 3*1024*1024 {
+		t.Errorf("layout = %d+%d block %d, want 12+4 block 3MiB", got.DataShards, got.ParityShards, got.BlockSize)
+	}
+}
+
+// A split -dir/-dir.idx layout keeps the sidecar with the INDEX. Probing only
+// the data base reports "absent", and absent is the one answer that selects the
+// legacy layout.
+func TestNewEcVolumeFindsSidecarInTheIndexDirectory(t *testing.T) {
+	for _, tc := range []struct {
+		name    string
+		writeVif bool
+	}{
+		{name: "no vif at all"},
+		{name: "vif without ec config", writeVif: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			dataDir, idxDir := t.TempDir(), t.TempDir()
+			idxBase := filepath.Join(idxDir, "2")
+			if err := os.WriteFile(idxBase+".ecx", []byte{}, 0644); err != nil {
+				t.Fatalf("seed .ecx: %v", err)
+			}
+			if tc.writeVif {
+				if err := volume_info.SaveVolumeInfo(idxBase+".vif", &volume_server_pb.VolumeInfo{
+					Version: uint32(needle.Version3),
+				}); err != nil {
+					t.Fatalf("seed .vif: %v", err)
+				}
+			}
+			uniformSidecar(t, idxBase, 12, 4, 3*1024*1024)
+
+			ev, err := NewEcVolume(types.HardDriveType, dataDir, idxDir, "", needle.VolumeId(2))
+			if err != nil {
+				t.Fatalf("mount: %v", err)
+			}
+			defer ev.Close()
+			if got := ev.ECContext; got.DataShards != 12 || got.ParityShards != 4 || got.BlockSize != 3*1024*1024 {
+				t.Errorf("layout = %d+%d block %d, want 12+4 block 3MiB", got.DataShards, got.ParityShards, got.BlockSize)
+			}
+		})
+	}
+}
+
+// Absence stays legal — a volume encoded before either record exists is a
+// genuine legacy volume, and only that case may select the legacy layout.
+func TestNewEcVolumeConfigFreeVifWithNoSidecarUsesDefaults(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "3")
+	if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+		t.Fatalf("seed .ecx: %v", err)
+	}
+	if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+		Version: uint32(needle.Version3),
+	}); err != nil {
+		t.Fatalf("seed .vif: %v", err)
+	}
+	ev, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(3))
+	if err != nil {
+		t.Fatalf("mount: %v", err)
+	}
+	defer ev.Close()
+	if got := ev.ECContext; got.DataShards != DataShardsCount || got.ParityShards != ParityShardsCount || got.BlockSize != 0 {
+		t.Errorf("layout = %d+%d block %d, want the legacy defaults", got.DataShards, got.ParityShards, got.BlockSize)
+	}
+}
+
+// Present but unusable must fail the mount rather than default, in the
+// config-free-vif branch exactly as in the absent-vif one.
+func TestNewEcVolumeConfigFreeVifWithUnusableSidecarFails(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "4")
+	if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+		t.Fatalf("seed .ecx: %v", err)
+	}
+	if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+		Version: uint32(needle.Version3),
+	}); err != nil {
+		t.Fatalf("seed .vif: %v", err)
+	}
+	// Stamped for another generation: not a record of these shards.
+	if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), &volume_server_pb.EcBitrotProtection{
+		Algorithm:     volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+		BlockSize:     uint32(BitrotBlockSize),
+		Generation:    5,
+		EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 12, ParityShards: 4},
+	}); err != nil {
+		t.Fatalf("seed .ecsum: %v", err)
+	}
+	if _, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(4)); err == nil {
+		t.Fatal("an unusable sole layout record must fail the mount")
+	}
+}
+
+// Startup mirroring gives each shard-bearing disk its own .ecx/.ecj/.vif but
+// deliberately not the .ecsum, and a repair delivers exactly one copy. A
+// runtime restricted to its own two directories therefore reports no
+// protection no matter how often it reloads — so the resolution has to reach
+// the sibling disk that actually received the manifest.
+func TestReloadBitrotSidecarFindsItOnASiblingDisk(t *testing.T) {
+	mine, sibling := t.TempDir(), t.TempDir()
+	myBase := filepath.Join(mine, "7")
+	if err := os.WriteFile(myBase+".ecx", []byte{}, 0644); err != nil {
+		t.Fatalf("seed .ecx: %v", err)
+	}
+	if err := volume_info.SaveVolumeInfo(myBase+".vif", &volume_server_pb.VolumeInfo{
+		Version: uint32(needle.Version3),
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards: 10, ParityShards: 4, BlockSize: 0,
+		},
+	}); err != nil {
+		t.Fatalf("seed .vif: %v", err)
+	}
+
+	ev, err := NewEcVolume(types.HardDriveType, mine, mine, "", needle.VolumeId(7))
+	if err != nil {
+		t.Fatalf("mount: %v", err)
+	}
+	defer ev.Close()
+	if got := ev.bitrotStatus; got != BitrotOff {
+		t.Fatalf("precondition: status = %v, want BitrotOff before the delivery", got)
+	}
+
+	// The delivery lands the manifest on the sibling disk only.
+	prot := &volume_server_pb.EcBitrotProtection{
+		Algorithm:  volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+		BlockSize:  uint32(BitrotBlockSize),
+		Generation: 0,
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards: 10, ParityShards: 4, BlockSize: 0,
+		},
+	}
+	for i := 0; i < 14; i++ {
+		prot.Shards = append(prot.Shards, &volume_server_pb.EcShardChecksums{
+			ShardId: uint32(i), CoveredSize: 1, BlockCrc32C: make([]byte, 4),
+		})
+	}
+	if err := SaveBitrotSidecar(BitrotSidecarPath(filepath.Join(sibling, "7"), 0), prot); err != nil {
+		t.Fatalf("seed sibling .ecsum: %v", err)
+	}
+
+	// Reloading against only its own directories still finds nothing.
+	ev.ReloadBitrotSidecar()
+	if got := ev.bitrotStatus; got != BitrotOff {
+		t.Errorf("own-directories reload: status = %v, want BitrotOff", got)
+	}
+
+	ev.ReloadBitrotSidecar(mine, sibling)
+	if got := ev.bitrotStatus; got != BitrotOn {
+		t.Errorf("cross-disk reload: status = %v, want BitrotOn", got)
+	}
+}
+
+// "Does this volume already have a manifest" cannot be answered from one base
+// name. The rebuild's opportunistic backfill asks it before writing a TOFU
+// baseline, and a false "no" there writes a second sidecar at the data base
+// that then shadows the real one — blessing whatever the shards currently say.
+func TestFindBitrotSidecarSearchesEveryCandidate(t *testing.T) {
+	dataDir, idxDir, sibling := t.TempDir(), t.TempDir(), t.TempDir()
+	dataBase := filepath.Join(dataDir, "8")
+	idxBase := filepath.Join(idxDir, "8")
+	siblingBase := filepath.Join(sibling, "8")
+
+	if got := FindBitrotSidecar(0, dataBase, idxBase, sibling); got != "" {
+		t.Errorf("no sidecar anywhere should answer empty, got %q", got)
+	}
+
+	// Each location on its own: any one of them is enough to answer "yes".
+	for _, tc := range []struct{ name, base string }{
+		{"data directory", dataBase},
+		{"index directory", idxBase},
+		{"sibling disk", siblingBase},
+	} {
+		want := BitrotSidecarPath(tc.base, 0)
+		if err := os.WriteFile(want, []byte("x"), 0644); err != nil {
+			t.Fatalf("%s: seed: %v", tc.name, err)
+		}
+		if got := FindBitrotSidecar(0, dataBase, idxBase, sibling); got != want {
+			t.Errorf("%s: got %q, want %q", tc.name, got, want)
+		}
+		if err := os.Remove(want); err != nil {
+			t.Fatalf("%s: cleanup: %v", tc.name, err)
+		}
+	}
+
+	// With copies everywhere the data base wins, which is the order every
+	// resolver uses — so a stray copy there shadows the others.
+	for _, base := range []string{siblingBase, idxBase, dataBase} {
+		if err := os.WriteFile(BitrotSidecarPath(base, 0), []byte("x"), 0644); err != nil {
+			t.Fatalf("seed %s: %v", base, err)
+		}
+	}
+	if got, want := FindBitrotSidecar(0, dataBase, idxBase, sibling), BitrotSidecarPath(dataBase, 0); got != want {
+		t.Errorf("precedence: got %q, want the data base %q", got, want)
+	}
+}

weed/storage/erasure_coding/ec_volume_test.go

@@ -0,0 +1,258 @@
+package erasure_coding
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+)
+
+func TestPositioning(t *testing.T) {
+
+	ecxFile, err := os.OpenFile("./test_files/389.ecx", os.O_RDONLY, 0)
+	if err != nil {
+		t.Errorf("failed to open ecx file: %v", err)
+	}
+	defer ecxFile.Close()
+
+	stat, _ := ecxFile.Stat()
+	fileSize := stat.Size()
+
+	tests := []struct {
+		needleId string
+		offset   int64
+		size     int
+	}{
+		{needleId: "0f0edb92", offset: 31300679656, size: 1167},
+		{needleId: "0ef7d7f8", offset: 11513014944, size: 66044},
+	}
+
+	for _, test := range tests {
+		needleId, _ := types.ParseNeedleId(test.needleId)
+		offset, size, err := SearchNeedleFromSortedIndex(ecxFile, fileSize, needleId, nil)
+		assert.Equal(t, nil, err, "SearchNeedleFromSortedIndex")
+		fmt.Printf("offset: %d size: %d\n", offset.ToActualOffset(), size)
+	}
+
+	needleId, _ := types.ParseNeedleId("0f087622")
+	offset, size, err := SearchNeedleFromSortedIndex(ecxFile, fileSize, needleId, nil)
+	assert.Equal(t, nil, err, "SearchNeedleFromSortedIndex")
+	fmt.Printf("offset: %d size: %d\n", offset.ToActualOffset(), size)
+
+	var shardEcdFileSize int64 = 1118830592 // 1024*1024*1024*3
+	intervals := LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shardEcdFileSize, offset.ToActualOffset(), types.Size(needle.GetActualSize(size, needle.GetCurrentVersion())))
+
+	for _, interval := range intervals {
+		shardId, shardOffset := interval.ToShardIdAndOffset(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize)
+		fmt.Printf("interval: %+v, shardId: %d, shardOffset: %d\n", interval, shardId, shardOffset)
+	}
+
+}
+
+// TestNewEcVolumeLoadsEncodeTsNs pins that the per-encode identity stamped into
+// .vif is loaded onto the EcVolume, so reads can reject a shard from a different
+// encode run.
+func TestNewEcVolumeLoadsEncodeTsNs(t *testing.T) {
+	dir := t.TempDir()
+	const vid = needle.VolumeId(123)
+	base := EcShardFileName("", dir, int(vid))
+
+	// A 0-byte .ecx is a valid index (no live needles) and lets NewEcVolume mount.
+	if err := os.WriteFile(base+".ecx", nil, 0o644); err != nil {
+		t.Fatalf("write .ecx: %v", err)
+	}
+
+	const tsNs int64 = 1717000000000000123
+	vi := &volume_server_pb.VolumeInfo{
+		Version: uint32(needle.Version3),
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards:   10,
+			ParityShards: 4,
+			EncodeTsNs:   tsNs,
+		},
+	}
+	if err := volume_info.SaveVolumeInfo(base+".vif", vi); err != nil {
+		t.Fatalf("save .vif: %v", err)
+	}
+
+	ev, err := NewEcVolume(types.HardDriveType, dir, dir, "", vid)
+	if err != nil {
+		t.Fatalf("NewEcVolume: %v", err)
+	}
+	defer ev.Close()
+
+	if ev.EncodeTsNs != tsNs {
+		t.Errorf("EncodeTsNs = %d, want %d", ev.EncodeTsNs, tsNs)
+	}
+}
+
+// TestNewEcVolumeDoesNotWriteStubVif pins that mounting an EC volume whose .vif
+// is missing does NOT fabricate a stub .vif. A version-only stub would imply
+// the default ratio with DatFileSize=0 and no encode identity, which the
+// custom-ratio resolver and startup credibility checks must not trust.
+func TestNewEcVolumeDoesNotWriteStubVif(t *testing.T) {
+	dir := t.TempDir()
+	const vid = needle.VolumeId(124)
+	base := EcShardFileName("", dir, int(vid))
+
+	if err := os.WriteFile(base+".ecx", nil, 0o644); err != nil {
+		t.Fatalf("write .ecx: %v", err)
+	}
+
+	ev, err := NewEcVolume(types.HardDriveType, dir, dir, "", vid)
+	if err != nil {
+		t.Fatalf("NewEcVolume: %v", err)
+	}
+	defer ev.Close()
+
+	if _, statErr := os.Stat(base + ".vif"); !os.IsNotExist(statErr) {
+		t.Fatalf("mounting without a .vif must not create one, stat err=%v", statErr)
+	}
+	// Mount still succeeds with the build's default EC ratio in memory.
+	if ev.ECContext == nil || ev.ECContext.DataShards != int(DataShardsCount) {
+		t.Fatalf("expected default EC context, got %+v", ev.ECContext)
+	}
+}
+
+// TestNewEcVolumeLoadsCustomRatio pins that a volume's own EC ratio is loaded from
+// its .vif into ECContext, so the read/recover/decode paths reconstruct with the
+// matrix that produced the shards rather than the build default. (Custom ratios are
+// an enterprise feature; in OSS the .vif always records 10+4.)
+func TestNewEcVolumeLoadsCustomRatio(t *testing.T) {
+	dir := t.TempDir()
+	const vid = needle.VolumeId(125)
+	base := EcShardFileName("", dir, int(vid))
+
+	if err := os.WriteFile(base+".ecx", nil, 0o644); err != nil {
+		t.Fatalf("write .ecx: %v", err)
+	}
+	if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+		Version: uint32(needle.Version3),
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards:   9,
+			ParityShards: 3,
+		},
+	}); err != nil {
+		t.Fatalf("save .vif: %v", err)
+	}
+
+	ev, err := NewEcVolume(types.HardDriveType, dir, dir, "", vid)
+	if err != nil {
+		t.Fatalf("NewEcVolume: %v", err)
+	}
+	defer ev.Close()
+
+	if ev.ECContext == nil {
+		t.Fatalf("ECContext must be set from the .vif")
+	}
+	if ev.ECContext.DataShards != 9 || ev.ECContext.ParityShards != 3 {
+		t.Fatalf("ECContext = %d+%d, want 9+3", ev.ECContext.DataShards, ev.ECContext.ParityShards)
+	}
+}
+
+// ecIndexDirsFixture lays out an EC volume across a data dir and a separate
+// index dir (the -dir.idx split), planting a .vif next to the data and .ecx
+// files of the given sizes (-1 = absent). It returns the two bases.
+func ecIndexDirsFixture(t *testing.T, vid needle.VolumeId, localEcx, sharedEcx int) (dataDir, idxDir, dataBase, idxBase string) {
+	t.Helper()
+	root := t.TempDir()
+	dataDir = filepath.Join(root, "data")
+	idxDir = filepath.Join(root, "idx")
+	for _, d := range []string{dataDir, idxDir} {
+		if err := os.MkdirAll(d, 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	dataBase = EcShardFileName("", dataDir, int(vid))
+	idxBase = EcShardFileName("", idxDir, int(vid))
+	writeEcx := func(base string, size int) {
+		if size < 0 {
+			return
+		}
+		if err := os.WriteFile(base+".ecx", make([]byte, size), 0o644); err != nil {
+			t.Fatalf("write .ecx: %v", err)
+		}
+	}
+	writeEcx(dataBase, localEcx)
+	writeEcx(idxBase, sharedEcx)
+	if err := volume_info.SaveVolumeInfo(dataBase+".vif", &volume_server_pb.VolumeInfo{
+		Version:       uint32(needle.Version3),
+		EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4},
+	}); err != nil {
+		t.Fatalf("save .vif: %v", err)
+	}
+	return
+}
+
+// TestNewEcVolumePrefersLocalEcx pins the local-first resolution: with a
+// non-empty .ecx co-located with the shard data AND one in the shared index
+// dir, NewEcVolume opens the local copy — where a move or reconstruct leaves
+// the index.
+func TestNewEcVolumePrefersLocalEcx(t *testing.T) {
+	const vid = needle.VolumeId(200)
+	dataDir, idxDir, _, _ := ecIndexDirsFixture(t, vid, 16, 16)
+
+	ev, err := NewEcVolume(types.HardDriveType, dataDir, idxDir, "", vid)
+	if err != nil {
+		t.Fatalf("NewEcVolume: %v", err)
+	}
+	defer ev.Close()
+
+	if got := filepath.Dir(ev.FileName(".ecx")); got != dataDir {
+		t.Errorf(".ecx resolved in %q, want the data dir %q (local-first)", got, dataDir)
+	}
+}
+
+// TestNewEcVolumeZeroByteLocalYieldsToSharedEcx pins that a 0-byte local .ecx
+// (an interrupted-copy stub is indistinguishable from a legitimately empty
+// index by size) yields to a *non-empty* copy in the shared index dir, keeping
+// the cross-disk fallback (#9212).
+func TestNewEcVolumeZeroByteLocalYieldsToSharedEcx(t *testing.T) {
+	const vid = needle.VolumeId(201)
+	dataDir, idxDir, _, _ := ecIndexDirsFixture(t, vid, 0, 16)
+
+	ev, err := NewEcVolume(types.HardDriveType, dataDir, idxDir, "", vid)
+	if err != nil {
+		t.Fatalf("NewEcVolume: %v", err)
+	}
+	defer ev.Close()
+
+	if got := filepath.Dir(ev.FileName(".ecx")); got != idxDir {
+		t.Errorf(".ecx resolved in %q, want the shared idx dir %q (non-empty wins over 0-byte local)", got, idxDir)
+	}
+}
+
+// TestEcVolumeDestroySweepsBothDirs pins that Destroy removes the EC index
+// files from both the data and index directories, so a stale copy left by a
+// move or reconstruct cannot re-mount as a phantom EC volume.
+func TestEcVolumeDestroySweepsBothDirs(t *testing.T) {
+	const vid = needle.VolumeId(202)
+	dataDir, idxDir, dataBase, idxBase := ecIndexDirsFixture(t, vid, 16, 16)
+	// A stale .ecj in each dir too.
+	for _, base := range []string{dataBase, idxBase} {
+		if err := os.WriteFile(base+".ecj", nil, 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	ev, err := NewEcVolume(types.HardDriveType, dataDir, idxDir, "", vid)
+	if err != nil {
+		t.Fatalf("NewEcVolume: %v", err)
+	}
+	ev.Destroy()
+
+	for _, base := range []string{dataBase, idxBase} {
+		for _, ext := range []string{".ecx", ".ecj"} {
+			if _, err := os.Stat(base + ext); !os.IsNotExist(err) {
+				t.Errorf("%s%s survived Destroy (err=%v), want removed from both dirs", base, ext, err)
+			}
+		}
+	}
+}

weed/storage/erasure_coding/ec_volume_vif_test.go

@@ -0,0 +1,187 @@
+package erasure_coding
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+)
+
+// A present-but-malformed .vif must FAIL the mount: every new encode
+// records a positive uniform block size there, and silently defaulting
+// to the legacy layout would serve those shards with the wrong offset
+// math. Absence stays legal — legacy volumes predate the sidecar.
+func TestNewEcVolumeFailsOnMalformedVif(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "1")
+	if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+		t.Fatalf("seed .ecx: %v", err)
+	}
+	if err := os.WriteFile(base+".vif", []byte("not json"), 0644); err != nil {
+		t.Fatalf("seed .vif: %v", err)
+	}
+	_, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(1))
+	if err == nil {
+		t.Fatal("mount over a malformed .vif must fail")
+	}
+	if !strings.Contains(err.Error(), ".vif") {
+		t.Errorf("error should name the vif: %v", err)
+	}
+}
+
+func TestNewEcVolumeAbsentVifMountsWithDefaults(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "1")
+	if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+		t.Fatalf("seed .ecx: %v", err)
+	}
+	ev, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(1))
+	if err != nil {
+		t.Fatalf("legacy mount without .vif must succeed: %v", err)
+	}
+	defer ev.Close()
+	if ev.ECContext.BlockSize != 0 {
+		t.Errorf("legacy mount must use the legacy layout: BlockSize=%d", ev.ECContext.BlockSize)
+	}
+}
+
+// A block size no encoder could have produced (negative, or not a whole
+// number of small blocks) maps every read to the wrong shard offset, so the
+// mount refuses it rather than serving those bytes.
+func TestNewEcVolumeRejectsImplausibleBlockSize(t *testing.T) {
+	for _, blockSize := range []int64{1, -1, 3*1024*1024 + 1} {
+		dir := t.TempDir()
+		base := filepath.Join(dir, "1")
+		if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+			t.Fatalf("seed .ecx: %v", err)
+		}
+		if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+			Version: uint32(needle.Version3),
+			EcShardConfig: &volume_server_pb.EcShardConfig{
+				DataShards: 10, ParityShards: 4, BlockSize: blockSize,
+			},
+		}); err != nil {
+			t.Fatalf("seed .vif: %v", err)
+		}
+		if _, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(1)); err == nil {
+			t.Errorf("block size %d must fail the mount", blockSize)
+		}
+	}
+}
+
+func TestNewEcVolumeAcceptsAlignedBlockSize(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "1")
+	if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+		t.Fatalf("seed .ecx: %v", err)
+	}
+	if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+		Version: uint32(needle.Version3),
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards: 10, ParityShards: 4, BlockSize: 3 * 1024 * 1024,
+		},
+	}); err != nil {
+		t.Fatalf("seed .vif: %v", err)
+	}
+	ev, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(1))
+	if err != nil {
+		t.Fatalf("aligned block size must mount: %v", err)
+	}
+	defer ev.Close()
+	if ev.ECContext.BlockSize != 3*1024*1024 {
+		t.Errorf("BlockSize = %d, want 3MiB", ev.ECContext.BlockSize)
+	}
+}
+
+// The .vif and the .ecsum both record the layout their generation was encoded
+// with. When a generation-matching sidecar disagrees, one of them is wrong and
+// reads through the other land at the wrong shard offsets — so the mount must
+// refuse rather than quietly drop to unprotected reads.
+func TestNewEcVolumeRejectsSidecarGeometryDisagreement(t *testing.T) {
+	dir := t.TempDir()
+	base := filepath.Join(dir, "1")
+	if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+		t.Fatalf("seed .ecx: %v", err)
+	}
+	if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+		Version: uint32(needle.Version3),
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards: 10, ParityShards: 4, BlockSize: 0, // says legacy
+		},
+	}); err != nil {
+		t.Fatalf("seed .vif: %v", err)
+	}
+	// The sidecar for the same generation says uniform.
+	prot := &volume_server_pb.EcBitrotProtection{
+		Algorithm:  volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+		BlockSize:  uint32(BitrotBlockSize),
+		Generation: 0,
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards: 10, ParityShards: 4, BlockSize: 3 * 1024 * 1024,
+		},
+	}
+	if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), prot); err != nil {
+		t.Fatalf("seed .ecsum: %v", err)
+	}
+
+	_, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(1))
+	if err == nil {
+		t.Fatal("a generation-matching layout disagreement must fail the mount")
+	}
+	if !strings.Contains(err.Error(), "block") {
+		t.Errorf("the error should name the disagreement: %v", err)
+	}
+}
+
+// With no .vif the bitrot sidecar is the ONLY record of the volume's layout.
+// Present but unusable — unreadable, stamped for another generation, or
+// carrying an incomplete ratio — must fail the mount: answering reads from a
+// guessed layout returns wrong bytes rather than none. Genuine absence stays
+// legal, and is covered by TestNewEcVolumeAbsentVifMountsWithDefaults.
+func TestNewEcVolumeRejectsUnusableSoleSidecar(t *testing.T) {
+	seed := map[string]*volume_server_pb.EcBitrotProtection{
+		"another generation": {
+			Algorithm:     volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+			BlockSize:     uint32(BitrotBlockSize),
+			Generation:    5,
+			EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4},
+		},
+		"no ec config": {
+			Algorithm:  volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+			BlockSize:  uint32(BitrotBlockSize),
+			Generation: 0,
+		},
+		"incomplete ratio": {
+			Algorithm:     volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+			BlockSize:     uint32(BitrotBlockSize),
+			Generation:    0,
+			EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10},
+		},
+		"invalid block size": {
+			Algorithm:  volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+			BlockSize:  uint32(BitrotBlockSize),
+			Generation: 0,
+			EcShardConfig: &volume_server_pb.EcShardConfig{
+				DataShards: 10, ParityShards: 4, BlockSize: 3*1024*1024 + 1,
+			},
+		},
+	}
+	for name, prot := range seed {
+		dir := t.TempDir()
+		base := filepath.Join(dir, "1")
+		if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+			t.Fatalf("%s: seed .ecx: %v", name, err)
+		}
+		if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), prot); err != nil {
+			t.Fatalf("%s: seed .ecsum: %v", name, err)
+		}
+		if _, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(1)); err == nil {
+			t.Errorf("%s: an unusable sole layout record must fail the mount", name)
+		}
+	}
+}

weed/storage/erasure_coding/ecbalancer/balancer_test.go

@@ -0,0 +1,863 @@
+package ecbalancer
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+)
+
+func bits(ids ...int) erasure_coding.ShardBits {
+	var b erasure_coding.ShardBits
+	for _, id := range ids {
+		b = b.Set(erasure_coding.ShardId(id))
+	}
+	return b
+}
+
+// addEmptyNode adds an EC-empty destination node with six disks and capacity.
+func addEmptyNode(t *Topology, id, rackKey string) {
+	n := t.AddNode(id, "dc1", rackKey, 600)
+	for d := uint32(1); d <= 6; d++ {
+		n.AddDisk(d, "", 100, 0)
+	}
+}
+
+func ratio(d, p int) func(string) (int, int) {
+	return func(string) (int, int) { return d, p }
+}
+
+func TestPickBestDiskOnNode(t *testing.T) {
+	const vid = uint32(100)
+	const ds = erasure_coding.DataShardsCount
+	vk := volKey{collection: "c", vid: vid}
+
+	t.Run("skips disks with no free slots", func(t *testing.T) {
+		topo := NewTopology()
+		n := topo.AddNode("n1", "dc1", "dc1:r1", 100)
+		n.AddDisk(1, "", 0, 0)
+		n.AddDisk(2, "", 10, 0)
+		if got := pickBestDiskOnNode(n, vk, "", 0, ds); got != 2 {
+			t.Errorf("got disk %d, want 2", got)
+		}
+	})
+
+	t.Run("spreads a volume's shards across disks", func(t *testing.T) {
+		topo := NewTopology()
+		n := topo.AddNode("n1", "dc1", "dc1:r1", 100)
+		n.AddDisk(1, "", 10, 1)
+		n.AddDisk(2, "", 10, 0)
+		n.AddShards(vid, "c", 1, bits(0))
+		if got := pickBestDiskOnNode(n, vk, "", 5, ds); got != 2 {
+			t.Errorf("got disk %d, want 2 (disk 1 already holds this volume)", got)
+		}
+	})
+
+	t.Run("data shard avoids disk holding parity", func(t *testing.T) {
+		topo := NewTopology()
+		n := topo.AddNode("n1", "dc1", "dc1:r1", 100)
+		n.AddDisk(1, "", 10, 1)
+		n.AddDisk(2, "", 10, 0)
+		n.AddShards(vid, "c", 1, bits(ds)) // parity on disk 1
+		if got := pickBestDiskOnNode(n, vk, "", 0, ds); got != 2 {
+			t.Errorf("got disk %d, want 2 (anti-affinity)", got)
+		}
+	})
+
+	t.Run("anti-affinity follows the supplied ratio boundary", func(t *testing.T) {
+		topo := NewTopology()
+		n := topo.AddNode("n1", "dc1", "dc1:r1", 100)
+		n.AddDisk(1, "", 10, 1)
+		n.AddDisk(2, "", 10, 2)
+		n.AddShards(vid, "c", 1, bits(7)) // parity at 6+3
+		n.AddShards(vid, "c", 2, bits(2)) // data
+		if got := pickBestDiskOnNode(n, vk, "", 1, 6); got != 2 {
+			t.Errorf("ratio 6: got disk %d, want 2", got)
+		}
+		if got := pickBestDiskOnNode(n, vk, "", 1, erasure_coding.DataShardsCount); got != 1 {
+			t.Errorf("boundary 10: got disk %d, want 1", got)
+		}
+	})
+
+	t.Run("only matching disk type when set", func(t *testing.T) {
+		topo := NewTopology()
+		n := topo.AddNode("n1", "dc1", "dc1:r1", 100)
+		n.AddDisk(1, "ssd", 10, 0)
+		n.AddDisk(2, "hdd", 10, 0)
+		if got := pickBestDiskOnNode(n, vk, "hdd", 0, ds); got != 2 {
+			t.Errorf("got disk %d, want 2 (only hdd)", got)
+		}
+	})
+}
+
+func TestPlanSourceDiskAttribution(t *testing.T) {
+	shardsByDisk := map[uint32][]int{0: {0, 1, 2}, 1: {3, 4, 5}, 2: {6, 7}, 3: {8, 9}, 4: {10, 11}, 5: {12, 13}}
+	shardToDisk := map[int]uint32{}
+	for d, ss := range shardsByDisk {
+		for _, s := range ss {
+			shardToDisk[s] = d
+		}
+	}
+	topo := NewTopology()
+	src := topo.AddNode("node1", "dc1", "dc1:rack1", 0)
+	for d, ss := range shardsByDisk {
+		src.AddDisk(d, "", 0, len(ss))
+		src.AddShards(100, "col1", d, bits(ss...))
+	}
+	addEmptyNode(topo, "node2", "dc1:rack2")
+
+	moves := Plan(topo, Options{ImbalanceThreshold: 0.01, Ratio: ratio(10, 4)})
+	if len(moves) == 0 {
+		t.Fatal("expected moves")
+	}
+	for _, m := range moves {
+		if m.Phase != "cross_rack" {
+			continue
+		}
+		if want := shardToDisk[m.ShardID]; m.SourceDisk != want {
+			t.Errorf("shard %d: source disk %d, want %d", m.ShardID, m.SourceDisk, want)
+		}
+	}
+}
+
+func TestPlanSpreadsAcrossDestinationDisks(t *testing.T) {
+	topo := NewTopology()
+	src := topo.AddNode("node1", "dc1", "dc1:rack1", 0)
+	src.AddDisk(0, "", 0, 14)
+	src.AddShards(100, "col1", 0, bits(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13))
+	addEmptyNode(topo, "node2", "dc1:rack2")
+
+	moves := Plan(topo, Options{ImbalanceThreshold: 0.01, Ratio: ratio(10, 4)})
+	distinct := map[uint32]bool{}
+	crossRack := 0
+	for _, m := range moves {
+		if m.Phase == "cross_rack" {
+			crossRack++
+			distinct[m.TargetDisk] = true
+		}
+	}
+	if crossRack != 7 {
+		t.Fatalf("got %d cross-rack moves, want 7", crossRack)
+	}
+	if len(distinct) != 6 {
+		t.Errorf("cross-rack moves used %d distinct disks, want 6: %v", len(distinct), distinct)
+	}
+}
+
+func TestPlanCrossRackParityAntiAffinity(t *testing.T) {
+	topo := NewTopology()
+	src := topo.AddNode("node1", "dc1", "dc1:rack1", 0)
+	src.AddDisk(0, "", 0, 3)
+	src.AddShards(100, "col1", 0, bits(0, 1, 2)) // 1 data + 2 parity at ratio 1+2
+	addEmptyNode(topo, "node2", "dc1:rack2")
+	addEmptyNode(topo, "node3", "dc1:rack3")
+
+	moves := Plan(topo, Options{ImbalanceThreshold: 0.01, Ratio: ratio(1, 2)})
+	if len(moves) == 0 {
+		t.Fatal("expected parity moves across racks")
+	}
+	for _, m := range moves {
+		if m.ShardID < 1 {
+			t.Errorf("data shard %d moved; it fits in rack1", m.ShardID)
+		}
+		if m.TargetNode == "node1" {
+			t.Errorf("parity shard %d moved onto data-bearing node1", m.ShardID)
+		}
+	}
+}
+
+func TestWithinRackParityAntiAffinity(t *testing.T) {
+	// Test the within-rack phase in isolation (the global phase, which balances
+	// total load, would otherwise also act on this single rack).
+	topo := NewTopology()
+	src := topo.AddNode("node1", "dc1", "dc1:rack1", 600)
+	src.AddDisk(0, "", 600, 3)
+	src.AddShards(100, "col1", 0, bits(0, 1, 2))
+	addEmptyNode(topo, "node2", "dc1:rack1")
+	addEmptyNode(topo, "node3", "dc1:rack1")
+
+	racks := buildRacks(topo.nodes)
+	moves := detectWithinRackImbalance(volKey{collection: "col1", vid: 100}, topo.nodes, racks, "", 0.01, 1, 2, nil)
+	if len(moves) == 0 {
+		t.Fatal("expected parity moves within rack")
+	}
+	for _, m := range moves {
+		if m.shardID < 1 {
+			t.Errorf("data shard %d moved; it fits on node1", m.shardID)
+		}
+		if m.target.id == "node1" {
+			t.Errorf("parity shard %d moved onto data-bearing node1", m.shardID)
+		}
+	}
+}
+
+func TestPlanReplicaPlacementCapsPerRack(t *testing.T) {
+	build := func() *Topology {
+		topo := NewTopology()
+		src := topo.AddNode("node1", "dc1", "dc1:rack1", 0)
+		src.AddDisk(0, "", 0, 6)
+		src.AddShards(100, "col1", 0, bits(0, 1, 2, 3, 4, 5)) // all data at ratio 6+0
+		addEmptyNode(topo, "node2", "dc1:rack2")
+		addEmptyNode(topo, "node3", "dc1:rack3")
+		return topo
+	}
+
+	countCross := func(moves []Move) int {
+		n := 0
+		for _, m := range moves {
+			if m.Phase == "cross_rack" {
+				n++
+			}
+		}
+		return n
+	}
+
+	if got := countCross(Plan(build(), Options{ImbalanceThreshold: 0.01, Ratio: ratio(6, 0)})); got != 4 {
+		t.Fatalf("without replica placement: %d cross-rack moves, want 4", got)
+	}
+	rp := &super_block.ReplicaPlacement{DiffRackCount: 1}
+	if got := countCross(Plan(build(), Options{ImbalanceThreshold: 0.01, ReplicaPlacement: rp, Ratio: ratio(6, 0)})); got != 2 {
+		t.Errorf("with DiffRackCount=1: %d cross-rack moves, want 2", got)
+	}
+}
+
+func TestPlanDedup(t *testing.T) {
+	topo := NewTopology()
+	n1 := topo.AddNode("node1", "dc1", "dc1:rack1", 5)
+	n1.AddDisk(0, "", 5, 2)
+	n1.AddShards(100, "col1", 0, bits(0, 1))
+	n2 := topo.AddNode("node2", "dc1", "dc1:rack2", 10)
+	n2.AddDisk(0, "", 10, 1)
+	n2.AddShards(100, "col1", 0, bits(0)) // shard 0 duplicated on node2
+
+	var dedup []Move
+	for _, m := range Plan(topo, Options{ImbalanceThreshold: 0.01, Ratio: ratio(10, 4)}) {
+		if m.Phase == "dedup" {
+			dedup = append(dedup, m)
+		}
+	}
+	if len(dedup) != 1 {
+		t.Fatalf("got %d dedup moves, want 1", len(dedup))
+	}
+	if dedup[0].ShardID != 0 || dedup[0].SourceNode != "node1" || dedup[0].TargetNode != "node1" {
+		t.Errorf("dedup move = %+v, want shard 0 deleted on node1 (fewer free slots)", dedup[0])
+	}
+}
+
+func TestCeilDivide(t *testing.T) {
+	for _, tc := range []struct{ a, b, want int }{{14, 3, 5}, {10, 3, 4}, {0, 5, 0}, {5, 0, 0}} {
+		if got := ceilDivide(tc.a, tc.b); got != tc.want {
+			t.Errorf("ceilDivide(%d,%d)=%d want %d", tc.a, tc.b, got, tc.want)
+		}
+	}
+}
+
+func allBits(n int) erasure_coding.ShardBits {
+	var b erasure_coding.ShardBits
+	for i := 0; i < n; i++ {
+		b = b.Set(erasure_coding.ShardId(i))
+	}
+	return b
+}
+
+// TestPlanConvergesToBalancedLoadOverRounds starts skewed -- machine 10.0.0.1 holds
+// 4 shards of every volume, the others 2 -- and runs repeated worker-style balance
+// passes (capped moves per pass). It must converge to an even per-machine shard load.
+func TestPlanConvergesToBalancedLoadOverRounds(t *testing.T) {
+	topo := NewTopology()
+	hosts := []string{"10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4", "10.0.0.5", "10.0.0.6"}
+	for _, h := range hosts {
+		for _, p := range []string{":8080", ":8081"} {
+			n := topo.AddNode(h+p, "dc1", "dc1:rack1", 4000)
+			n.SetHost(h)
+			n.AddDisk(0, "", 4000, 0)
+		}
+	}
+	add := func(id string, vid uint32, ids ...int) {
+		n := topo.nodes[id]
+		n.AddShards(vid, "c1", 0, bits(ids...))
+		n.disks[0].shardCount += len(ids)
+		n.disks[0].freeSlots -= len(ids)
+		n.freeSlots -= len(ids)
+	}
+	const volumes = 60
+	for v := 0; v < volumes; v++ {
+		vid := uint32(v)
+		add("10.0.0.1:8080", vid, 0, 1) // machine 10.0.0.1 gets 4 shards/volume
+		add("10.0.0.1:8081", vid, 2, 3)
+		add("10.0.0.2:8080", vid, 4)
+		add("10.0.0.2:8081", vid, 5)
+		add("10.0.0.3:8080", vid, 6)
+		add("10.0.0.3:8081", vid, 7)
+		add("10.0.0.4:8080", vid, 8)
+		add("10.0.0.4:8081", vid, 9)
+		add("10.0.0.5:8080", vid, 10)
+		add("10.0.0.5:8081", vid, 11)
+		add("10.0.0.6:8080", vid, 12)
+		add("10.0.0.6:8081", vid, 13)
+	}
+
+	perMachine := func() map[string]int {
+		m := map[string]int{}
+		for _, h := range hosts {
+			m[h] = 0 // seed every host so a fully drained one still counts
+		}
+		for _, n := range topo.nodes {
+			for _, info := range n.shards {
+				m[n.host] += info.shardBits.Count()
+			}
+		}
+		return m
+	}
+	opts := Options{Ratio: ratio(10, 4), GlobalUtilizationBased: true, GlobalMaxMovesPerRack: 8}
+
+	rounds, converged := 0, false
+	for ; rounds < 200; rounds++ {
+		if len(Plan(topo, opts)) == 0 {
+			converged = true
+			break
+		}
+	}
+	if !converged {
+		t.Fatalf("balance did not converge within %d rounds", rounds)
+	}
+
+	pm := perMachine()
+	min, max := 1<<30, 0
+	for _, c := range pm {
+		if c < min {
+			min = c
+		}
+		if c > max {
+			max = c
+		}
+	}
+	t.Logf("converged after %d rounds: %v", rounds, pm)
+	if rounds < 2 {
+		t.Errorf("expected convergence to take multiple rounds (capped moves), took %d", rounds)
+	}
+	if float64(max) > 1.1*float64(min) {
+		t.Errorf("did not converge to balanced load: min=%d max=%d %v", min, max, pm)
+	}
+}
+
+func TestGlobalImbalanceMovesFromFullToEmpty(t *testing.T) {
+	topo := NewTopology()
+	n1 := topo.AddNode("node1", "dc1", "dc1:rack1", 5)
+	n1.AddShards(100, "col1", 0, allBits(14))
+	n1.AddShards(200, "col1", 0, allBits(6))
+	n2 := topo.AddNode("node2", "dc1", "dc1:rack1", 30)
+	n2.AddShards(300, "col1", 0, allBits(2))
+
+	moves := detectGlobalImbalance(topo.nodes, buildRacks(topo.nodes), "", 0.01, nil, nil, 0, true)
+	if len(moves) == 0 {
+		t.Fatal("expected global balance moves")
+	}
+	for _, m := range moves {
+		if m.phase != "global" || m.source.id != "node1" || m.target.id != "node2" {
+			t.Errorf("move = %+v, want global node1->node2", m)
+		}
+	}
+}
+
+// TestGlobalImbalanceHeterogeneousCapacity: node1 holds more shards but is less
+// utilized (high capacity); moves must drain the more-utilized node2.
+func TestGlobalImbalanceHeterogeneousCapacity(t *testing.T) {
+	topo := NewTopology()
+	n1 := topo.AddNode("node1", "dc1", "dc1:rack1", 90)
+	n1.AddShards(100, "col1", 0, allBits(10))
+	n2 := topo.AddNode("node2", "dc1", "dc1:rack1", 2)
+	n2.AddShards(200, "col1", 0, allBits(3))
+
+	moves := detectGlobalImbalance(topo.nodes, buildRacks(topo.nodes), "", 0.01, nil, nil, 0, true)
+	if len(moves) == 0 {
+		t.Fatal("expected moves from high-util node2 to low-util node1")
+	}
+	seen := map[[2]int]bool{}
+	for _, m := range moves {
+		if m.source.id != "node2" || m.target.id != "node1" {
+			t.Errorf("move = %+v, want node2->node1", m)
+		}
+		key := [2]int{int(m.volumeID), m.shardID}
+		if seen[key] {
+			t.Errorf("duplicate move for volume %d shard %d", m.volumeID, m.shardID)
+		}
+		seen[key] = true
+	}
+}
+
+func TestGlobalImbalanceSkipsFullNodes(t *testing.T) {
+	topo := NewTopology()
+	n1 := topo.AddNode("node1", "dc1", "dc1:rack1", 10)
+	n1.AddShards(100, "col1", 0, allBits(14))
+	n2 := topo.AddNode("node2", "dc1", "dc1:rack1", 0) // full, cannot receive
+	n2.AddShards(200, "col1", 0, allBits(2))
+
+	if moves := detectGlobalImbalance(topo.nodes, buildRacks(topo.nodes), "", 0.01, nil, nil, 0, true); len(moves) != 0 {
+		t.Fatalf("expected 0 moves (node2 full), got %d", len(moves))
+	}
+}
+
+// TestPlanBalancesSkewedDataParityWithEvenTotals guards the per-type gate: two
+// racks hold equal shard totals (7 each) but the data shards are skewed (7 vs 3).
+// A total-count gate would skip balancing; the per-type gate must still act.
+func TestPlanBalancesSkewedDataParityWithEvenTotals(t *testing.T) {
+	topo := NewTopology()
+	n1 := topo.AddNode("node1", "dc1", "dc1:rack1", 100)
+	n1.AddDisk(0, "", 100, 7)
+	n1.AddShards(100, "col1", 0, bits(0, 1, 2, 3, 4, 5, 6)) // 7 data shards
+	n2 := topo.AddNode("node2", "dc1", "dc1:rack2", 100)
+	n2.AddDisk(0, "", 100, 7)
+	n2.AddShards(100, "col1", 0, bits(7, 8, 9, 10, 11, 12, 13)) // 3 data + 4 parity
+
+	moves := Plan(topo, Options{ImbalanceThreshold: 0, Ratio: ratio(10, 4)})
+
+	crossRack, dataMoved := 0, 0
+	for _, m := range moves {
+		if m.Phase == "cross_rack" {
+			crossRack++
+			if m.ShardID < 10 {
+				dataMoved++
+			}
+		}
+	}
+	if crossRack == 0 {
+		t.Fatal("even totals masked a data/parity skew: no cross-rack moves produced")
+	}
+	if dataMoved == 0 {
+		t.Error("expected skewed data shards to rebalance across racks")
+	}
+}
+
+// TestPlanVolumeRatioOverridesCollection verifies the per-volume ratio (reported on
+// the heartbeat, surfaced via Options.VolumeRatio) takes precedence over the
+// collection ratio, so a mixed-ratio collection is classified and spread by each
+// volume's own data/parity split. The same 14-shard volume is planned three ways:
+// the per-volume 7+7 override must reproduce the 7+7-collection plan and differ from
+// the 10+4-collection plan. A VolumeRatio that returns 0 defers to the collection
+// ratio (the always-0 OSS case), so existing collection-keyed behavior is preserved.
+func TestPlanVolumeRatioOverridesCollection(t *testing.T) {
+	build := func() *Topology {
+		topo := NewTopology()
+		n1 := topo.AddNode("node1", "dc1", "dc1:rack1", 100)
+		n1.AddDisk(0, "", 100, 7)
+		n1.AddShards(100, "col1", 0, bits(0, 1, 2, 3, 4, 5, 6))
+		n2 := topo.AddNode("node2", "dc1", "dc1:rack2", 100)
+		n2.AddDisk(0, "", 100, 7)
+		n2.AddShards(100, "col1", 0, bits(7, 8, 9, 10, 11, 12, 13))
+		return topo
+	}
+	crossRack := func(moves []Move) int {
+		n := 0
+		for _, m := range moves {
+			if m.Phase == "cross_rack" {
+				n++
+			}
+		}
+		return n
+	}
+
+	collection104 := crossRack(Plan(build(), Options{ImbalanceThreshold: 0, Ratio: ratio(10, 4)}))
+	collection77 := crossRack(Plan(build(), Options{ImbalanceThreshold: 0, Ratio: ratio(7, 7)}))
+	if collection104 == collection77 {
+		t.Fatalf("test setup: 10+4 and 7+7 collection plans must differ, both gave %d cross-rack moves", collection104)
+	}
+
+	// Collection ratio stays 10+4, but vol100 reports 7+7 per-volume; the planner
+	// must classify/spread vol100 as 7+7 and match the 7+7-collection plan.
+	perVolume := crossRack(Plan(build(), Options{
+		ImbalanceThreshold: 0,
+		Ratio:              ratio(10, 4),
+		VolumeRatio: func(collection string, vid uint32) (int, int) {
+			if collection == "col1" && vid == 100 {
+				return 7, 7
+			}
+			return 0, 0
+		},
+	}))
+	if perVolume != collection77 {
+		t.Errorf("per-volume 7+7 override gave %d cross-rack moves, want %d (the 7+7-collection plan)", perVolume, collection77)
+	}
+	if perVolume == collection104 {
+		t.Errorf("per-volume override had no effect: %d cross-rack moves, same as the 10+4 collection plan", perVolume)
+	}
+}
+
+// TestGlobalPrefersVolumeAbsentFromDestination guards the global phase's
+// volume-diversity preference: when draining a node, move a shard of a volume the
+// destination does not hold at all before piling a second shard of an
+// already-present volume onto it. node1 (full) holds vol100 and vol200; node2
+// (empty) holds only vol100, so the first global move should be a vol200 shard.
+func TestGlobalPrefersVolumeAbsentFromDestination(t *testing.T) {
+	topo := NewTopology()
+	n1 := topo.AddNode("node1", "dc1", "dc1:rack1", 0)
+	n1.AddShards(100, "col1", 0, bits(0, 1))
+	n1.AddShards(200, "col1", 0, bits(0, 1))
+	n2 := topo.AddNode("node2", "dc1", "dc1:rack1", 3)
+	n2.AddShards(100, "col1", 0, bits(2))
+
+	moves := detectGlobalImbalance(topo.nodes, buildRacks(topo.nodes), "", 0.01, nil, nil, 0, true)
+	if len(moves) == 0 {
+		t.Fatal("expected a global move from the full node")
+	}
+	if moves[0].volumeID != 200 {
+		t.Errorf("first global move is volume %d, want 200 (the volume absent from node2)", moves[0].volumeID)
+	}
+	for _, m := range moves {
+		if m.source.id != "node1" || m.target.id != "node2" {
+			t.Errorf("move %+v, want node1->node2", m)
+		}
+	}
+}
+
+// TestPlanKeepsCollectionsWithSameVolumeIdDistinct guards EC identity: a numeric
+// volume id reused across collections must not be merged. (A,5) on node1 and
+// (B,5) on node2 are different volumes, so neither dedup nor any move should
+// treat them as copies of one another.
+func TestPlanKeepsCollectionsWithSameVolumeIdDistinct(t *testing.T) {
+	topo := NewTopology()
+	n1 := topo.AddNode("node1", "dc1", "dc1:rack1", 100)
+	n1.AddDisk(0, "", 100, 1)
+	n1.AddShards(5, "A", 0, bits(0))
+	n2 := topo.AddNode("node2", "dc1", "dc1:rack2", 100)
+	n2.AddDisk(0, "", 100, 1)
+	n2.AddShards(5, "B", 0, bits(0))
+
+	for _, m := range Plan(topo, Options{ImbalanceThreshold: 0.01, Ratio: ratio(10, 4)}) {
+		if m.Phase == "dedup" {
+			t.Errorf("dedup %+v: (A,5) and (B,5) are different volumes and must not be deduped", m)
+		}
+	}
+}
+
+// TestDedupFreesCapacityForLaterPhases checks that capacity opened by deleting a
+// duplicate is usable by a later phase in the same Plan. node2 is full (0 free)
+// but holds a duplicate of node1's shard 0; node1 is roomier, so dedup deletes
+// node2's copy, freeing a slot. The within-rack phase must then be able to move a
+// shard onto node2.
+func TestDedupFreesCapacityForLaterPhases(t *testing.T) {
+	topo := NewTopology()
+	n1 := topo.AddNode("node1", "dc1", "dc1:rack1", 5)
+	n1.AddDisk(0, "", 5, 7)
+	n1.AddShards(100, "col1", 0, bits(0, 1, 2, 3, 4, 5, 6))
+	n2 := topo.AddNode("node2", "dc1", "dc1:rack1", 0) // full
+	n2.AddDisk(0, "", 0, 1)
+	n2.AddShards(100, "col1", 0, bits(0)) // duplicate of node1's shard 0
+
+	moves := Plan(topo, Options{ImbalanceThreshold: 0.01, Ratio: ratio(10, 4)})
+
+	dedup := false
+	toNode2 := 0
+	for _, m := range moves {
+		if m.Phase == "dedup" {
+			dedup = true
+			continue
+		}
+		if m.TargetNode == "node2" {
+			toNode2++
+		}
+	}
+	if !dedup {
+		t.Fatal("expected a dedup move for the duplicated shard 0")
+	}
+	if toNode2 == 0 {
+		t.Error("slot freed by dedup on node2 was not usable by a later phase")
+	}
+}
+
+// TestWithinRackSpreadsAcrossMachines: a 10+4 volume's 14 shards all sit on boxA
+// (two of its servers), with three other machines free. Four machines is enough for
+// the spread to stay within parity, so the within-rack phase must move shards out
+// until no machine holds more than ceil(14/4)=4, even though they looked spread
+// across boxA's two nodes.
+func TestWithinRackSpreadsAcrossMachines(t *testing.T) {
+	topo := NewTopology()
+	mk := func(id, host string) *Node {
+		n := topo.AddNode(id, "dc1", "dc1:rack1", 100)
+		n.SetHost(host)
+		n.AddDisk(0, "", 100, 0)
+		return n
+	}
+	a1 := mk("a1", "boxA")
+	a2 := mk("a2", "boxA")
+	mk("b1", "boxB")
+	mk("c1", "boxC")
+	mk("d1", "boxD")
+	a1.AddShards(100, "col1", 0, bits(0, 1, 2, 3, 4, 5, 6))     // 7 shards on boxA
+	a2.AddShards(100, "col1", 0, bits(7, 8, 9, 10, 11, 12, 13)) // 7 shards on boxA (all 14)
+
+	detectWithinRackImbalance(volKey{"col1", 100}, topo.nodes, buildRacks(topo.nodes), "", 0, 10, 4, nil)
+
+	perHost := map[string]int{}
+	for _, n := range topo.nodes {
+		if info, ok := n.shards[volKey{"col1", 100}]; ok {
+			perHost[n.host] += info.shardBits.Count()
+		}
+	}
+	if len(perHost) < 3 {
+		t.Fatalf("shards not spread across machines: %v", perHost)
+	}
+	for h, c := range perHost {
+		if c > 4 {
+			t.Errorf("machine %s holds %d shards, want <=4 (ceil(14/4), within parity)", h, c)
+		}
+	}
+}
+
+// TestWithinRackMachineSpreadBalancesCombinedOccupancy: a 5+5 volume on two machines
+// where each shard type is already within its own per-type cap (boxA 3 data + 3
+// parity = 6, boxB 2 data + 2 parity = 4). Spreading data and parity independently
+// would leave boxA at 6, past parity; balancing the combined count must move one
+// shard to reach 5/5 so a machine loss stays recoverable.
+func TestWithinRackMachineSpreadBalancesCombinedOccupancy(t *testing.T) {
+	topo := NewTopology()
+	mk := func(id, host string) *Node {
+		n := topo.AddNode(id, "dc1", "dc1:rack1", 100)
+		n.SetHost(host)
+		n.AddDisk(0, "", 100, 0)
+		return n
+	}
+	a1 := mk("a1", "boxA")
+	a2 := mk("a2", "boxA") // two nodes on boxA -> numMachines(2) < numNodes(3)
+	b1 := mk("b1", "boxB")
+	a1.AddShards(100, "col1", 0, bits(0, 1, 2))    // 3 data
+	a2.AddShards(100, "col1", 0, bits(5, 6, 7))    // 3 parity (ids >= 5)
+	b1.AddShards(100, "col1", 0, bits(3, 4, 8, 9)) // 2 data + 2 parity
+
+	// dataShards=5, parityShards=5: feasibility ceil(10/2)=5 <= 5, so machine spread runs.
+	detectWithinRackImbalance(volKey{"col1", 100}, topo.nodes, buildRacks(topo.nodes), "", 0, 5, 5, nil)
+
+	perHost := map[string]int{}
+	for _, n := range topo.nodes {
+		if info, ok := n.shards[volKey{"col1", 100}]; ok {
+			perHost[n.host] += info.shardBits.Count()
+		}
+	}
+	if perHost["boxA"] > 5 {
+		t.Errorf("boxA holds %d shards, want <=5 (parity); data and parity spread independently", perHost["boxA"])
+	}
+}
+
+// TestWithinRackMachineSpreadActsOnExactlyHalfSkew: a 5/4/3 machine layout for a 10+4
+// volume is only 50% skewed ((5-3)/4 = 0.5), which a 0.5 relative-imbalance threshold
+// would skip -- but the 5-shard machine is already past parity. The spread must act
+// regardless (the even cap, not a skew threshold, is the bound) and bring every
+// machine to <=ceil(12/3)=4.
+func TestWithinRackMachineSpreadActsOnExactlyHalfSkew(t *testing.T) {
+	topo := NewTopology()
+	mk := func(id, host string) *Node {
+		n := topo.AddNode(id, "dc1", "dc1:rack1", 100)
+		n.SetHost(host)
+		n.AddDisk(0, "", 100, 0)
+		return n
+	}
+	a1 := mk("a1", "boxA")
+	a2 := mk("a2", "boxA")
+	b1 := mk("b1", "boxB")
+	b2 := mk("b2", "boxB")
+	c1 := mk("c1", "boxC")
+	c2 := mk("c2", "boxC")
+	a1.AddShards(100, "col1", 0, bits(0, 1, 2))
+	a2.AddShards(100, "col1", 0, bits(3, 4)) // boxA = 5
+	b1.AddShards(100, "col1", 0, bits(5, 6))
+	b2.AddShards(100, "col1", 0, bits(7, 8)) // boxB = 4
+	c1.AddShards(100, "col1", 0, bits(9, 10))
+	c2.AddShards(100, "col1", 0, bits(11)) // boxC = 3
+
+	detectWithinRackImbalance(volKey{"col1", 100}, topo.nodes, buildRacks(topo.nodes), "", 0, 10, 4, nil)
+
+	perHost := map[string]int{}
+	for _, n := range topo.nodes {
+		if info, ok := n.shards[volKey{"col1", 100}]; ok {
+			perHost[n.host] += info.shardBits.Count()
+		}
+	}
+	for h, c := range perHost {
+		if c > 4 {
+			t.Errorf("machine %s holds %d shards, want <=4 (ceil(12/3)); a 50%% skew was skipped", h, c)
+		}
+	}
+}
+
+// TestWithinRackSpreadUsesRackLocalShardCount: a rack holds only 7 shards of a 10+4
+// volume (cross-rack spreading moved the rest to other racks). Two machines can hold
+// those 7 within parity (ceil(7/2)=4), so machine spread must apply -- gating on the
+// full 14-shard total would wrongly fall back to node spread and pile 6 onto boxA's
+// two nodes, exceeding parity.
+func TestWithinRackSpreadUsesRackLocalShardCount(t *testing.T) {
+	topo := NewTopology()
+	mk := func(id, host string) *Node {
+		n := topo.AddNode(id, "dc1", "dc1:rack1", 100)
+		n.SetHost(host)
+		n.AddDisk(0, "", 100, 0)
+		return n
+	}
+	a1 := mk("a1", "boxA")
+	a2 := mk("a2", "boxA")
+	mk("b1", "boxB")
+	a1.AddShards(100, "col1", 0, bits(0, 1, 2, 3))
+	a2.AddShards(100, "col1", 0, bits(4, 5, 6)) // boxA holds all 7 of this rack's shards
+
+	detectWithinRackImbalance(volKey{"col1", 100}, topo.nodes, buildRacks(topo.nodes), "", 0, 10, 4, nil)
+
+	perHost := map[string]int{}
+	for _, n := range topo.nodes {
+		if info, ok := n.shards[volKey{"col1", 100}]; ok {
+			perHost[n.host] += info.shardBits.Count()
+		}
+	}
+	if perHost["boxA"] > 4 {
+		t.Errorf("boxA holds %d shards, want <=4 (parity); rack-local feasibility not used", perHost["boxA"])
+	}
+	if perHost["boxB"] == 0 {
+		t.Error("no shards moved to boxB; machine spread did not apply")
+	}
+}
+
+// TestWithinRackNodeFallbackHonorsThreshold: one server per host with a 6/4 data
+// split is the cosmetic node fallback (a 2-node rack can't be machine-fault-tolerant
+// for a 10+4 volume). At 40% skew it's below the 0.5 threshold, so it must be left to
+// the global utilization phase rather than churned -- a count move here can worsen
+// utilization on unequal-capacity nodes.
+func TestWithinRackNodeFallbackHonorsThreshold(t *testing.T) {
+	topo := NewTopology()
+	mk := func(id string) *Node {
+		n := topo.AddNode(id, "dc1", "dc1:rack1", 100)
+		n.AddDisk(0, "", 100, 0)
+		return n
+	}
+	n1 := mk("n1") // distinct hosts (default host = id) -> node fallback
+	n2 := mk("n2")
+	n1.AddShards(100, "col1", 0, bits(0, 1, 2, 3, 4, 5)) // 6 data
+	n2.AddShards(100, "col1", 0, bits(6, 7, 8, 9))       // 4 data
+
+	moves := detectWithinRackImbalance(volKey{"col1", 100}, topo.nodes, buildRacks(topo.nodes), "", 0.5, 10, 4, nil)
+	if len(moves) != 0 {
+		t.Errorf("node fallback moved %d shards on a 40%%-skewed 6/4 layout at threshold 0.5; want 0", len(moves))
+	}
+}
+
+// TestWithinRackSpreadDefaultsToNodes: with no SetHost each node is its own
+// machine, so the within-rack phase still spreads a volume off an overloaded node
+// exactly as before (machine grouping reduces to node grouping).
+func TestWithinRackSpreadDefaultsToNodes(t *testing.T) {
+	topo := NewTopology()
+	mk := func(id string) *Node {
+		n := topo.AddNode(id, "dc1", "dc1:rack1", 100)
+		n.AddDisk(0, "", 100, 0)
+		return n
+	}
+	n1 := mk("n1")
+	mk("n2")
+	mk("n3")
+	n1.AddShards(100, "col1", 0, allBits(14)) // all 14 piled on one node
+
+	detectWithinRackImbalance(volKey{"col1", 100}, topo.nodes, buildRacks(topo.nodes), "", 0, 10, 4, nil)
+
+	perNode := map[string]int{}
+	for id, n := range topo.nodes {
+		if info, ok := n.shards[volKey{"col1", 100}]; ok {
+			perNode[id] = info.shardBits.Count()
+		}
+	}
+	if perNode["n1"] == 14 {
+		t.Fatal("no within-rack spread happened with one server per host")
+	}
+	if len(perNode) < 3 {
+		t.Errorf("shards not spread across all three nodes: %v", perNode)
+	}
+	// Per-type even caps: ceil(10/3) data + ceil(4/3) parity = 4+2 per node.
+	for id, c := range perNode {
+		if c > 6 {
+			t.Errorf("node %s holds %d shards, want <=6 (dataCap+parityCap)", id, c)
+		}
+	}
+}
+
+// TestGlobalDoesNotConcentrateVolumeAcrossMachines: when the volume's machine spread
+// is achievable (here 2 machines, a 2+2 volume, one machine can hold <= parity), a
+// load move must not raise a machine's shard count of the volume past the source's.
+// boxA's only volume is also on boxB, so pass 0 finds nothing and the sole pass-1
+// option is the cross-machine boxA->boxB move, which must be rejected (boxB already
+// holds as many shards as boxA). Same-machine node rebalancing would still be fine.
+func TestGlobalDoesNotConcentrateVolumeAcrossMachines(t *testing.T) {
+	topo := NewTopology()
+	a1 := topo.AddNode("a1", "dc1", "dc1:rack1", 0) // full -> high util, the max node
+	a1.SetHost("boxA")
+	a1.AddShards(100, "col1", 0, bits(0, 1))
+	b1 := topo.AddNode("b1", "dc1", "dc1:rack1", 0) // full -> cannot receive
+	b1.SetHost("boxB")
+	b1.AddShards(100, "col1", 0, bits(2, 3))
+	b2 := topo.AddNode("b2", "dc1", "dc1:rack1", 10) // empty -> low util, the min node
+	b2.SetHost("boxB")
+
+	data := map[volKey]int{{collection: "col1", vid: 100}: 2}
+	parity := map[volKey]int{{collection: "col1", vid: 100}: 2}
+	for _, m := range detectGlobalImbalance(topo.nodes, buildRacks(topo.nodes), "", 0.01, data, parity, 0, true) {
+		if m.source.host != m.target.host {
+			t.Errorf("cross-machine global move %d.%d from %s to %s concentrates the volume on a machine",
+				m.volumeID, m.shardID, m.source.host, m.target.host)
+		}
+	}
+}
+
+// TestWithinRackMachineSkipsCappedMachine: a machine whose only node is already at
+// the per-node SameRackCount cap is not a viable target, and the within-rack spread
+// must move shards to the next machine that is, rather than picking the capped one
+// and skipping the move. Four machines (boxA has two nodes) make the spread active;
+// boxB's node holds two parity shards (== SameRackCount), so boxA's over-concentrated
+// data shards must land on the viable boxC/boxD.
+func TestWithinRackMachineSkipsCappedMachine(t *testing.T) {
+	rp, _ := super_block.NewReplicaPlacementFromString("002") // SameRackCount=2
+	topo := NewTopology()
+	mk := func(id, host string) *Node {
+		n := topo.AddNode(id, "dc1", "dc1:rack1", 100)
+		n.SetHost(host)
+		n.AddDisk(0, "", 100, 0)
+		return n
+	}
+	a1 := mk("a1", "boxA")
+	mk("a2", "boxA") // boxA's second node makes numMachines(4) < numNodes(5)
+	b1 := mk("b1", "boxB")
+	c1 := mk("c1", "boxC")
+	d1 := mk("d1", "boxD")
+	a1.AddShards(100, "col1", 0, bits(0, 1, 2, 3)) // 4 data shards, over-concentrated on boxA
+	b1.AddShards(100, "col1", 0, bits(10, 11))     // 2 parity -> b1 at SameRackCount cap
+
+	detectWithinRackImbalance(volKey{"col1", 100}, topo.nodes, buildRacks(topo.nodes), "", 0, 10, 4, rp)
+
+	moved := 0
+	for _, n := range []*Node{c1, d1} {
+		if info, ok := n.shards[volKey{"col1", 100}]; ok {
+			moved += info.shardBits.Count()
+		}
+	}
+	if moved == 0 {
+		t.Error("data shards were not moved to the viable machines boxC/boxD; the capped boxB blocked the move")
+	}
+}
+
+// TestGlobalPrefersVolumeAbsentFromDestinationMachine: the global load phase must
+// judge "volume already present" at the machine level. boxB's sibling node holds
+// vol100, so draining boxA onto boxB's empty node should move a vol200 shard first
+// (vol200 is absent from boxB) rather than piling a second vol100 copy onto boxB.
+func TestGlobalPrefersVolumeAbsentFromDestinationMachine(t *testing.T) {
+	topo := NewTopology()
+	n1 := topo.AddNode("n1", "dc1", "dc1:rack1", 0)
+	n1.SetHost("boxA")
+	n1.AddShards(100, "col1", 0, bits(0, 1, 2, 3))
+	n1.AddShards(200, "col1", 0, bits(0, 1, 2, 3))
+	n2 := topo.AddNode("n2", "dc1", "dc1:rack1", 5)
+	n2.SetHost("boxB")
+	n2.AddShards(100, "col1", 0, bits(4)) // vol100 already on boxB (sibling node)
+	n3 := topo.AddNode("n3", "dc1", "dc1:rack1", 5)
+	n3.SetHost("boxB") // empty destination node, but its machine holds vol100
+
+	moves := detectGlobalImbalance(topo.nodes, buildRacks(topo.nodes), "", 0.01, nil, nil, 0, true)
+	if len(moves) == 0 {
+		t.Fatal("expected a global move from the full node")
+	}
+	if moves[0].volumeID != 200 {
+		t.Errorf("first global move is volume %d, want 200 (vol100 already on the destination machine)", moves[0].volumeID)
+	}
+}

weed/storage/erasure_coding/ecbalancer/place.go

@@ -0,0 +1,579 @@
+package ecbalancer
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	storagetypes "github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// Constraints configures a Place call. Ratio resolves a collection's
+// (dataShards, parityShards); nil uses the standard scheme. ReplicaPlacement,
+// when non-nil, caps shards per rack (DiffRackCount = max shards/rack) and per
+// node within a rack (SameRackCount = max shards/node); both digits are direct
+// hard caps. The data-center digit (DiffDataCenterCount) is not honored:
+// the 1-byte volume ReplicaPlacement can only encode 0-2 there, too small to be a
+// meaningful per-DC EC shard cap, so EC relies on the rack/node even spread instead.
+//
+// DiskTypePolicy controls how DiskType constrains placement (Any / Prefer /
+// Require). PreferredTags drives whole-plan tag tiering: Place tries disks
+// carrying the earliest tags first and widens to all disks only if a tier cannot
+// place every shard.
+type Constraints struct {
+	DiskType         string
+	DiskTypePolicy   DiskTypePolicy
+	PreferredTags    []string
+	ReplicaPlacement *super_block.ReplicaPlacement
+	Ratio            func(collection string) (dataShards, parityShards int)
+}
+
+// DiskTypePolicy controls how Constraints.DiskType constrains placement.
+type DiskTypePolicy int
+
+const (
+	DiskTypeAny     DiskTypePolicy = iota // any disk type
+	DiskTypePrefer                        // prefer DiskType, spill to other types if needed
+	DiskTypeRequire                       // only DiskType (HardDriveType when "")
+)
+
+// diskTypeEqual compares disk types after normalization, so "" and "hdd" (both
+// HardDriveType) are equal.
+func diskTypeEqual(a, b string) bool {
+	return storagetypes.ToDiskType(a).String() == storagetypes.ToDiskType(b).String()
+}
+
+// diskHasAnyTag reports whether the disk carries any of the given tags.
+func diskHasAnyTag(d *disk, tags []string) bool {
+	for _, want := range tags {
+		for _, have := range d.tags {
+			if have == want {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// Destination is a chosen target for one shard. DataCenter and Rack are kept as
+// separate values (matching topology.DiskInfo) rather than a "dc:rack" composite,
+// so callers read them directly instead of parsing.
+type Destination struct {
+	Node       string
+	DiskID     uint32
+	DataCenter string
+	Rack       string // bare rack id within DataCenter
+}
+
+// PlaceResult holds the chosen destinations, which constraints had to be relaxed
+// (durability-first only), and whether placement spilled outside the preferred
+// disk type or tag tiers (for parity with today's logging).
+type PlaceResult struct {
+	Destinations                map[int]Destination
+	Relaxed                     []string
+	SpilledToOtherDiskType      bool
+	SpilledOutsidePreferredTags bool
+}
+
+// PlacementMode selects the strictness/relaxation policy.
+type PlacementMode int
+
+const (
+	// PlaceStrict: caps and ReplicaPlacement are hard. Place fails rather than
+	// violate them, so the caller can defer (leave the volume as-is and retry).
+	PlaceStrict PlacementMode = iota
+	// PlaceDurabilityFirst (used by both encode and repair): relax per-type caps ->
+	// data/parity anti-affinity -> ReplicaPlacement, in that order, until each shard
+	// lands, reporting what was relaxed in PlaceResult.Relaxed. The per-disk
+	// durability cap (<= parityShards per disk) is never relaxed. Fails only if no
+	// disk has free capacity. Encode places best-effort this way and rebalancing
+	// tightens the spread afterward.
+	PlaceDurabilityFirst
+)
+
+// relaxation controls which placement-quality constraints are enforced on an
+// attempt. preferring fresh nodes (repair's "avoid surviving-shard nodes") is not
+// listed: pickNodeInRack already selects the node with the fewest shards of the
+// volume, so survivors are deprioritized with built-in fallback.
+type relaxation struct {
+	caps         bool
+	antiAffinity bool
+	rp           bool
+}
+
+func (r relaxation) relaxedNames() []string {
+	var n []string
+	if !r.caps {
+		n = append(n, "caps")
+	}
+	if !r.antiAffinity {
+		n = append(n, "anti-affinity")
+	}
+	if !r.rp {
+		n = append(n, "replica-placement")
+	}
+	return n
+}
+
+var strictAttempts = []relaxation{{caps: true, antiAffinity: true, rp: true}}
+
+var durabilityAttempts = []relaxation{
+	{caps: true, antiAffinity: true, rp: true},
+	{caps: false, antiAffinity: true, rp: true},
+	{caps: false, antiAffinity: false, rp: true},
+	{caps: false, antiAffinity: false, rp: false},
+}
+
+type placedEntry struct {
+	node    *Node
+	sid     int
+	rackKey string
+}
+
+// Place assigns destinations for the `need` shard ids of volume (collection,vid),
+// reading the volume's already-placed shards from the snapshot (so encode passes
+// an empty-for-this-volume snapshot, repair passes one seeded with the surviving
+// shards).
+//
+// Tag tiering (whole-plan retry): it tries the preferred-tag tiers in order, each
+// a complete candidate set, and returns the first tier that places every shard;
+// only when it falls through to the no-tag tier does it set
+// SpilledOutsidePreferredTags. Within a tier, disk-type Prefer spills to other
+// types per shard (SpilledToOtherDiskType); Require filters strictly.
+func (t *Topology) Place(vid uint32, collection string, need []int, c Constraints, mode PlacementMode) (*PlaceResult, error) {
+	if len(need) == 0 {
+		return &PlaceResult{Destinations: map[int]Destination{}}, nil
+	}
+
+	vk := volKey{collection: collection, vid: vid}
+	dataShards, parityShards := erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount
+	if c.Ratio != nil {
+		if d, p := c.Ratio(collection); d > 0 && p > 0 {
+			dataShards, parityShards = d, p
+		}
+	}
+
+	racks := buildRacks(t.nodes)
+	if len(racks) == 0 {
+		return nil, fmt.Errorf("no racks available for EC placement")
+	}
+	rackKeys := sortedKeys(racks)
+
+	// Disk-type eligibility (Require filters; Any/Prefer admit all) and the soft
+	// type preference applied in scoring under Prefer.
+	typeEligible := func(d *disk) bool {
+		if c.DiskTypePolicy == DiskTypeRequire {
+			return diskTypeEqual(d.diskType, c.DiskType)
+		}
+		return true
+	}
+	var prefer func(*disk) bool
+	if c.DiskTypePolicy == DiskTypePrefer {
+		prefer = func(d *disk) bool { return diskTypeEqual(d.diskType, c.DiskType) }
+	}
+
+	// Whole-plan retry over preferred-tag tiers; the first tier that places every
+	// shard wins. Reaching the no-tag tier means we spilled outside the tags.
+	tiers := tagTiers(c.PreferredTags)
+	var lastErr error
+	for _, tierTags := range tiers {
+		tt := tierTags
+		eligible := func(d *disk) bool {
+			return typeEligible(d) && (len(tt) == 0 || diskHasAnyTag(d, tt))
+		}
+		res, err := t.tryPlace(vk, need, dataShards, parityShards, racks, rackKeys, mode, c.ReplicaPlacement, eligible, prefer)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		if len(c.PreferredTags) > 0 && len(tierTags) == 0 {
+			res.SpilledOutsidePreferredTags = true
+		}
+		return res, nil
+	}
+	return nil, lastErr
+}
+
+// tagTiers returns the eligibility tag-sets in increasing breadth, ending with an
+// empty set ("any disk"). Empty preferredTags yields a single any-disk tier.
+func tagTiers(preferredTags []string) [][]string {
+	if len(preferredTags) == 0 {
+		return [][]string{nil}
+	}
+	tiers := make([][]string, 0, len(preferredTags)+1)
+	for k := range preferredTags {
+		tiers = append(tiers, append([]string(nil), preferredTags[:k+1]...))
+	}
+	return append(tiers, nil)
+}
+
+// tryPlace runs one whole-plan placement attempt restricted to disks satisfying
+// `eligible`, with `prefer` (may be nil) ranking soft-preferred disks first. It
+// journals reservations and rolls them all back if any shard cannot be placed, so
+// a failed tier leaves the snapshot unchanged for the next attempt.
+func (t *Topology) tryPlace(vk volKey, need []int, dataShards, parityShards int, racks map[string]*rack, rackKeys []string, mode PlacementMode, rp *super_block.ReplicaPlacement, eligible func(*disk) bool, prefer func(*disk) bool) (*PlaceResult, error) {
+	result := &PlaceResult{Destinations: make(map[int]Destination, len(need))}
+
+	// Per-type shard ids per rack (even caps), total shard count per rack
+	// (DiffRackCount), and the racks bearing each type (anti-affinity) — all seeded
+	// from the volume's existing shards.
+	shardsPerRack := map[bool]map[string][]int{true: {}, false: {}}
+	rackShardCount := map[string]int{}
+	bearing := map[bool]map[string]bool{true: {}, false: {}}
+	for _, n := range t.nodes {
+		info, ok := n.shards[vk]
+		if !ok {
+			continue
+		}
+		for sid := range info.shardBits.All() {
+			s := int(sid)
+			isData := s < dataShards
+			shardsPerRack[isData][n.rack] = append(shardsPerRack[isData][n.rack], s)
+			rackShardCount[n.rack]++
+			bearing[isData][n.rack] = true
+		}
+	}
+
+	// Even per-rack caps divide by racks that actually have an eligible free disk,
+	// not all racks (the snapshot keeps every disk type/tag), so a valid tiered
+	// cluster — e.g. SSDs in only 2 of 4 racks — is not capped impossibly low.
+	numEligibleRacks := 0
+	for _, rk := range rackKeys {
+		if rackHasFreeDisk(racks[rk], eligible) {
+			numEligibleRacks++
+		}
+	}
+	if numEligibleRacks < 1 {
+		numEligibleRacks = 1
+	}
+
+	attempts := strictAttempts
+	if mode == PlaceDurabilityFirst {
+		attempts = durabilityAttempts
+	}
+
+	var journal []placedEntry
+	relaxedSeen := map[string]bool{}
+	spilledType := false
+
+	placeShard := func(sid int, isData bool) bool {
+		typeTotal := dataShards
+		if !isData {
+			typeTotal = parityShards
+		}
+		for _, rl := range attempts {
+			node, diskID, spilled, ok := chooseShardDest(vk, sid, isData, dataShards, typeTotal, numEligibleRacks, parityShards, racks, rackKeys, rp, eligible, prefer, shardsPerRack[isData], rackShardCount, bearing, rl)
+			if !ok {
+				continue
+			}
+			reserveShard(node, vk, sid, diskID)
+			node.freeSlots--
+			racks[node.rack].freeSlots--
+			shardsPerRack[isData][node.rack] = append(shardsPerRack[isData][node.rack], sid)
+			rackShardCount[node.rack]++
+			bearing[isData][node.rack] = true
+			journal = append(journal, placedEntry{node: node, sid: sid, rackKey: node.rack})
+			result.Destinations[sid] = Destination{
+				Node:       node.id,
+				DiskID:     diskID,
+				DataCenter: node.dc,
+				Rack:       strings.TrimPrefix(node.rack, node.dc+":"),
+			}
+			if spilled {
+				spilledType = true
+			}
+			for _, name := range rl.relaxedNames() {
+				relaxedSeen[name] = true
+			}
+			return true
+		}
+		return false
+	}
+
+	// Data shards first, then parity, so parity can avoid data-bearing racks.
+	for _, isData := range []bool{true, false} {
+		for _, sid := range shardsOfType(need, isData, dataShards) {
+			if placeShard(sid, isData) {
+				continue
+			}
+			for _, e := range journal {
+				releaseShard(e.node, vk, e.sid)
+				e.node.freeSlots++
+				racks[e.rackKey].freeSlots++
+			}
+			return nil, fmt.Errorf("cannot place EC shard %d of volume %d (collection %q)", sid, vk.vid, vk.collection)
+		}
+	}
+
+	result.SpilledToOtherDiskType = spilledType
+	for name := range relaxedSeen {
+		result.Relaxed = append(result.Relaxed, name)
+	}
+	sort.Strings(result.Relaxed)
+	return result, nil
+}
+
+// chooseShardDest selects a (node, disk) for one shard at the given relaxation
+// level: pick a rack (even per-type cap + ReplicaPlacement caps + two-pass
+// anti-affinity to the opposite type), then the least-loaded eligible node, then
+// the best eligible disk. The third return reports whether the disk spilled off
+// the soft-preferred type. ok=false when no rack/node/disk fits.
+func chooseShardDest(vk volKey, sid int, isData bool, dataShards, typeTotal, numEligibleRacks, maxPerDisk int, racks map[string]*rack, rackKeys []string, rp *super_block.ReplicaPlacement, eligible func(*disk) bool, prefer func(*disk) bool, shardsPerRackType map[string][]int, rackShardCount map[string]int, bearing map[bool]map[string]bool, rl relaxation) (*Node, uint32, bool, bool) {
+	maxPerRack := numEligibleRacks*typeTotal + 1 // effectively unlimited when caps are relaxed
+	if rl.caps {
+		if maxPerRack = ceilDivide(typeTotal, numEligibleRacks); maxPerRack < 1 {
+			maxPerRack = 1
+		}
+	}
+
+	var anti map[string]bool
+	if rl.antiAffinity {
+		anti = bearing[!isData] // racks already holding the opposite shard type
+	}
+
+	if !rl.rp {
+		rp = nil
+	}
+	// A rack is eligible only if it is under the per-rack shard cap (DiffRackCount),
+	// enforced only when set (and relaxed with rp).
+	withinLimit := func(r string) bool {
+		if rp == nil {
+			return true
+		}
+		if rp.DiffRackCount > 0 && rackShardCount[r] >= rp.DiffRackCount {
+			return false
+		}
+		return true
+	}
+
+	destRack, ok := pickTarget(rackKeys, shardsPerRackType, maxPerRack, anti,
+		func(r string) bool { return racks[r].freeSlots > 0 && rackHasFreeDisk(racks[r], eligible) },
+		withinLimit)
+	if !ok {
+		return nil, 0, false, false
+	}
+	node := pickNodeInRackEligible(racks[destRack], vk, rp, eligible)
+	if node == nil {
+		return nil, 0, false, false
+	}
+	diskID, ok, spilled := pickBestDiskEligible(node, vk, eligible, prefer, sid, dataShards, maxPerDisk)
+	if !ok {
+		return nil, 0, false, false
+	}
+	return node, diskID, spilled, true
+}
+
+// nodeHasFreeDisk reports whether the node has a free disk satisfying eligible.
+func nodeHasFreeDisk(n *Node, eligible func(*disk) bool) bool {
+	for _, d := range n.disks {
+		if d.freeSlots > 0 && eligible(d) {
+			return true
+		}
+	}
+	return false
+}
+
+// rackHasFreeDisk reports whether any node in the rack has a free eligible disk.
+func rackHasFreeDisk(r *rack, eligible func(*disk) bool) bool {
+	for _, n := range r.nodes {
+		if n.freeSlots > 0 && nodeHasFreeDisk(n, eligible) {
+			return true
+		}
+	}
+	return false
+}
+
+// pickNodeInRackEligible is pickNodeInRack restricted to nodes that have a free
+// eligible disk. FromActiveTopology keeps all disk types/tags in the snapshot, so
+// without this a node with free volume slots but no eligible disk could be chosen.
+//
+// Among eligible nodes it ranks by fewest shards of the volume per machine, then per
+// node, with free capacity breaking ties. The free-capacity tie-break (not sorted id)
+// keeps the lowest-id machine from winning every volume's first shard against the
+// shared encode snapshot and piling up load.
+func pickNodeInRackEligible(r *rack, vk volKey, rp *super_block.ReplicaPlacement, eligible func(*disk) bool) *Node {
+	machineShards := countShardsByHost(vk, r.nodes)
+	machineFree := freeSlotsByHost(r.nodes)
+	var best *Node
+	var bestMCount, bestMFree, bestNCount, bestNFree int
+	for _, id := range sortedNodeKeys(r.nodes) {
+		node := r.nodes[id]
+		if node.freeSlots <= 0 {
+			continue
+		}
+		if !nodeHasFreeDisk(node, eligible) {
+			continue
+		}
+		count := volumeShardCount(node, vk)
+		if rp != nil && rp.SameRackCount > 0 && count >= rp.SameRackCount {
+			continue
+		}
+		mCount, mFree := machineShards[node.host], machineFree[node.host]
+		better := false
+		switch {
+		case best == nil:
+			better = true
+		case mCount != bestMCount:
+			better = mCount < bestMCount
+		case mFree != bestMFree:
+			better = mFree > bestMFree
+		case count != bestNCount:
+			better = count < bestNCount
+		default:
+			better = node.freeSlots > bestNFree
+		}
+		if better {
+			best, bestMCount, bestMFree, bestNCount, bestNFree = node, mCount, mFree, count, node.freeSlots
+		}
+	}
+	return best
+}
+
+// pickBestDiskEligible chooses the best eligible disk on a node, ranking
+// soft-preferred disks (prefer != nil && prefer(d)) ahead of others so disk-type
+// Prefer uses the preferred type when available but spills otherwise. Returns the
+// disk id, whether one was found, and whether the chosen disk spilled off the
+// preferred type.
+func pickBestDiskEligible(node *Node, vk volKey, eligible func(*disk) bool, prefer func(*disk) bool, shardID, dataShardCount, maxPerDisk int) (uint32, bool, bool) {
+	isDataShard := dataShardCount > 0 && shardID < dataShardCount
+	info := node.shards[vk]
+	var bestDiskID uint32
+	bestScore := -1
+	bestPreferred := false
+	for _, diskID := range sortedDiskKeys(node.disks) {
+		d := node.disks[diskID]
+		if !eligible(d) || d.freeSlots <= 0 {
+			continue
+		}
+		existingShards := 0
+		hasData := false
+		hasParity := false
+		if info != nil {
+			bits := info.diskShardBits[diskID]
+			existingShards = bits.Count()
+			if dataShardCount > 0 {
+				for sid := range bits.All() {
+					if int(sid) < dataShardCount {
+						hasData = true
+					} else {
+						hasParity = true
+					}
+				}
+			}
+		}
+		// Durability: never put more than maxPerDisk (parityShards) shards of this
+		// volume on one disk, or losing that disk would lose more than EC can
+		// recover. Hard cap, enforced even under durability-first relaxation.
+		if maxPerDisk > 0 && existingShards >= maxPerDisk {
+			continue
+		}
+		score := d.shardCount*10 + existingShards*100
+		if dataShardCount > 0 {
+			if isDataShard && hasParity {
+				score += 1000
+			} else if !isDataShard && hasData {
+				score += 1000
+			}
+		}
+		preferred := prefer == nil || prefer(d)
+		if !preferred {
+			score += 100000 // strongly deprioritize spilling to a non-preferred type
+		}
+		if bestScore == -1 || score < bestScore {
+			bestScore = score
+			bestDiskID = diskID
+			bestPreferred = preferred
+		}
+	}
+	if bestScore == -1 {
+		return 0, false, false
+	}
+	return bestDiskID, true, prefer != nil && !bestPreferred
+}
+
+// clearShardAccounting removes one shard copy of a volume from the snapshot's
+// per-domain accounting (the volume's shard bits) WITHOUT crediting disk capacity.
+// It clears only the given physical disk's bit, then recomputes the node-level
+// union from the remaining disk bits, so a kept copy of the same shard on another
+// disk of the same node still counts toward caps / ReplicaPlacement / anti-affinity.
+//
+// Repair uses this to drop the duplicate/mismatched copies it plans to delete
+// before placing missing shards, so those copies do not inflate placement
+// accounting. Capacity is deliberately NOT credited: the deletes run only after
+// the rebuilt shards are distributed, so the slots are not free at plan time. This
+// is distinct from releaseShard, which credits freeSlots and clears the union.
+func clearShardAccounting(node *Node, vk volKey, shardID int, diskID uint32) {
+	info, ok := node.shards[vk]
+	if !ok {
+		return
+	}
+	sid := erasure_coding.ShardId(shardID)
+	if bits, ok := info.diskShardBits[diskID]; ok {
+		info.diskShardBits[diskID] = bits.Clear(sid)
+	}
+	var union erasure_coding.ShardBits
+	for _, b := range info.diskShardBits {
+		union |= b
+	}
+	info.shardBits = union
+}
+
+// ClearShardAccounting drops one shard copy of a volume from placement accounting
+// without crediting capacity (see clearShardAccounting). Repair calls it for each
+// copy it plans to delete before placing missing shards, so those copies do not
+// inflate caps/RP/anti-affinity. No-op for an unknown node.
+func (t *Topology) ClearShardAccounting(nodeID, collection string, vid uint32, shardID int, diskID uint32) {
+	n, ok := t.nodes[nodeID]
+	if !ok {
+		return
+	}
+	clearShardAccounting(n, volKey{collection: collection, vid: vid}, shardID, diskID)
+}
+
+// ReleaseVolumeShards removes every shard of a volume from the snapshot and
+// credits the freed disk capacity. A greenfield encode calls this so any stale
+// EC shards left by a prior failed attempt (which the encode task deletes before
+// distributing the new shards) neither occupy capacity nor skew anti-affinity /
+// per-disk caps during planning. Unlike repair's ClearShardAccounting, it credits
+// freeSlots because the deletes run before the new writes.
+func (t *Topology) ReleaseVolumeShards(collection string, vid uint32) {
+	vk := volKey{collection: collection, vid: vid}
+	for _, n := range t.nodes {
+		info, ok := n.shards[vk]
+		if !ok {
+			continue
+		}
+		// freed is the total disk-slots the volume occupies on this node (a shard may
+		// sit on more than one disk). releaseShard credits each disk's freeSlots;
+		// credit the node's freeSlots by the same total, since rack capacity is summed
+		// from node freeSlots (buildRacks) and node freeSlots gates node eligibility.
+		freed := 0
+		for _, bits := range info.diskShardBits {
+			freed += bits.Count()
+		}
+		sids := make([]int, 0, info.shardBits.Count())
+		for sid := range info.shardBits.All() {
+			sids = append(sids, int(sid))
+		}
+		for _, sid := range sids {
+			releaseShard(n, vk, sid)
+		}
+		n.freeSlots += freed
+		delete(n.shards, vk)
+	}
+}
+
+// shardsOfType returns the sorted subset of need that are data shards (id <
+// dataShards) when isData, else the parity subset.
+func shardsOfType(need []int, isData bool, dataShards int) []int {
+	var out []int
+	for _, s := range need {
+		if (s < dataShards) == isData {
+			out = append(out, s)
+		}
+	}
+	sort.Ints(out)
+	return out
+}

weed/storage/erasure_coding/ecbalancer/place_test.go

@@ -0,0 +1,502 @@
+package ecbalancer
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+)
+
+// buildPlaceTopo makes a topology of racks x nodesPerRack, each node one disk with
+// perDiskFree free EC shard slots.
+func buildPlaceTopo(racks, nodesPerRack, perDiskFree int) *Topology {
+	topo := NewTopology()
+	for r := 0; r < racks; r++ {
+		rackKey := fmt.Sprintf("dc1:rack%d", r)
+		for n := 0; n < nodesPerRack; n++ {
+			id := fmt.Sprintf("10.0.%d.%d:8080", r, n)
+			node := topo.AddNode(id, "dc1", rackKey, perDiskFree)
+			node.AddDisk(0, "", perDiskFree, 0)
+		}
+	}
+	return topo
+}
+
+func allShards() []int {
+	out := make([]int, erasure_coding.TotalShardsCount)
+	for i := range out {
+		out[i] = i
+	}
+	return out
+}
+
+// TestPlaceStrictSpreadAndCaps places a fresh 10+4 volume and checks every shard
+// lands on a distinct node and no rack exceeds the even per-type cap.
+func TestPlaceStrictSpreadAndCaps(t *testing.T) {
+	const racks = 4
+	topo := buildPlaceTopo(racks, 4, 50)
+
+	res, err := topo.Place(1, "c1", allShards(), Constraints{}, PlaceStrict)
+	if err != nil {
+		t.Fatalf("Place: %v", err)
+	}
+	if len(res.Destinations) != erasure_coding.TotalShardsCount {
+		t.Fatalf("placed %d shards, want %d", len(res.Destinations), erasure_coding.TotalShardsCount)
+	}
+
+	usedNodes := map[string]bool{}
+	dataPerRack := map[string]int{}
+	parityPerRack := map[string]int{}
+	for sid, d := range res.Destinations {
+		if usedNodes[d.Node] {
+			t.Errorf("node %s reused for shard %d (expected distinct nodes with ample capacity)", d.Node, sid)
+		}
+		usedNodes[d.Node] = true
+		if sid < erasure_coding.DataShardsCount {
+			dataPerRack[d.Rack]++
+		} else {
+			parityPerRack[d.Rack]++
+		}
+	}
+	dataCap := ceilDivide(erasure_coding.DataShardsCount, racks)
+	parityCap := ceilDivide(erasure_coding.ParityShardsCount, racks)
+	for rk, n := range dataPerRack {
+		if n > dataCap {
+			t.Errorf("rack %s holds %d data shards, cap %d", rk, n, dataCap)
+		}
+	}
+	for rk, n := range parityPerRack {
+		if n > parityCap {
+			t.Errorf("rack %s holds %d parity shards, cap %d", rk, n, parityCap)
+		}
+	}
+}
+
+// TestPlaceDistributesEvenlyAcrossManyVolumes: placing many volumes against the shared
+// encode snapshot must not pile shards onto the sorted-first machine.
+func TestPlaceDistributesEvenlyAcrossManyVolumes(t *testing.T) {
+	topo := NewTopology()
+	hosts := []string{"10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4", "10.0.0.5", "10.0.0.6"}
+	for _, h := range hosts {
+		for _, port := range []string{":8080", ":8081"} {
+			n := topo.AddNode(h+port, "dc1", "dc1:rack1", 1000)
+			n.SetHost(h)
+			n.AddDisk(0, "", 1000, 0)
+		}
+	}
+
+	const volumes = 120
+	for v := 0; v < volumes; v++ {
+		if _, err := topo.Place(uint32(v), "c1", allShards(), Constraints{}, PlaceDurabilityFirst); err != nil {
+			t.Fatalf("Place volume %d: %v", v, err)
+		}
+	}
+
+	perMachine := map[string]int{}
+	for _, n := range topo.nodes {
+		for _, d := range n.disks {
+			perMachine[n.host] += d.shardCount
+		}
+	}
+	min, max := 1<<30, 0
+	for _, c := range perMachine {
+		if c < min {
+			min = c
+		}
+		if c > max {
+			max = c
+		}
+	}
+	// Even distribution is 120*14/6 = 280 shards/machine. Catch a lopsided pileup
+	// (the production symptom was the first machine at ~1.7x the rest).
+	if float64(max) > 1.25*float64(min) {
+		t.Errorf("EC shards piled unevenly across machines: min=%d max=%d %v", min, max, perMachine)
+	}
+}
+
+// TestPlaceSpreadsAcrossMachines: one rack runs two physical machines (each with
+// four volume servers). Placing a 10+4 volume must spread its shards across both
+// machines so no single machine holds more than ceil(14/2)=7, otherwise losing one
+// box would take out more shards than EC can recover even though they look spread
+// across distinct nodes.
+func TestPlaceSpreadsAcrossMachines(t *testing.T) {
+	topo := NewTopology()
+	host := map[string]string{}
+	add := func(id, h string) {
+		n := topo.AddNode(id, "dc1", "dc1:rack0", 50)
+		n.SetHost(h)
+		n.AddDisk(0, "", 50, 0)
+		host[id] = h
+	}
+	for i := 0; i < 4; i++ {
+		add(fmt.Sprintf("a%d", i), "10.0.0.1")
+		add(fmt.Sprintf("b%d", i), "10.0.0.2")
+	}
+
+	res, err := topo.Place(1, "c1", allShards(), Constraints{}, PlaceStrict)
+	if err != nil {
+		t.Fatalf("Place: %v", err)
+	}
+	perMachine := map[string]int{}
+	for _, d := range res.Destinations {
+		perMachine[host[d.Node]]++
+	}
+	if len(perMachine) < 2 {
+		t.Fatalf("shards not spread across machines: %v", perMachine)
+	}
+	maxPerMachine := ceilDivide(erasure_coding.TotalShardsCount, 2)
+	for h, c := range perMachine {
+		if c > maxPerMachine {
+			t.Errorf("machine %s holds %d shards, want <=%d (ceil(14/2))", h, c, maxPerMachine)
+		}
+	}
+}
+
+// TestPlaceStrictFailsAndRollsBack: a single tiny disk cannot hold 14 shards, so
+// strict Place fails and leaves the snapshot untouched.
+func TestPlaceStrictFailsAndRollsBack(t *testing.T) {
+	topo := buildPlaceTopo(1, 1, 2) // one node, room for 2 shards
+	node := topo.nodes["10.0.0.0:8080"]
+	freeBefore := node.freeSlots
+	diskFreeBefore := node.disks[0].freeSlots
+
+	_, err := topo.Place(1, "c1", allShards(), Constraints{}, PlaceStrict)
+	if err == nil {
+		t.Fatal("expected Place to fail on insufficient capacity")
+	}
+	if info, ok := node.shards[volKey{collection: "c1", vid: 1}]; ok && info.shardBits.Count() != 0 {
+		t.Errorf("volume shard bits left on node after failed strict Place (rollback incomplete): %b", info.shardBits)
+	}
+	if node.freeSlots != freeBefore {
+		t.Errorf("node freeSlots = %d after rollback, want %d", node.freeSlots, freeBefore)
+	}
+	if node.disks[0].freeSlots != diskFreeBefore {
+		t.Errorf("disk freeSlots = %d after rollback, want %d", node.disks[0].freeSlots, diskFreeBefore)
+	}
+}
+
+// TestPlaceDurabilityFirstRelaxesRP: a ReplicaPlacement rack limit too tight for
+// the shard count makes strict fail, while durability-first relaxes RP to place
+// everything and reports the relaxation.
+func TestPlaceDurabilityFirstRelaxesRP(t *testing.T) {
+	rp := &super_block.ReplicaPlacement{DiffRackCount: 3} // <=3 shards per rack
+	topo := buildPlaceTopo(2, 8, 50)                      // 2 racks: 2*3=6 < 14 under RP
+
+	if _, err := topo.Place(1, "c1", allShards(), Constraints{ReplicaPlacement: rp}, PlaceStrict); err == nil {
+		t.Fatal("strict Place should fail when RP rack limit cannot fit all shards")
+	}
+
+	topo = buildPlaceTopo(2, 8, 50)
+	res, err := topo.Place(1, "c1", allShards(), Constraints{ReplicaPlacement: rp}, PlaceDurabilityFirst)
+	if err != nil {
+		t.Fatalf("durability-first Place: %v", err)
+	}
+	if len(res.Destinations) != erasure_coding.TotalShardsCount {
+		t.Fatalf("placed %d shards, want %d", len(res.Destinations), erasure_coding.TotalShardsCount)
+	}
+	relaxedRP := false
+	for _, r := range res.Relaxed {
+		if r == "replica-placement" {
+			relaxedRP = true
+		}
+	}
+	if !relaxedRP {
+		t.Errorf("expected replica-placement relaxation, got %v", res.Relaxed)
+	}
+}
+
+// TestPlaceSameRackCountIsDirectPerNodeCap: the 3rd ReplicaPlacement digit
+// (SameRackCount) caps shards per node directly (max == digit), matching the
+// per-rack DiffRackCount cap rather than allowing digit+1 per node.
+func TestPlaceSameRackCountIsDirectPerNodeCap(t *testing.T) {
+	rp := &super_block.ReplicaPlacement{SameRackCount: 2} // <=2 shards per node
+
+	// 5 single-node racks: 5 nodes * 2 = 10 < 14, so a strict 10+4 placement
+	// cannot satisfy the per-node cap and must fail. Under the old digit+1 reading
+	// the cap would be 3/node => 15 slots and this would have wrongly succeeded.
+	topo := buildPlaceTopo(5, 1, 50)
+	if _, err := topo.Place(1, "c1", allShards(), Constraints{ReplicaPlacement: rp}, PlaceStrict); err == nil {
+		t.Fatal("strict Place should fail: 5 nodes cannot hold 14 shards at <=2 per node")
+	}
+
+	// Durability-first relaxes the unsatisfiable per-node cap, still places every
+	// shard, and reports the relaxation so it isn't silently weakened.
+	topo = buildPlaceTopo(5, 1, 50)
+	res, err := topo.Place(1, "c1", allShards(), Constraints{ReplicaPlacement: rp}, PlaceDurabilityFirst)
+	if err != nil {
+		t.Fatalf("durability-first Place: %v", err)
+	}
+	if len(res.Destinations) != erasure_coding.TotalShardsCount {
+		t.Fatalf("placed %d shards, want %d", len(res.Destinations), erasure_coding.TotalShardsCount)
+	}
+	relaxedRP := false
+	for _, r := range res.Relaxed {
+		if r == "replica-placement" {
+			relaxedRP = true
+		}
+	}
+	if !relaxedRP {
+		t.Errorf("expected replica-placement relaxation, got %v", res.Relaxed)
+	}
+}
+
+// TestPlaceDiskTypeHardFilter: with DiskType set, shards land only on disks of
+// that type, even though the snapshot also contains other-typed disks.
+func TestPlaceDiskTypeHardFilter(t *testing.T) {
+	topo := NewTopology()
+	for r := 0; r < 4; r++ {
+		rackKey := fmt.Sprintf("dc1:rack%d", r)
+		ssd := topo.AddNode(fmt.Sprintf("ssd-%d:8080", r), "dc1", rackKey, 50)
+		ssd.AddDisk(0, "ssd", 50, 0)
+		hdd := topo.AddNode(fmt.Sprintf("hdd-%d:8080", r), "dc1", rackKey, 50)
+		hdd.AddDisk(0, "hdd", 50, 0)
+	}
+
+	res, err := topo.Place(1, "c1", allShards(), Constraints{DiskType: "ssd", DiskTypePolicy: DiskTypeRequire}, PlaceStrict)
+	if err != nil {
+		t.Fatalf("Place ssd: %v", err)
+	}
+	for sid, d := range res.Destinations {
+		node := topo.nodes[d.Node]
+		disk := node.disks[d.DiskID]
+		if disk == nil || disk.diskType != "ssd" {
+			t.Errorf("shard %d placed on non-ssd disk: node=%s diskID=%d", sid, d.Node, d.DiskID)
+		}
+	}
+}
+
+// TestPlaceDiskTypeUnavailableFails: a request for a disk type with no matching
+// disks fails rather than silently placing on the wrong tier.
+func TestPlaceDiskTypeUnavailableFails(t *testing.T) {
+	topo := NewTopology()
+	for r := 0; r < 4; r++ {
+		n := topo.AddNode(fmt.Sprintf("hdd-%d:8080", r), "dc1", fmt.Sprintf("dc1:rack%d", r), 50)
+		n.AddDisk(0, "hdd", 50, 0)
+	}
+	if _, err := topo.Place(1, "c1", allShards(), Constraints{DiskType: "ssd", DiskTypePolicy: DiskTypeRequire}, PlaceStrict); err == nil {
+		t.Fatal("expected Place to fail when no disks of the requested type exist")
+	}
+}
+
+// TestPlaceHDDRequestMatchesEmptyTypeDisks: a "hdd" request normalizes to
+// HardDriveType ("") and must land on the HDD disk (reported as ""), never the SSD
+// disk, even on nodes that have both.
+func TestPlaceHDDRequestMatchesEmptyTypeDisks(t *testing.T) {
+	topo := NewTopology()
+	for r := 0; r < 6; r++ {
+		n := topo.AddNode(fmt.Sprintf("n%d:8080", r), "dc1", fmt.Sprintf("dc1:rack%d", r), 100)
+		n.AddDisk(0, "", 50, 0)    // HDD (HardDriveType, reported as "")
+		n.AddDisk(1, "ssd", 50, 0) // SSD
+	}
+
+	res, err := topo.Place(1, "c1", allShards(), Constraints{DiskType: "hdd", DiskTypePolicy: DiskTypeRequire}, PlaceStrict)
+	if err != nil {
+		t.Fatalf("Place hdd: %v", err)
+	}
+	for sid, d := range res.Destinations {
+		if d.DiskID != 0 { // disk 0 is the HDD disk on every node
+			t.Errorf("shard %d placed on disk %d (expected HDD disk 0) on node %s", sid, d.DiskID, d.Node)
+		}
+	}
+}
+
+// TestPlaceDurabilityCapRejectsSkewed: in a near-full cluster where only one disk
+// has spare room, Place must not pile more than parityShards shards onto it (losing
+// it would then lose more than EC can recover). It fails instead, so the caller
+// leaves the volume unencoded rather than minting an unrecoverable layout.
+func TestPlaceDurabilityCapRejectsSkewed(t *testing.T) {
+	topo := NewTopology()
+	// One spacious node plus four nearly-full ones, all in a single rack.
+	a := topo.AddNode("a:8080", "dc1", "dc1:rack0", 100)
+	a.AddDisk(0, "", 100, 0)
+	for i := 0; i < 4; i++ {
+		n := topo.AddNode(fmt.Sprintf("b%d:8080", i), "dc1", "dc1:rack0", 1)
+		n.AddDisk(0, "", 1, 0)
+	}
+
+	// 14 shards, parity 4: node a is capped at 4, the others hold 1 each -> at most
+	// 4+4=8 placeable without exceeding parityShards on a disk, so Place must fail.
+	// (Without the per-disk cap, a would greedily absorb 10 shards and "succeed".)
+	if _, err := topo.Place(1, "c1", allShards(), Constraints{}, PlaceDurabilityFirst); err == nil {
+		t.Fatal("expected Place to fail rather than pile >parityShards shards on one disk")
+	}
+}
+
+// TestReleaseVolumeShards: removes all of a volume's shards from the snapshot and
+// credits the freed capacity at BOTH disk and node level (rack capacity sums node
+// freeSlots), mirroring how FromActiveTopology accounts stale shards.
+func TestReleaseVolumeShards(t *testing.T) {
+	topo := NewTopology()
+	// Node total 20 = two disks of 10. Two stale shards occupy one slot each, so the
+	// snapshot would show disk0/disk1 at 9 and node at 18 (as FromActiveTopology does).
+	n := topo.AddNode("n0:8080", "dc1", "dc1:rack0", 20)
+	n.AddDisk(0, "", 10, 0)
+	n.AddDisk(1, "", 10, 0)
+	vk := volKey{collection: "c1", vid: 1}
+	n.AddShards(1, "c1", 0, erasure_coding.ShardBits(uint32(1)<<3))
+	n.AddShards(1, "c1", 1, erasure_coding.ShardBits(uint32(1)<<7))
+	n.disks[0].freeSlots = 9
+	n.disks[1].freeSlots = 9
+	n.freeSlots = 18
+
+	topo.ReleaseVolumeShards("c1", 1)
+
+	if _, ok := n.shards[vk]; ok {
+		t.Error("volume shards should be gone after ReleaseVolumeShards")
+	}
+	if n.disks[0].freeSlots != 10 || n.disks[1].freeSlots != 10 {
+		t.Errorf("disk freeSlots not restored: disk0=%d, disk1=%d (want 10, 10)", n.disks[0].freeSlots, n.disks[1].freeSlots)
+	}
+	if n.freeSlots != 20 {
+		t.Errorf("node freeSlots = %d, want 20 (must be credited at node level too)", n.freeSlots)
+	}
+}
+
+// TestClearShardAccounting: dropping one disk's copy of a shard preserves a kept
+// copy of the same shard on another disk of the same node, and credits no capacity.
+func TestClearShardAccounting(t *testing.T) {
+	topo := NewTopology()
+	n := topo.AddNode("n0:8080", "dc1", "dc1:rack0", 50)
+	n.AddDisk(0, "", 50, 0)
+	n.AddDisk(1, "", 50, 0)
+	vk := volKey{collection: "c1", vid: 1}
+	// Shard 3 lives on disk 0 (keep) and disk 1 (duplicate to delete).
+	n.AddShards(1, "c1", 0, erasure_coding.ShardBits(uint32(1)<<3))
+	n.AddShards(1, "c1", 1, erasure_coding.ShardBits(uint32(1)<<3))
+	if got := n.shards[vk].shardBits.Count(); got != 1 {
+		t.Fatalf("union count = %d, want 1", got)
+	}
+	freeBefore := n.disks[1].freeSlots
+
+	clearShardAccounting(n, vk, 3, 1)
+
+	if !n.shards[vk].shardBits.Has(erasure_coding.ShardId(3)) {
+		t.Error("kept copy of shard 3 (disk 0) lost from the node-level union")
+	}
+	if n.shards[vk].diskShardBits[1].Has(erasure_coding.ShardId(3)) {
+		t.Error("disk-1 copy of shard 3 was not cleared")
+	}
+	if n.disks[1].freeSlots != freeBefore {
+		t.Errorf("freeSlots changed %d -> %d; clearShardAccounting must not credit capacity", freeBefore, n.disks[1].freeSlots)
+	}
+}
+
+// TestPlaceDiskTypePreferSpills: DiskTypePrefer fills the preferred type first and
+// spills the remainder to other types, reporting SpilledToOtherDiskType. SSD is
+// scarce (one tiny SSD per node) so the volume must spill to HDD, but there are
+// enough disks to keep each within the parityShards durability cap.
+func TestPlaceDiskTypePreferSpills(t *testing.T) {
+	topo := NewTopology()
+	for r := 0; r < 8; r++ {
+		n := topo.AddNode(fmt.Sprintf("n%d:8080", r), "dc1", fmt.Sprintf("dc1:rack%d", r), 100)
+		n.AddDisk(0, "ssd", 1, 0) // tiny SSD: 1 shard
+		n.AddDisk(1, "", 50, 0)   // roomy HDD
+	}
+
+	res, err := topo.Place(1, "c1", allShards(), Constraints{DiskType: "ssd", DiskTypePolicy: DiskTypePrefer}, PlaceDurabilityFirst)
+	if err != nil {
+		t.Fatalf("Place: %v", err)
+	}
+	if len(res.Destinations) != erasure_coding.TotalShardsCount {
+		t.Fatalf("placed %d, want %d", len(res.Destinations), erasure_coding.TotalShardsCount)
+	}
+	ssd, hdd := 0, 0
+	for _, d := range res.Destinations {
+		if topo.nodes[d.Node].disks[d.DiskID].diskType == "ssd" {
+			ssd++
+		} else {
+			hdd++
+		}
+	}
+	if ssd == 0 || hdd == 0 {
+		t.Errorf("expected prefer-then-spill: some shards on SSD and some on HDD, got ssd=%d hdd=%d", ssd, hdd)
+	}
+	if !res.SpilledToOtherDiskType {
+		t.Error("expected SpilledToOtherDiskType when SSD cannot hold every shard")
+	}
+}
+
+// TestPlacePreferredTagsUseTaggedDisks: when tagged disks can hold the whole plan,
+// every shard lands on a tagged disk and no spill is reported.
+func TestPlacePreferredTagsUseTaggedDisks(t *testing.T) {
+	topo := NewTopology()
+	for r := 0; r < 4; r++ {
+		rackKey := fmt.Sprintf("dc1:rack%d", r)
+		fast := topo.AddNode(fmt.Sprintf("fast-%d:8080", r), "dc1", rackKey, 50)
+		fast.AddDisk(0, "", 50, 0)
+		fast.AddDiskTags(0, []string{"fast"})
+		topo.AddNode(fmt.Sprintf("slow-%d:8080", r), "dc1", rackKey, 50).AddDisk(0, "", 50, 0)
+	}
+
+	res, err := topo.Place(1, "c1", allShards(), Constraints{PreferredTags: []string{"fast"}}, PlaceStrict)
+	if err != nil {
+		t.Fatalf("Place: %v", err)
+	}
+	for sid, d := range res.Destinations {
+		if !diskHasAnyTag(topo.nodes[d.Node].disks[d.DiskID], []string{"fast"}) {
+			t.Errorf("shard %d placed on an untagged disk (node %s)", sid, d.Node)
+		}
+	}
+	if res.SpilledOutsidePreferredTags {
+		t.Error("did not expect tag spill when tagged disks suffice")
+	}
+}
+
+// TestPlacePreferredTagsSpillWhenInsufficient: when the tagged tier cannot hold the
+// whole plan, Place falls back to all disks and reports SpilledOutsidePreferredTags.
+func TestPlacePreferredTagsSpillWhenInsufficient(t *testing.T) {
+	topo := NewTopology()
+	for r := 0; r < 4; r++ {
+		n := topo.AddNode(fmt.Sprintf("n-%d:8080", r), "dc1", fmt.Sprintf("dc1:rack%d", r), 50)
+		if r == 0 {
+			n.AddDisk(0, "", 5, 0) // the only tagged disk, too small for 14 shards
+			n.AddDiskTags(0, []string{"fast"})
+		} else {
+			n.AddDisk(0, "", 50, 0)
+		}
+	}
+
+	res, err := topo.Place(1, "c1", allShards(), Constraints{PreferredTags: []string{"fast"}}, PlaceStrict)
+	if err != nil {
+		t.Fatalf("Place: %v", err)
+	}
+	if len(res.Destinations) != erasure_coding.TotalShardsCount {
+		t.Fatalf("placed %d, want %d", len(res.Destinations), erasure_coding.TotalShardsCount)
+	}
+	if !res.SpilledOutsidePreferredTags {
+		t.Error("expected SpilledOutsidePreferredTags when the single fast disk cannot hold the plan")
+	}
+}
+
+// TestPlaceStrictCapsCountEligibleRacks: with DiskTypeRequire, the even per-rack
+// cap divides by racks that have a matching disk, not all racks, so SSDs in only
+// some racks still place successfully.
+func TestPlaceStrictCapsCountEligibleRacks(t *testing.T) {
+	topo := NewTopology()
+	// SSDs live in only 2 of 4 racks, with several SSD nodes per rack so 14 shards
+	// fit at <= parityShards per disk. The even per-rack cap must divide by the 2
+	// eligible racks (ceil(10/2)=5 data/rack), not all 4 (ceil(10/4)=3 -> infeasible).
+	for r := 0; r < 4; r++ {
+		rackKey := fmt.Sprintf("dc1:rack%d", r)
+		topo.AddNode(fmt.Sprintf("hdd-%d:8080", r), "dc1", rackKey, 50).AddDisk(0, "", 50, 0)
+		if r < 2 {
+			for n := 0; n < 4; n++ {
+				topo.AddNode(fmt.Sprintf("ssd-%d-%d:8080", r, n), "dc1", rackKey, 50).AddDisk(0, "ssd", 50, 0)
+			}
+		}
+	}
+
+	res, err := topo.Place(1, "c1", allShards(), Constraints{DiskType: "ssd", DiskTypePolicy: DiskTypeRequire}, PlaceStrict)
+	if err != nil {
+		t.Fatalf("Place ssd in 2/4 racks: %v", err)
+	}
+	if len(res.Destinations) != erasure_coding.TotalShardsCount {
+		t.Fatalf("placed %d, want %d", len(res.Destinations), erasure_coding.TotalShardsCount)
+	}
+	for sid, d := range res.Destinations {
+		if disk := topo.nodes[d.Node].disks[d.DiskID]; disk == nil || disk.diskType != "ssd" {
+			t.Errorf("shard %d not on an SSD disk: node=%s disk=%d", sid, d.Node, d.DiskID)
+		}
+	}
+}

weed/storage/erasure_coding/ecbalancer/shard_ratio.go

@@ -0,0 +1,24 @@
+package ecbalancer
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+// shardDataShards returns the data-shard count of the volume an EC shard belongs
+// to, used to size the shard's disk footprint. OSS uses the standard ratio for
+// every volume; custom per-volume ratios are an enterprise feature, so the
+// enterprise build overrides this to read the per-shard ratio.
+func shardDataShards(eci *master_pb.VolumeEcShardInformationMessage) int {
+	return erasure_coding.DataShardsCount
+}
+
+// VolumeShardRatio returns the RAW per-volume (dataShards, parityShards) reported
+// on an EC shard's heartbeat, with 0 meaning "not reported". Custom per-volume
+// ratios are an enterprise feature and the OSS proto has no data_shards/parity_shards
+// fields, so this returns 0, 0 and the balancer falls back to the collection ratio
+// (the standard scheme). The enterprise build overrides this to read the per-shard
+// ratio so a mixed-ratio collection is spread by each volume's own data/parity split.
+func VolumeShardRatio(eci *master_pb.VolumeEcShardInformationMessage) (dataShards, parityShards int) {
+	return 0, 0
+}

weed/storage/erasure_coding/ecbalancer/snapshot.go

@@ -0,0 +1,132 @@
+package ecbalancer
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/admin/topology"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/topology/balancer"
+)
+
+// diskTooFull reports whether a disk's physical filesystem is at/above the
+// balancing high-water mark, making it ineligible as an EC-shard target even
+// when slot math says it has room. Statfs free bytes already include EC shard
+// files. No-opinion when the disk reports no bytes (slot-only fallback).
+func diskTooFull(d *topology.DiskInfo) bool {
+	return balancer.DiskTooFullAfter(d.DiskInfo.DiskTotalBytes, d.DiskInfo.DiskFreeBytes, 0, balancer.DefaultMaxDiskUsagePercent)
+}
+
+// FromActiveTopology builds a Topology snapshot from the cluster's ActiveTopology
+// using the reservation-aware effective-capacity view that EC encode and repair
+// rely on. It collects ALL EC-eligible disks with no hard disk-type filter;
+// disk-type preference is applied later by callers (Place). Per-disk free EC shard
+// slots come from GetEffectiveAvailableEcShardSlots (shard-granular, so in-flight
+// task reservations that are not whole-volume multiples are not lost) minus the EC
+// shards already persisted on the disk. Rack keys are composite "dc:rack".
+//
+// dataShards is the target collection's data-shard count, used to size free EC
+// shard slots correctly for custom ratios (a 4+2 volume's shards are larger, so
+// fewer fit per volume slot). Pass <= 0 for the default scheme. Because of this,
+// the snapshot is ratio-specific; build one per collection ratio being placed.
+//
+// This is the encode/repair-side constructor; balance keeps its own raw-topology
+// builder (buildBalancerTopology) until the snapshot sources are reconciled.
+func FromActiveTopology(at *topology.ActiveTopology, dataShards int) *Topology {
+	topo := NewTopology()
+	if at == nil {
+		return topo
+	}
+
+	disks := at.GetDisksWithEffectiveCapacity(topology.TaskTypeErasureCoding, "", 0)
+
+	// Accumulate node-level free slots and group the node's disks together.
+	nodeFree := make(map[string]int)
+	nodeDC := make(map[string]string)
+	nodeRack := make(map[string]string)
+	nodeAddr := make(map[string]string)
+	byNode := make(map[string][]*topology.DiskInfo)
+	for _, d := range disks {
+		if d == nil || d.DiskInfo == nil {
+			continue
+		}
+		if free := perDiskFreeECSlots(at, d, dataShards); free > 0 && !diskTooFull(d) {
+			nodeFree[d.NodeID] += free
+		}
+		nodeDC[d.NodeID] = d.DataCenter
+		nodeRack[d.NodeID] = d.DataCenter + ":" + d.Rack
+		nodeAddr[d.NodeID] = d.Address
+		byNode[d.NodeID] = append(byNode[d.NodeID], d)
+	}
+
+	for nodeID, ds := range byNode {
+		node := topo.AddNode(nodeID, nodeDC[nodeID], nodeRack[nodeID], nodeFree[nodeID])
+		// Group by physical machine: derive the host from the address (NodeID may be
+		// an opaque id), falling back to the id when no address is recorded.
+		addr := nodeAddr[nodeID]
+		if addr == "" {
+			addr = nodeID
+		}
+		node.SetHost(pb.ServerAddress(addr).ToHost())
+		for _, d := range ds {
+			free := perDiskFreeECSlots(at, d, dataShards)
+			if free < 0 || diskTooFull(d) {
+				free = 0
+			}
+			node.AddDisk(d.DiskID, d.DiskType, free, ecShardCountOnDisk(d))
+			node.AddDiskTags(d.DiskID, d.DiskInfo.Tags)
+			for _, eci := range d.DiskInfo.EcShardInfos {
+				if eci.DiskId != d.DiskID {
+					continue
+				}
+				node.AddShards(eci.Id, eci.Collection, d.DiskID, erasure_coding.ShardBits(eci.EcIndexBits))
+			}
+		}
+	}
+
+	return topo
+}
+
+// perDiskFreeECSlots returns the disk's free EC shard slots for a volume with
+// dataShards data shards: the topology's shard-granular effective availability
+// (sized by the target ratio) minus the EC shards already on the disk, also
+// expressed in the target ratio's shard slots so mixed-ratio disks are charged by
+// size rather than raw count.
+func perDiskFreeECSlots(at *topology.ActiveTopology, d *topology.DiskInfo, dataShards int) int {
+	return at.GetEffectiveAvailableEcShardSlots(d.NodeID, d.DiskID, dataShards) - ecShardSlotsOnDisk(d, dataShards)
+}
+
+// ecShardCountOnDisk counts the EC shards physically on this disk (matching
+// eci.DiskId), across all volumes. Used as a per-disk load metric for scoring.
+func ecShardCountOnDisk(d *topology.DiskInfo) int {
+	count := 0
+	for _, eci := range d.DiskInfo.EcShardInfos {
+		if eci.DiskId == d.DiskID {
+			count += erasure_coding.GetShardCount(eci)
+		}
+	}
+	return count
+}
+
+// ecShardSlotsOnDisk returns the EC shards already on the disk expressed in the
+// TARGET collection's shard slots. A shard of a collection with d data shards
+// occupies ~1/d of a volume, i.e. targetDataShards/d target slots, so a 2+1 shard
+// counted against a 10+4 snapshot consumes ~5 slots, not 1.
+func ecShardSlotsOnDisk(d *topology.DiskInfo, targetDataShards int) int {
+	if targetDataShards <= 0 {
+		targetDataShards = erasure_coding.DataShardsCount
+	}
+	total := 0
+	for _, eci := range d.DiskInfo.EcShardInfos {
+		if eci.DiskId != d.DiskID {
+			continue
+		}
+		ds := shardDataShards(eci)
+		if ds <= 0 {
+			ds = erasure_coding.DataShardsCount
+		}
+		// Round up so an existing shard always consumes at least its fractional
+		// footprint; flooring lets a low-data-shard volume (targetDataShards < ds)
+		// count as zero target slots and overstate the disk's free capacity.
+		total += (erasure_coding.GetShardCount(eci)*targetDataShards + ds - 1) / ds
+	}
+	return total
+}

weed/storage/erasure_coding/ecbalancer/snapshot_test.go

@@ -0,0 +1,226 @@
+package ecbalancer
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/admin/topology"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+// TestFromActiveTopology verifies the encode/repair-side snapshot constructor maps
+// nodes, per-disk EC shard counts, per-volume shard bits, and free slots from an
+// ActiveTopology. Shard accounting is asserted exactly; free slots are asserted to
+// be positive (their exact value depends on effective-capacity internals).
+func TestFromActiveTopology(t *testing.T) {
+	const vid uint32 = 7
+	at := topology.NewActiveTopology(10)
+
+	// Node A holds one EC shard (id 3) of volume 7 on disk 0; node B is empty.
+	nodeA := &master_pb.DataNodeInfo{
+		Id: "10.0.0.1:8080",
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"hdd": {
+				DiskId:         0,
+				MaxVolumeCount: 100,
+				VolumeCount:    1,
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{{
+					Id:          vid,
+					Collection:  "c1",
+					EcIndexBits: uint32(1) << 3,
+					DiskId:      0,
+				}},
+			},
+		},
+	}
+	nodeB := &master_pb.DataNodeInfo{
+		Id: "10.0.0.2:8080",
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"hdd": {DiskId: 0, MaxVolumeCount: 100, VolumeCount: 0},
+		},
+	}
+	if err := at.UpdateTopology(&master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{{
+			Id: "dc1",
+			RackInfos: []*master_pb.RackInfo{{
+				Id:            "rack1",
+				DataNodeInfos: []*master_pb.DataNodeInfo{nodeA, nodeB},
+			}},
+		}},
+	}); err != nil {
+		t.Fatalf("UpdateTopology: %v", err)
+	}
+
+	topo := FromActiveTopology(at, 0)
+
+	if got := len(topo.nodes); got != 2 {
+		t.Fatalf("node count = %d, want 2", got)
+	}
+
+	a := topo.nodes["10.0.0.1:8080"]
+	if a == nil {
+		t.Fatal("node A missing from snapshot")
+	}
+	if a.rack != "dc1:rack1" {
+		t.Errorf("node A rack = %q, want dc1:rack1", a.rack)
+	}
+	diskA := a.disks[0]
+	if diskA == nil {
+		t.Fatal("node A disk 0 missing")
+	}
+	if diskA.shardCount != 1 {
+		t.Errorf("node A disk 0 shardCount = %d, want 1", diskA.shardCount)
+	}
+	vs := a.shards[volKey{collection: "c1", vid: vid}]
+	if vs == nil {
+		t.Fatal("volume 7 shards not recorded on node A")
+	}
+	if !vs.shardBits.Has(erasure_coding.ShardId(3)) {
+		t.Errorf("node A volume 7 shardBits %b missing shard 3", vs.shardBits)
+	}
+	if vs.shardBits.Count() != 1 {
+		t.Errorf("node A volume 7 shard count = %d, want 1", vs.shardBits.Count())
+	}
+
+	b := topo.nodes["10.0.0.2:8080"]
+	if b == nil {
+		t.Fatal("node B missing from snapshot")
+	}
+	if b.rack != "dc1:rack1" {
+		t.Errorf("node B rack = %q, want dc1:rack1", b.rack)
+	}
+	if diskB := b.disks[0]; diskB == nil || diskB.shardCount != 0 {
+		t.Errorf("node B disk 0 shardCount = %v, want 0", diskB)
+	}
+	if len(b.shards) != 0 {
+		t.Errorf("node B should hold no volume shards, got %d", len(b.shards))
+	}
+
+	// Free slots should be positive on both near-empty disks.
+	if a.freeSlots <= 0 || b.freeSlots <= 0 {
+		t.Errorf("free slots not positive: A=%d B=%d", a.freeSlots, b.freeSlots)
+	}
+}
+
+// A physically near-full disk must contribute zero free EC slots to the snapshot,
+// even though its slot math (MaxVolumeCount - VolumeCount) says it has room, so
+// the planner never places shards onto it. Mirrors the volume-balance #10160 gate.
+func TestFromActiveTopologySkipsPhysicallyFullDisk(t *testing.T) {
+	const gb = uint64(1) << 30
+	at := topology.NewActiveTopology(10)
+
+	// Both disks look slot-empty; only their physical byte fullness differs.
+	full := &master_pb.DataNodeInfo{
+		Id: "10.0.0.1:8080",
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"hdd": {DiskId: 0, MaxVolumeCount: 100, VolumeCount: 0, DiskTotalBytes: 1000 * gb, DiskFreeBytes: 40 * gb}, // 96% used
+		},
+	}
+	empty := &master_pb.DataNodeInfo{
+		Id: "10.0.0.2:8080",
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"hdd": {DiskId: 0, MaxVolumeCount: 100, VolumeCount: 0, DiskTotalBytes: 1000 * gb, DiskFreeBytes: 900 * gb}, // 10% used
+		},
+	}
+	if err := at.UpdateTopology(&master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{{
+			Id:        "dc1",
+			RackInfos: []*master_pb.RackInfo{{Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{full, empty}}},
+		}},
+	}); err != nil {
+		t.Fatalf("UpdateTopology: %v", err)
+	}
+
+	topo := FromActiveTopology(at, 0)
+
+	fullNode := topo.nodes["10.0.0.1:8080"]
+	if fullNode == nil {
+		t.Fatal("full node missing")
+	}
+	if fullNode.freeSlots != 0 {
+		t.Errorf("physically full node freeSlots = %d, want 0", fullNode.freeSlots)
+	}
+	if d := fullNode.disks[0]; d == nil || d.freeSlots != 0 {
+		t.Errorf("physically full disk freeSlots = %v, want 0", d)
+	}
+
+	emptyNode := topo.nodes["10.0.0.2:8080"]
+	if emptyNode == nil || emptyNode.freeSlots <= 0 {
+		t.Errorf("physically empty node should keep free slots, got %v", emptyNode)
+	}
+}
+
+// TestFromActiveTopologyGroupsByAddressHost verifies the snapshot derives a node's
+// machine from its address, not its (possibly opaque) id: two volume servers with
+// distinct ids but the same host must land on one machine so EC placement spreads
+// shards across boxes even when explicit node ids are configured.
+func TestFromActiveTopologyGroupsByAddressHost(t *testing.T) {
+	at := topology.NewActiveTopology(10)
+	mk := func(id, addr string) *master_pb.DataNodeInfo {
+		return &master_pb.DataNodeInfo{
+			Id:        id,
+			Address:   addr,
+			DiskInfos: map[string]*master_pb.DiskInfo{"hdd": {DiskId: 0, MaxVolumeCount: 100, VolumeCount: 0}},
+		}
+	}
+	// vs-a and vs-b are two servers on the same physical host; vs-c is another box.
+	nodes := []*master_pb.DataNodeInfo{
+		mk("vs-a", "10.0.0.9:8080"),
+		mk("vs-b", "10.0.0.9:8081"),
+		mk("vs-c", "10.0.0.10:8080"),
+	}
+	if err := at.UpdateTopology(&master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{{
+			Id:        "dc1",
+			RackInfos: []*master_pb.RackInfo{{Id: "rack1", DataNodeInfos: nodes}},
+		}},
+	}); err != nil {
+		t.Fatalf("UpdateTopology: %v", err)
+	}
+
+	topo := FromActiveTopology(at, 0)
+	if topo.nodes["vs-a"].host != topo.nodes["vs-b"].host {
+		t.Errorf("vs-a host %q != vs-b host %q; same-machine servers not grouped",
+			topo.nodes["vs-a"].host, topo.nodes["vs-b"].host)
+	}
+	if topo.nodes["vs-a"].host == topo.nodes["vs-c"].host {
+		t.Errorf("vs-a and vs-c share host %q but are different machines", topo.nodes["vs-a"].host)
+	}
+	if got := topo.nodes["vs-a"].host; got != "10.0.0.9" {
+		t.Errorf("vs-a host = %q, want 10.0.0.9", got)
+	}
+}
+
+// TestEcShardSlotsOnDiskRoundsUp covers the mixed-ratio (targetDataShards <
+// existingDataShards) conversion: an existing shard's fractional footprint must
+// round up so it is never floored to zero, which would overstate free capacity.
+// OSS always uses the standard ratio at runtime, but ecShardSlotsOnDisk takes the
+// target data-shard count as a parameter, so the fractional path is exercised
+// directly here; the enterprise build reaches it with real per-volume ratios.
+func TestEcShardSlotsOnDiskRoundsUp(t *testing.T) {
+	// A single shard (id 3) of a standard 10-data-shard volume on disk 0.
+	disk := &topology.DiskInfo{
+		DiskID: 0,
+		DiskInfo: &master_pb.DiskInfo{
+			DiskId: 0,
+			EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{{
+				Id:          7,
+				Collection:  "c1",
+				EcIndexBits: uint32(1) << 3,
+				DiskId:      0,
+			}},
+		},
+	}
+
+	// Against a 2-data-shard target the shard occupies 2/10 of a slot, which must
+	// round up to 1 rather than floor to 0.
+	if got := ecShardSlotsOnDisk(disk, 2); got != 1 {
+		t.Errorf("ecShardSlotsOnDisk(target=2) = %d, want 1 (rounded up from 0.2)", got)
+	}
+
+	// Identity case: target equals the existing data-shard count, so the shard
+	// consumes exactly its whole-number footprint.
+	if got := ecShardSlotsOnDisk(disk, erasure_coding.DataShardsCount); got != 1 {
+		t.Errorf("ecShardSlotsOnDisk(target=%d) = %d, want 1", erasure_coding.DataShardsCount, got)
+	}
+}

weed/storage/erasure_coding/shard_distribution.go

@@ -0,0 +1,393 @@
+package erasure_coding
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"reflect"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	"google.golang.org/grpc"
+)
+
+func ensureLogger(logger logger) logger {
+	if logger == nil {
+		return &glogFallbackLogger{}
+	}
+	return logger
+}
+
+type logger interface {
+	Info(string, ...interface{})
+	Warning(string, ...interface{})
+	Error(string, ...interface{})
+}
+
+type withFieldLogger interface {
+	WithFields(map[string]interface{}) logger
+}
+
+func withFields(log logger, fields map[string]interface{}) logger {
+	if fields == nil {
+		return log
+	}
+	if wf, ok := log.(withFieldLogger); ok {
+		return wf.WithFields(fields)
+	}
+
+	val := reflect.ValueOf(log)
+	method := val.MethodByName("WithFields")
+	if !method.IsValid() {
+		return log
+	}
+	methodType := method.Type()
+	if methodType.NumIn() != 1 || methodType.NumOut() != 1 {
+		return log
+	}
+	argType := methodType.In(0)
+	mapType := reflect.TypeOf(map[string]interface{}{})
+	var arg reflect.Value
+	if mapType.AssignableTo(argType) {
+		arg = reflect.ValueOf(fields)
+	} else if mapType.ConvertibleTo(argType) {
+		arg = reflect.ValueOf(fields).Convert(argType)
+	} else {
+		return log
+	}
+	result := method.Call([]reflect.Value{arg})[0]
+	if result.IsValid() && result.CanInterface() {
+		if enhanced, ok := result.Interface().(logger); ok {
+			return enhanced
+		}
+	}
+	return log
+}
+
+type glogFallbackLogger struct{}
+
+func (g *glogFallbackLogger) Info(msg string, args ...interface{}) {
+	if len(args) > 0 {
+		glog.Infof(msg, args...)
+		return
+	}
+	glog.Info(msg)
+}
+
+func (g *glogFallbackLogger) Warning(msg string, args ...interface{}) {
+	if len(args) > 0 {
+		glog.Warningf(msg, args...)
+		return
+	}
+	glog.Warning(msg)
+}
+
+func (g *glogFallbackLogger) Error(msg string, args ...interface{}) {
+	if len(args) > 0 {
+		glog.Errorf(msg, args...)
+		return
+	}
+	glog.Error(msg)
+}
+
+// DistributeEcShards distributes locally generated EC shards to destination servers.
+// Returns the shard assignment map used for mounting.
+func DistributeEcShards(volumeID uint32, collection string, targets []*worker_pb.TaskTarget, shardFiles map[string]string, dialOption grpc.DialOption, logger logger) (map[string][]string, error) {
+	if len(targets) == 0 {
+		return nil, fmt.Errorf("no targets specified for EC shard distribution")
+	}
+
+	if len(shardFiles) == 0 {
+		return nil, fmt.Errorf("no shard files available for distribution")
+	}
+
+	log := ensureLogger(logger)
+
+	shardAssignment := make(map[string][]string)
+	// node → shardType → planner-assigned diskID. Metadata files (ecx/ecj/vif)
+	// inherit the first data shard's disk so they land next to the shards.
+	shardDisks := make(map[string]map[string]uint32)
+
+	for _, target := range targets {
+		if len(target.ShardIds) == 0 {
+			continue
+		}
+
+		var assignedShards []string
+		for _, shardId := range target.ShardIds {
+			shardType := fmt.Sprintf("ec%02d", shardId)
+			assignedShards = append(assignedShards, shardType)
+		}
+
+		if len(assignedShards) > 0 {
+			if _, hasEcx := shardFiles["ecx"]; hasEcx {
+				assignedShards = append(assignedShards, "ecx")
+			}
+			if _, hasEcj := shardFiles["ecj"]; hasEcj {
+				assignedShards = append(assignedShards, "ecj")
+			}
+			if _, hasVif := shardFiles["vif"]; hasVif {
+				assignedShards = append(assignedShards, "vif")
+			}
+			if _, hasEcsum := shardFiles["ecsum"]; hasEcsum {
+				assignedShards = append(assignedShards, "ecsum")
+			}
+		}
+
+		if shardDisks[target.Node] == nil {
+			shardDisks[target.Node] = make(map[string]uint32)
+		}
+		for _, shardType := range assignedShards {
+			if _, already := shardDisks[target.Node][shardType]; !already {
+				shardDisks[target.Node][shardType] = target.DiskId
+			}
+		}
+
+		existing := shardAssignment[target.Node]
+		if len(existing) == 0 {
+			shardAssignment[target.Node] = assignedShards
+			continue
+		}
+
+		seen := make(map[string]struct{}, len(existing))
+		for _, shard := range existing {
+			seen[shard] = struct{}{}
+		}
+		for _, shard := range assignedShards {
+			if _, ok := seen[shard]; ok {
+				continue
+			}
+			seen[shard] = struct{}{}
+			existing = append(existing, shard)
+		}
+		shardAssignment[target.Node] = existing
+	}
+
+	if len(shardAssignment) == 0 {
+		return nil, fmt.Errorf("no shard assignments found from planning phase")
+	}
+
+	for destNode, assignedShards := range shardAssignment {
+		withFields(log, map[string]interface{}{
+			"destination":     destNode,
+			"assigned_shards": len(assignedShards),
+			"shard_types":     assignedShards,
+		}).Info("Starting shard distribution to destination server")
+
+		var transferredBytes int64
+		for _, shardType := range assignedShards {
+			filePath, exists := shardFiles[shardType]
+			if !exists {
+				return nil, fmt.Errorf("shard file %s not found for destination %s", shardType, destNode)
+			}
+
+			if info, err := os.Stat(filePath); err == nil {
+				transferredBytes += info.Size()
+				withFields(log, map[string]interface{}{
+					"destination": destNode,
+					"shard_type":  shardType,
+					"file_path":   filePath,
+					"size_bytes":  info.Size(),
+					"size_kb":     float64(info.Size()) / 1024,
+				}).Info("Starting shard file transfer")
+			}
+
+			diskID := uint32(0)
+			if byShard, ok := shardDisks[destNode]; ok {
+				diskID = byShard[shardType]
+			}
+			if err := sendShardFileToDestination(volumeID, collection, dialOption, destNode, diskID, filePath, shardType); err != nil {
+				return nil, fmt.Errorf("failed to send %s to %s: %w", shardType, destNode, err)
+			}
+
+			withFields(log, map[string]interface{}{
+				"destination": destNode,
+				"shard_type":  shardType,
+			}).Info("Shard file transfer completed")
+		}
+
+		withFields(log, map[string]interface{}{
+			"destination":        destNode,
+			"shards_transferred": len(assignedShards),
+			"total_bytes":        transferredBytes,
+			"total_mb":           float64(transferredBytes) / (1024 * 1024),
+		}).Info("All shards distributed to destination server")
+	}
+
+	glog.V(1).Infof("Successfully distributed EC shards to %d destinations", len(shardAssignment))
+	return shardAssignment, nil
+}
+
+// MountEcShards mounts EC shards on destination servers using an assignment map.
+// sourceDiskType is forwarded to VolumeEcShardsMount so the resulting EC volume
+// reports under the source's disk type rather than the destination's (#9423).
+func MountEcShards(volumeID uint32, collection string, shardAssignment map[string][]string, sourceDiskType string, dialOption grpc.DialOption, logger logger) error {
+	if shardAssignment == nil {
+		return fmt.Errorf("shard assignment not available for mounting")
+	}
+
+	log := ensureLogger(logger)
+
+	var mountErrors []error
+	for destNode, assignedShards := range shardAssignment {
+		var shardIds []uint32
+		var metadataFiles []string
+
+		for _, shardType := range assignedShards {
+			if strings.HasPrefix(shardType, "ec") && len(shardType) == 4 {
+				var shardId uint32
+				if _, err := fmt.Sscanf(shardType[2:], "%d", &shardId); err == nil {
+					shardIds = append(shardIds, shardId)
+				}
+			} else {
+				metadataFiles = append(metadataFiles, shardType)
+			}
+		}
+
+		withFields(log, map[string]interface{}{
+			"destination":    destNode,
+			"shard_ids":      shardIds,
+			"shard_count":    len(shardIds),
+			"metadata_files": metadataFiles,
+		}).Info("Starting EC shard mount operation")
+
+		if len(shardIds) == 0 {
+			withFields(log, map[string]interface{}{
+				"destination":    destNode,
+				"metadata_files": metadataFiles,
+			}).Info("No EC shards to mount (only metadata files)")
+			continue
+		}
+
+		err := operation.WithVolumeServerClient(false, pb.ServerAddress(destNode), dialOption,
+			func(client volume_server_pb.VolumeServerClient) error {
+				_, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
+					VolumeId:       volumeID,
+					Collection:     collection,
+					ShardIds:       shardIds,
+					SourceDiskType: sourceDiskType,
+				})
+				return mountErr
+			})
+
+		if err != nil {
+			mountErrors = append(mountErrors, fmt.Errorf("mount %s shards %v: %w", destNode, shardIds, err))
+			withFields(log, map[string]interface{}{
+				"destination": destNode,
+				"shard_ids":   shardIds,
+				"error":       err.Error(),
+			}).Error("Failed to mount EC shards")
+		} else {
+			withFields(log, map[string]interface{}{
+				"destination": destNode,
+				"shard_ids":   shardIds,
+				"volume_id":   volumeID,
+				"collection":  collection,
+			}).Info("Successfully mounted EC shards")
+		}
+	}
+
+	if len(mountErrors) > 0 {
+		return errors.Join(mountErrors...)
+	}
+	return nil
+}
+
+// diskID=0 leaves disk placement to the server's auto-select.
+func sendShardFileToDestination(volumeID uint32, collection string, dialOption grpc.DialOption, destServer string, diskID uint32, filePath, shardType string) error {
+	return operation.WithVolumeServerClient(false, pb.ServerAddress(destServer), dialOption,
+		func(client volume_server_pb.VolumeServerClient) error {
+			file, err := os.Open(filePath)
+			if err != nil {
+				return fmt.Errorf("failed to open shard file %s: %v", filePath, err)
+			}
+			defer file.Close()
+
+			fileInfo, err := file.Stat()
+			if err != nil {
+				return fmt.Errorf("failed to get file info for %s: %v", filePath, err)
+			}
+
+			var ext string
+			var shardId uint32
+			if shardType == "ecx" {
+				ext = ".ecx"
+				shardId = 0
+			} else if shardType == "ecj" {
+				ext = ".ecj"
+				shardId = 0
+			} else if shardType == "vif" {
+				ext = ".vif"
+				shardId = 0
+			} else if shardType == "ecsum" {
+				ext = BitrotSidecarExt
+				shardId = 0
+			} else if strings.HasPrefix(shardType, "ec") && len(shardType) == 4 {
+				ext = "." + shardType
+				fmt.Sscanf(shardType[2:], "%d", &shardId)
+			} else {
+				return fmt.Errorf("unknown shard type: %s", shardType)
+			}
+
+			stream, err := client.ReceiveFile(context.Background())
+			if err != nil {
+				return fmt.Errorf("failed to create receive stream: %v", err)
+			}
+
+			err = stream.Send(&volume_server_pb.ReceiveFileRequest{
+				Data: &volume_server_pb.ReceiveFileRequest_Info{
+					Info: &volume_server_pb.ReceiveFileInfo{
+						VolumeId:   volumeID,
+						Ext:        ext,
+						Collection: collection,
+						IsEcVolume: true,
+						ShardId:    shardId,
+						FileSize:   uint64(fileInfo.Size()),
+						DiskId:     diskID,
+					},
+				},
+			})
+			if err != nil {
+				return fmt.Errorf("failed to send file info: %v", err)
+			}
+
+			buffer := make([]byte, 64*1024)
+			for {
+				n, readErr := file.Read(buffer)
+				if n > 0 {
+					err = stream.Send(&volume_server_pb.ReceiveFileRequest{
+						Data: &volume_server_pb.ReceiveFileRequest_FileContent{
+							FileContent: buffer[:n],
+						},
+					})
+					if err != nil {
+						return fmt.Errorf("failed to send file content: %v", err)
+					}
+				}
+				if readErr == io.EOF {
+					break
+				}
+				if readErr != nil {
+					return fmt.Errorf("failed to read file: %v", readErr)
+				}
+			}
+
+			resp, err := stream.CloseAndRecv()
+			if err != nil {
+				return fmt.Errorf("failed to close stream: %v", err)
+			}
+
+			if resp.Error != "" {
+				return fmt.Errorf("server error: %s", resp.Error)
+			}
+
+			glog.V(2).Infof("Successfully sent %s (%d bytes) to %s", shardType, resp.BytesWritten, destServer)
+			return nil
+		})
+}

weed/storage/erasure_coding/verification.go

@@ -0,0 +1,234 @@
+package erasure_coding
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"google.golang.org/grpc"
+)
+
+type ServerShardInventory struct {
+	Bits       ShardBits
+	QueryError error
+	// BlockSize is the shard block layout this holder reports for the volume —
+	// the one it will serve reads through. A binary predating the uniform
+	// layout drops the field off the wire and reports 0, which is how
+	// RequireAgreedBlockLayout tells such a holder from one that agrees.
+	BlockSize int64
+}
+
+// Query errors are recorded per-server and treated as zero shards rather
+// than aborting the scan, so the caller still sees partial coverage from
+// healthy peers when one server is down. The caller gates destructive
+// actions on RequireRecoverableShardSet against the returned union.
+func VerifyShardsAcrossServers(ctx context.Context, volumeID uint32,
+	servers []string, dialOption grpc.DialOption) (
+	union ShardBits, perServer map[string]ServerShardInventory) {
+
+	perServer = make(map[string]ServerShardInventory, len(servers))
+
+	for _, server := range servers {
+		if server == "" {
+			continue
+		}
+		if _, seen := perServer[server]; seen {
+			continue
+		}
+
+		var inv ServerShardInventory
+
+		callErr := operation.WithVolumeServerClient(false, pb.ServerAddress(server), dialOption,
+			func(client volume_server_pb.VolumeServerClient) error {
+				resp, e := client.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+					VolumeId: volumeID,
+				})
+				if e != nil {
+					return e
+				}
+				for _, s := range resp.EcShardInfos {
+					if s.VolumeId != volumeID || s.ShardId >= MaxShardCount {
+						continue
+					}
+					inv.Bits = inv.Bits.Set(ShardId(s.ShardId))
+				}
+				inv.BlockSize = resp.GetEcShardConfig().GetBlockSize()
+				return nil
+			})
+		if callErr != nil {
+			inv.QueryError = callErr
+		}
+
+		perServer[server] = inv
+		union = ShardBits(uint32(union) | uint32(inv.Bits))
+	}
+
+	return union, perServer
+}
+
+// RequireRecoverableShardSet gates source-volume deletion after EC encode:
+// a non-empty .dat may only be deleted when enough distinct shards exist to
+// reconstruct the volume (>= dataShards). A full set returns (false, nil); a
+// degraded-but-recoverable set returns (true, nil) so the caller can warn and
+// proceed -- the missing shards can be rebuilt from the survivors, while
+// keeping the source next to live shards is the more dangerous mixed state.
+// Below dataShards it returns an error and the source must be kept.
+// dataShards/totalShards are passed as parameters (not derived from the
+// package constants) so enterprise builds with custom EC ratios share this
+// helper verbatim.
+func RequireRecoverableShardSet(volumeID uint32, shardsPresent ShardBits, dataShards, totalShards int) (degraded bool, err error) {
+	if totalShards <= 0 || totalShards > MaxShardCount {
+		return false, fmt.Errorf("invalid totalShards %d for volume %d (must be in [1, %d])",
+			totalShards, volumeID, MaxShardCount)
+	}
+	if dataShards <= 0 || dataShards > totalShards {
+		return false, fmt.Errorf("invalid dataShards %d for volume %d (must be in [1, %d])",
+			dataShards, volumeID, totalShards)
+	}
+	var missing []int
+	for id := 0; id < totalShards; id++ {
+		if !shardsPresent.Has(ShardId(id)) {
+			missing = append(missing, id)
+		}
+	}
+	if len(missing) == 0 {
+		return false, nil
+	}
+	if totalShards-len(missing) >= dataShards {
+		return true, nil
+	}
+	sort.Ints(missing)
+	return false, fmt.Errorf("EC shard set unrecoverable for volume %d: %d/%d shards present, need %d to reconstruct, missing shard ids %v",
+		volumeID, totalShards-len(missing), totalShards, dataShards, missing)
+}
+
+// RequireAgreedBlockLayout gates source-volume deletion on every holder that
+// answered agreeing with the layout the shards were encoded under.
+//
+// The uniform block layout lives in a `.vif` field that older volume servers
+// have never heard of: such a server parses the file, silently discards the
+// unknown field, and mounts the volume on the legacy 1GiB/1MiB striping. Its
+// reads then land at the wrong shard offsets and return wrong bytes, and
+// nothing else in the encode path notices — the shard files are the same
+// length under either layout. Asking each holder which layout it is serving is
+// the one question that separates the two, and asking it here means the answer
+// arrives while the source .dat is still on disk.
+//
+// Holders that could not be reached, or that report no shard of this volume,
+// are skipped: RequireRecoverableShardSet already covers a missing holder, and
+// one that answered nothing has promised nothing.
+func RequireAgreedBlockLayout(volumeID uint32, encodedBlockSize int64, perServer map[string]ServerShardInventory) error {
+	disagreeing := make([]string, 0, len(perServer))
+	for server, inv := range perServer {
+		if inv.QueryError != nil || inv.Bits.Count() == 0 {
+			continue
+		}
+		if inv.BlockSize != encodedBlockSize {
+			disagreeing = append(disagreeing, fmt.Sprintf("%s serves block size %d", server, inv.BlockSize))
+		}
+	}
+	if len(disagreeing) == 0 {
+		return nil
+	}
+	sort.Strings(disagreeing)
+	return fmt.Errorf("volume %d was encoded with shard block size %d but %v; upgrade every volume server to a build that understands the uniform shard block layout before encoding",
+		volumeID, encodedBlockSize, disagreeing)
+}
+
+func SummarizeShardInventory(perServer map[string]ServerShardInventory) string {
+	servers := make([]string, 0, len(perServer))
+	for s := range perServer {
+		servers = append(servers, s)
+	}
+	sort.Strings(servers)
+
+	var b []byte
+	for i, s := range servers {
+		if i > 0 {
+			b = append(b, ' ')
+		}
+		inv := perServer[s]
+		b = append(b, s...)
+		b = append(b, '=')
+		b = append(b, '[')
+		ids := make([]int, 0)
+		for id := 0; id < MaxShardCount; id++ {
+			if inv.Bits.Has(ShardId(id)) {
+				ids = append(ids, id)
+			}
+		}
+		for j, id := range ids {
+			if j > 0 {
+				b = append(b, ' ')
+			}
+			b = append(b, []byte(fmt.Sprintf("%d", id))...)
+		}
+		if inv.QueryError != nil {
+			if len(ids) > 0 {
+				b = append(b, ' ')
+			}
+			b = append(b, []byte("ERR:"+inv.QueryError.Error())...)
+		}
+		b = append(b, ']')
+	}
+	return string(b)
+}
+
+// VerifyShardsOnServer confirms one server really holds the named shards of a
+// specific (collection, volume), for callers about to delete another copy.
+//
+// Collection is checked, unlike in VerifyShardsAcrossServers: the inventory RPC
+// is keyed by volume id alone, so a server answering for volume N says nothing
+// about which collection's volume N it means. Approving a delete on the
+// strength of a different collection's shard would remove the last real copy —
+// the exact outcome the caller is trying to prevent.
+//
+// A server that cannot be queried is unknown, not confirmed, and returns an
+// error: treating an unreachable peer as proof of a surviving copy is how a
+// network blip becomes data loss.
+// CollectShardsOnServer asks one volume server which shards of the volume it
+// actually serves right now — its live inventory, as opposed to what the
+// master's possibly stale topology claims for it.
+func CollectShardsOnServer(ctx context.Context, collection string, volumeID uint32,
+	server string, dialOption grpc.DialOption) (present ShardBits, err error) {
+	err = operation.WithVolumeServerClient(false, pb.ServerAddress(server), dialOption,
+		func(client volume_server_pb.VolumeServerClient) error {
+			resp, e := client.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+				VolumeId: volumeID,
+			})
+			if e != nil {
+				return e
+			}
+			for _, s := range resp.EcShardInfos {
+				if s.VolumeId != volumeID || s.Collection != collection || s.ShardId >= MaxShardCount {
+					continue
+				}
+				present = present.Set(ShardId(s.ShardId))
+			}
+			return nil
+		})
+	return present, err
+}
+
+func VerifyShardsOnServer(ctx context.Context, collection string, volumeID uint32,
+	server string, shardIDs []uint32, dialOption grpc.DialOption) error {
+
+	if server == "" {
+		return fmt.Errorf("no server given to verify volume %d shard(s) %v", volumeID, shardIDs)
+	}
+
+	present, callErr := CollectShardsOnServer(ctx, collection, volumeID, server, dialOption)
+	if callErr != nil {
+		return fmt.Errorf("verify volume %d shard(s) %v on %s: %w", volumeID, shardIDs, server, callErr)
+	}
+
+	for _, sid := range shardIDs {
+		if !present.Has(ShardId(sid)) {
+			return fmt.Errorf("%s does not hold ec shard %d.%d of collection %q", server, volumeID, sid, collection)
+		}
+	}
+	return nil
+}

weed/storage/erasure_coding/verification_test.go

@@ -0,0 +1,142 @@
+package erasure_coding
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestRequireRecoverableShardSet_AllPresent(t *testing.T) {
+	var bits ShardBits
+	for id := 0; id < TotalShardsCount; id++ {
+		bits = bits.Set(ShardId(id))
+	}
+	degraded, err := RequireRecoverableShardSet(42, bits, DataShardsCount, TotalShardsCount)
+	if err != nil {
+		t.Fatalf("unexpected error for full set: %v", err)
+	}
+	if degraded {
+		t.Error("full set must not be reported as degraded")
+	}
+}
+
+func TestRequireRecoverableShardSet_DegradedButRecoverable(t *testing.T) {
+	// 12 of 14 shards: enough to reconstruct (>= 10), so the source may be
+	// deleted, but the caller is told to warn and schedule a rebuild.
+	var bits ShardBits
+	for id := 0; id < TotalShardsCount; id++ {
+		if id == 3 || id == 7 {
+			continue
+		}
+		bits = bits.Set(ShardId(id))
+	}
+	degraded, err := RequireRecoverableShardSet(42, bits, DataShardsCount, TotalShardsCount)
+	if err != nil {
+		t.Fatalf("recoverable set must not error: %v", err)
+	}
+	if !degraded {
+		t.Error("missing shards must be reported as degraded")
+	}
+}
+
+func TestRequireRecoverableShardSet_BelowDataShards(t *testing.T) {
+	// 9 of 14 shards: one short of reconstructable; the source must be kept.
+	var bits ShardBits
+	for id := 0; id < DataShardsCount-1; id++ {
+		bits = bits.Set(ShardId(id))
+	}
+	_, err := RequireRecoverableShardSet(42, bits, DataShardsCount, TotalShardsCount)
+	if err == nil {
+		t.Fatal("expected error for unrecoverable set, got nil")
+	}
+	msg := err.Error()
+	if !strings.Contains(msg, "volume 42") {
+		t.Errorf("error should name the volume id: %s", msg)
+	}
+	if !strings.Contains(msg, "9/14") {
+		t.Errorf("error should report 9/14 shards present: %s", msg)
+	}
+	if !strings.Contains(msg, "[9 10 11 12 13]") {
+		t.Errorf("error should list the missing ids: %s", msg)
+	}
+}
+
+func TestRequireRecoverableShardSet_EmptyBitmap(t *testing.T) {
+	_, err := RequireRecoverableShardSet(1, 0, DataShardsCount, TotalShardsCount)
+	if err == nil {
+		t.Fatal("expected error for empty bitmap")
+	}
+	if !strings.Contains(err.Error(), "0/14") {
+		t.Errorf("error should report 0/14 shards: %s", err.Error())
+	}
+}
+
+func TestRequireRecoverableShardSet_CustomRatio(t *testing.T) {
+	// 6+3 ratio: total=9, all present
+	var bits ShardBits
+	for id := 0; id < 9; id++ {
+		bits = bits.Set(ShardId(id))
+	}
+	if degraded, err := RequireRecoverableShardSet(7, bits, 6, 9); err != nil || degraded {
+		t.Fatalf("full 6+3 set: degraded=%v err=%v", degraded, err)
+	}
+
+	// 6+3, missing shard 5: 8 >= 6 remain, recoverable but degraded
+	bits = bits.Clear(5)
+	if degraded, err := RequireRecoverableShardSet(7, bits, 6, 9); err != nil || !degraded {
+		t.Fatalf("8/9 shards of 6+3: degraded=%v err=%v", degraded, err)
+	}
+
+	// 6+3, only 5 shards left: below dataShards, must error
+	bits = ShardBits(0)
+	for id := 0; id < 5; id++ {
+		bits = bits.Set(ShardId(id))
+	}
+	_, err := RequireRecoverableShardSet(7, bits, 6, 9)
+	if err == nil {
+		t.Fatal("expected error with 5/9 shards in 6+3 ratio")
+	}
+	if !strings.Contains(err.Error(), "5/9") {
+		t.Errorf("error should report 5/9: %s", err.Error())
+	}
+}
+
+func TestRequireRecoverableShardSet_RejectsInvalidParams(t *testing.T) {
+	if _, err := RequireRecoverableShardSet(1, 0, DataShardsCount, 0); err == nil {
+		t.Error("expected error for totalShards=0")
+	}
+	if _, err := RequireRecoverableShardSet(1, 0, DataShardsCount, MaxShardCount+1); err == nil {
+		t.Errorf("expected error for totalShards > MaxShardCount")
+	}
+	if _, err := RequireRecoverableShardSet(1, 0, 0, TotalShardsCount); err == nil {
+		t.Error("expected error for dataShards=0")
+	}
+	if _, err := RequireRecoverableShardSet(1, 0, TotalShardsCount+1, TotalShardsCount); err == nil {
+		t.Error("expected error for dataShards > totalShards")
+	}
+}
+
+func TestSummarizeShardInventory_Deterministic(t *testing.T) {
+	perServer := map[string]ServerShardInventory{
+		"10.0.0.2:8080": {Bits: ShardBits(0).Set(4).Set(5).Set(6)},
+		"10.0.0.1:8080": {Bits: ShardBits(0).Set(0).Set(1).Set(2).Set(3)},
+	}
+	got := SummarizeShardInventory(perServer)
+	want := "10.0.0.1:8080=[0 1 2 3] 10.0.0.2:8080=[4 5 6]"
+	if got != want {
+		t.Errorf("summary mismatch\n  got:  %q\n  want: %q", got, want)
+	}
+}
+
+func TestSummarizeShardInventory_IncludesError(t *testing.T) {
+	perServer := map[string]ServerShardInventory{
+		"10.0.0.1:8080": {Bits: ShardBits(0).Set(0).Set(1), QueryError: errStr("dial timeout")},
+	}
+	got := SummarizeShardInventory(perServer)
+	if !strings.Contains(got, "ERR:dial timeout") {
+		t.Errorf("expected error tag in summary, got %q", got)
+	}
+}
+
+type errStr string
+
+func (e errStr) Error() string { return string(e) }