snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/topology/allocate_volume.go

@@ -0,0 +1,56 @@
+package topology
+
+import (
+	"context"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"google.golang.org/grpc"
+)
+
+// Cap the RPC so a hung volume server can't strand the layout's
+// growRequest flag and block all future automatic growth.
+const allocateVolumeTimeout = 1 * time.Minute
+
+type AllocateVolumeResult struct {
+	Error string
+}
+
+func AllocateVolume(dn *DataNode, grpcDialOption grpc.DialOption, vid needle.VolumeId, option *VolumeGrowOption) error {
+
+	return operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+
+		ctx, cancel := context.WithTimeout(context.Background(), allocateVolumeTimeout)
+		defer cancel()
+
+		_, allocateErr := client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{
+			VolumeId:           uint32(vid),
+			Collection:         option.Collection,
+			Replication:        option.ReplicaPlacement.String(),
+			Ttl:                option.Ttl.String(),
+			Preallocate:        option.Preallocate,
+			MemoryMapMaxSizeMb: option.MemoryMapMaxSizeMb,
+			DiskType:           string(option.DiskType),
+			Version:            option.Version,
+		})
+		return allocateErr
+	})
+
+}
+
+func DeleteVolume(dn *DataNode, grpcDialOption grpc.DialOption, vid needle.VolumeId) error {
+
+	return operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+
+		ctx, cancel := context.WithTimeout(context.Background(), allocateVolumeTimeout)
+		defer cancel()
+
+		_, allocateErr := client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
+			VolumeId: uint32(vid),
+		})
+		return allocateErr
+	})
+
+}

weed/topology/balancer/density.go

@@ -0,0 +1,52 @@
+package balancer
+
+import "math"
+
+// zeroVolumeDensityWeight is the ratio given to a server holding no data, so it
+// still ranks below any loaded server (which has ratio >= 1) but is distinguished
+// from a server with unknown capacity.
+const zeroVolumeDensityWeight = 0.5
+
+// UsedVolumeEquivalents converts the bytes a server holds into whole-volume
+// equivalents (volumeBytes / volumeSizeLimit). Returns 0 when the limit is unset.
+func UsedVolumeEquivalents(volumeBytes, volumeSizeLimitBytes uint64) uint64 {
+	if volumeSizeLimitBytes == 0 {
+		return 0
+	}
+	return volumeBytes / volumeSizeLimitBytes
+}
+
+// VolumeDensity is the bytes-aware capacity view balancing ranks servers by:
+// used = volumeBytes / volumeSizeLimit, capacity = maxVolumeCount - used. Using
+// real data (not just volume count) means an over-configured maxVolumeCount can't
+// make a byte-full server look empty. capacity may be <= 0 for a server already
+// past its configured slots.
+func VolumeDensity(maxVolumeCount int64, volumeBytes, volumeSizeLimitBytes uint64) (capacity float64, usedVolumes uint64) {
+	usedVolumes = UsedVolumeEquivalents(volumeBytes, volumeSizeLimitBytes)
+	return float64(maxVolumeCount - int64(usedVolumes)), usedVolumes
+}
+
+// DensityRatio is a server's load: usedVolumes / capacity, higher = fuller. An
+// empty server gets a small positive ratio so it sorts below loaded servers.
+// A server with no free capacity (capacity <= 0: full, or over its configured
+// slots so capacity is negative) ranks as the fullest (+Inf), so ascending
+// consumers treat it as a move source and never as the emptiest target.
+func DensityRatio(capacity float64, usedVolumes uint64) float64 {
+	if capacity <= 0 {
+		return math.Inf(1)
+	}
+	if usedVolumes == 0 {
+		return zeroVolumeDensityWeight / capacity
+	}
+	return float64(usedVolumes) / capacity
+}
+
+// DensityNextRatio is a server's load after one more volume lands on it:
+// (usedVolumes+1) / capacity. Used to test whether a move would overshoot. A
+// server with no free capacity (capacity <= 0) ranks as the fullest (+Inf).
+func DensityNextRatio(capacity float64, usedVolumes uint64) float64 {
+	if capacity <= 0 {
+		return math.Inf(1)
+	}
+	return float64(usedVolumes+1) / capacity
+}

weed/topology/balancer/density_test.go

@@ -0,0 +1,81 @@
+package balancer
+
+import (
+	"math"
+	"sort"
+	"testing"
+)
+
+func TestVolumeDensity(t *testing.T) {
+	const gb = uint64(1) << 30
+	// 20 full-ish volumes (~1900 GB) on a disk with a 30 GB limit and Max 1000.
+	cap, used := VolumeDensity(1000, 1900*gb, 30*gb)
+	if used != 63 { // 1900/30 = 63.33 -> 63
+		t.Errorf("usedVolumes = %d, want 63", used)
+	}
+	if cap != float64(1000-63) {
+		t.Errorf("capacity = %v, want %v", cap, float64(1000-63))
+	}
+	// Unset limit -> 0 used, capacity = maxVolumeCount.
+	if c, u := VolumeDensity(10, 1000*gb, 0); u != 0 || c != 10 {
+		t.Errorf("unset limit: got cap=%v used=%d, want 10/0", c, u)
+	}
+}
+
+func TestDensityRatio(t *testing.T) {
+	// Empty server: small positive ratio (below any loaded server).
+	empty := DensityRatio(1000, 0)
+	loaded := DensityRatio(1000, 1)
+	if !(empty > 0 && empty < loaded) {
+		t.Errorf("empty ratio %v should be >0 and < loaded %v", empty, loaded)
+	}
+	// Loaded: used/capacity.
+	if got := DensityRatio(3, 7); got != 7.0/3.0 {
+		t.Errorf("DensityRatio(3,7) = %v, want %v", got, 7.0/3.0)
+	}
+	// No free capacity (full) or over the slot limit (negative capacity) -> fullest.
+	if got := DensityRatio(0, 5); !math.IsInf(got, 1) {
+		t.Errorf("DensityRatio(0,5) = %v, want +Inf", got)
+	}
+	if got := DensityRatio(-4, 20); !math.IsInf(got, 1) {
+		t.Errorf("DensityRatio(-4,20) = %v, want +Inf (overfull ranks fullest, not negative)", got)
+	}
+}
+
+func TestDensityNextRatio(t *testing.T) {
+	if got := DensityNextRatio(3, 7); got != 8.0/3.0 {
+		t.Errorf("DensityNextRatio(3,7) = %v, want %v", got, 8.0/3.0)
+	}
+	if got := DensityNextRatio(0, 7); !math.IsInf(got, 1) {
+		t.Errorf("DensityNextRatio(0,7) = %v, want +Inf", got)
+	}
+	if got := DensityNextRatio(-1, 7); !math.IsInf(got, 1) {
+		t.Errorf("DensityNextRatio(-1,7) = %v, want +Inf", got)
+	}
+}
+
+// An overfull server (negative capacity) must sort as the fullest, not the
+// emptiest, when consumers rank ascending by DensityRatio.
+func TestDensityRatio_OverfullSortsFullest(t *testing.T) {
+	// empty (cap 100, used 0), half (cap 5, used 5), overfull (cap -3, used 13).
+	type server struct {
+		name        string
+		capacity    float64
+		usedVolumes uint64
+	}
+	servers := []server{
+		{"overfull", -3, 13},
+		{"empty", 100, 0},
+		{"half", 5, 5},
+	}
+	sort.Slice(servers, func(i, j int) bool {
+		return DensityRatio(servers[i].capacity, servers[i].usedVolumes) <
+			DensityRatio(servers[j].capacity, servers[j].usedVolumes)
+	})
+	if servers[0].name != "empty" {
+		t.Errorf("emptiest (ascending first) = %q, want empty", servers[0].name)
+	}
+	if servers[len(servers)-1].name != "overfull" {
+		t.Errorf("fullest (ascending last) = %q, want overfull", servers[len(servers)-1].name)
+	}
+}

weed/topology/balancer/disk_fullness.go

@@ -0,0 +1,33 @@
+// Package balancer holds balancing policy shared by the shell volume.balance
+// command and the maintenance balance worker so the two implementations do not
+// drift. It is dependency-light and takes plain values, so both the raw
+// master_pb world (shell) and the ActiveTopology world (worker) can call it.
+package balancer
+
+// DefaultMaxDiskUsagePercent is the default physical-disk high-water mark for
+// balancing: a move target whose disk is at or above this used percentage is
+// skipped. It guards against an over-configured maxVolumeCount making a
+// physically full server look under-utilized by slot count.
+const DefaultMaxDiskUsagePercent = 90
+
+// DiskTooFullAfter reports whether landing incomingBytes on a disk would push its
+// physical used space to or above maxUsagePercent of totalBytes. It judges the
+// disk against its own total, so heterogeneous disk sizes are handled correctly.
+//
+// It returns false (no opinion) when the gate is disabled (maxUsagePercent <= 0
+// or >= 100) or the disk reports no capacity (totalBytes == 0), so callers fall
+// back to slot-only behavior for servers that do not report disk bytes.
+//
+// Pass incomingBytes = 0 to test current fullness, or a volume's worth of bytes
+// to test fullness after the move.
+func DiskTooFullAfter(totalBytes, freeBytes, incomingBytes uint64, maxUsagePercent int) bool {
+	if maxUsagePercent <= 0 || maxUsagePercent >= 100 || totalBytes == 0 {
+		return false
+	}
+	var used uint64
+	if totalBytes > freeBytes {
+		used = totalBytes - freeBytes
+	}
+	usedAfter := float64(used) + float64(incomingBytes)
+	return usedAfter*100 >= float64(totalBytes)*float64(maxUsagePercent)
+}

weed/topology/balancer/disk_fullness_test.go

@@ -0,0 +1,31 @@
+package balancer
+
+import "testing"
+
+func TestDiskTooFullAfter(t *testing.T) {
+	const gb = uint64(1) << 30
+	tests := []struct {
+		name                  string
+		total, free, incoming uint64
+		pct                   int
+		want                  bool
+	}{
+		{"disabled at 0", 1000 * gb, 40 * gb, 0, 0, false},
+		{"disabled at 100", 1000 * gb, 40 * gb, 0, 100, false},
+		{"not reported (total 0)", 0, 0, 0, 90, false},
+		{"below mark", 1000 * gb, 300 * gb, 0, 90, false},                           // 70% used
+		{"at mark", 1000 * gb, 100 * gb, 0, 90, true},                               // exactly 90% used
+		{"above mark", 1000 * gb, 40 * gb, 0, 90, true},                             // 96% used
+		{"below but crosses with incoming", 1000 * gb, 120 * gb, 30 * gb, 90, true}, // 88% -> 91%
+		{"below and stays below with incoming", 1000 * gb, 300 * gb, 30 * gb, 90, false},
+		{"free exceeds total (defensive)", 1000 * gb, 2000 * gb, 0, 90, false},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := DiskTooFullAfter(tc.total, tc.free, tc.incoming, tc.pct); got != tc.want {
+				t.Errorf("DiskTooFullAfter(%d,%d,%d,%d) = %v, want %v",
+					tc.total, tc.free, tc.incoming, tc.pct, got, tc.want)
+			}
+		})
+	}
+}

weed/topology/balancer/replica_placement.go

@@ -0,0 +1,176 @@
+package balancer
+
+import (
+	"slices"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+)
+
+// Location identifies where a volume replica lives, at the granularity replica
+// placement reasons about: data center, rack, data node, and physical host
+// (machine). Servers sharing a host are one fault domain. It is the shared shape
+// the shell (from master_pb) and the worker (from ActiveTopology) both adapt to.
+type Location struct {
+	DataCenter string
+	Rack       string
+	NodeID     string
+	Host       string
+}
+
+// HostFromAddress returns the physical machine (host/IP) of a server address,
+// falling back to the node id (== ip:port in the common case) when no address is
+// known.
+func HostFromAddress(address, nodeID string) string {
+	if address == "" {
+		address = nodeID
+	}
+	return pb.ServerAddress(address).ToHost()
+}
+
+type rackKey struct {
+	DataCenter string
+	Rack       string
+}
+
+type nodeKey struct {
+	DataCenter string
+	Rack       string
+	NodeID     string
+}
+
+// IsGoodMove reports whether moving a volume from sourceNodeID onto target keeps
+// the volume's replica placement policy satisfied, given the current replica
+// locations. It refuses moving onto the source itself, onto a node that already
+// holds a replica, and onto a host that already holds another replica (best-effort
+// machine anti-affinity), then defers to SatisfyReplicaPlacement.
+func IsGoodMove(rp *super_block.ReplicaPlacement, existingReplicas []Location, sourceNodeID string, target Location) bool {
+	if rp == nil || !rp.HasReplication() {
+		return true // no replication constraint
+	}
+
+	// Never move onto the source node itself.
+	if target.NodeID == sourceNodeID {
+		return false
+	}
+
+	// Build the replica set after the move: remove the one replica being moved off
+	// the source, keeping any others (an over-replicated node may hold more than
+	// one replica of this volume, and only one is moving).
+	afterMove := make([]Location, 0, len(existingReplicas))
+	sourceFound := false
+	for _, r := range existingReplicas {
+		if r.NodeID == sourceNodeID && !sourceFound {
+			sourceFound = true
+		} else {
+			afterMove = append(afterMove, r)
+		}
+	}
+	if !sourceFound {
+		// Source not in the replica list — cluster state may be inconsistent.
+		// Treat as unsafe to avoid an incorrect placement decision.
+		return false
+	}
+
+	// Best-effort machine anti-affinity: don't move a replica onto a host that
+	// already holds another replica of this volume, so a single machine failure
+	// can't take out two replicas.
+	if target.Host != "" {
+		for _, r := range afterMove {
+			if r.Host == target.Host {
+				return false
+			}
+		}
+	}
+
+	return SatisfyReplicaPlacement(rp, afterMove, target)
+}
+
+// SatisfyReplicaPlacement reports whether placing a replica at target is
+// consistent with the replication policy given the existing replicas. It is the
+// shared implementation for the shell (volume.balance / fix.replication /
+// tier.move) and the maintenance balance worker.
+func SatisfyReplicaPlacement(rp *super_block.ReplicaPlacement, replicas []Location, target Location) bool {
+	existingDCs, _, existingNodes := countReplicas(replicas)
+
+	targetNK := nodeKey{DataCenter: target.DataCenter, Rack: target.Rack, NodeID: target.NodeID}
+	if _, found := existingNodes[targetNK]; found {
+		// avoid a duplicated volume on the same data node
+		return false
+	}
+
+	primaryDCs, _ := findTopDCKeys(existingDCs)
+
+	// ensure the data center count is within limit
+	if _, found := existingDCs[target.DataCenter]; !found {
+		// different from existing dcs
+		return len(existingDCs) < rp.DiffDataCenterCount+1
+	}
+	// same as one of the existing data centers
+	if !slices.Contains(primaryDCs, target.DataCenter) {
+		return false
+	}
+
+	// on a primary dc — check racks within this DC
+	primaryDcRacks := make(map[rackKey]int)
+	for _, r := range replicas {
+		if r.DataCenter != target.DataCenter {
+			continue
+		}
+		primaryDcRacks[rackKey{DataCenter: r.DataCenter, Rack: r.Rack}]++
+	}
+
+	targetRK := rackKey{DataCenter: target.DataCenter, Rack: target.Rack}
+	primaryRacks, _ := findTopRackKeys(primaryDcRacks)
+	sameRackCount := primaryDcRacks[targetRK]
+
+	if _, found := primaryDcRacks[targetRK]; !found {
+		// different from existing racks
+		return len(primaryDcRacks) < rp.DiffRackCount+1
+	}
+	// same as one of the existing racks
+	if !slices.Contains(primaryRacks, targetRK) {
+		return false
+	}
+
+	// on the primary rack — check the same-rack count
+	return sameRackCount < rp.SameRackCount+1
+}
+
+func countReplicas(replicas []Location) (dcCounts map[string]int, rackCounts map[rackKey]int, nodeCounts map[nodeKey]int) {
+	dcCounts = make(map[string]int)
+	rackCounts = make(map[rackKey]int)
+	nodeCounts = make(map[nodeKey]int)
+	for _, r := range replicas {
+		dcCounts[r.DataCenter]++
+		rackCounts[rackKey{DataCenter: r.DataCenter, Rack: r.Rack}]++
+		nodeCounts[nodeKey{DataCenter: r.DataCenter, Rack: r.Rack, NodeID: r.NodeID}]++
+	}
+	return
+}
+
+func findTopDCKeys(m map[string]int) (topKeys []string, max int) {
+	for k, c := range m {
+		if max < c {
+			topKeys = topKeys[:0]
+			topKeys = append(topKeys, k)
+			max = c
+		} else if max == c {
+			topKeys = append(topKeys, k)
+		}
+	}
+	return
+}
+
+func findTopRackKeys(m map[rackKey]int) (topKeys []rackKey, max int) {
+	for k, c := range m {
+		if max < c {
+			topKeys = topKeys[:0]
+			topKeys = append(topKeys, k)
+			max = c
+		} else if max == c {
+			topKeys = append(topKeys, k)
+		}
+	}
+	return
+}

weed/topology/balancer/replica_placement_test.go

@@ -0,0 +1,36 @@
+package balancer
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+)
+
+func loc(dc, rack, node string) Location {
+	return Location{DataCenter: dc, Rack: rack, NodeID: node}
+}
+
+// When a node abnormally holds more than one replica of a volume, moving one of
+// them off must keep the other in the after-move set, so the placement check
+// still sees the rack's true occupancy.
+func TestIsGoodMove_OverReplicatedSourceKeepsOtherReplica(t *testing.T) {
+	rp, err := super_block.NewReplicaPlacementFromString("001") // up to 2 copies in a rack
+	if err != nil {
+		t.Fatal(err)
+	}
+	// n1 abnormally holds two replicas; n2 holds a third — all in rack r1.
+	existing := []Location{loc("dc1", "r1", "n1"), loc("dc1", "r1", "n1"), loc("dc1", "r1", "n2")}
+
+	// Moving ONE replica off n1 onto a new node n3 in r1 must be rejected: r1 would
+	// still hold two replicas (the retained n1 and n2), at the SameRackCount limit.
+	if IsGoodMove(rp, existing, "n1", loc("dc1", "r1", "n3")) {
+		t.Error("expected reject: moving one replica off an over-replicated node must keep its other replica in the rack count")
+	}
+
+	// The normal case is unaffected: two replicas correctly in r1, move one to a
+	// third node in r1 — still within the SameRackCount=1 (two-per-rack) limit.
+	normal := []Location{loc("dc1", "r1", "n1"), loc("dc1", "r1", "n2")}
+	if !IsGoodMove(rp, normal, "n1", loc("dc1", "r1", "n3")) {
+		t.Error("expected allow: a normal move within the rack limit")
+	}
+}

weed/topology/balancer/replica_selection.go

@@ -0,0 +1,161 @@
+package balancer
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+)
+
+// Replica pairs a replica's placement location with the volume attributes
+// replica-selection decisions need. Like Location for placement checks, it is
+// the shared shape the shell (from master_pb) and workers adapt to. Selection
+// functions return an index into the given slice (-1 when nothing qualifies),
+// so callers can map the choice back to their own richer replica type.
+type Replica struct {
+	Location         Location
+	Size             uint64
+	ModifiedAtSecond int64
+	CompactRevision  uint32
+	ReadOnly         bool
+}
+
+// SatisfyReplicaCurrentLocation reports whether the existing replicas already
+// span the failure domains the policy requires (enough data centers, enough
+// racks, and the same-rack count somewhere). A volume whose replica count is
+// right but whose spread falls short still needs another replica.
+func SatisfyReplicaCurrentLocation(rp *super_block.ReplicaPlacement, replicas []Location) bool {
+	dcCounts, rackCounts, _ := countReplicas(replicas)
+
+	if rp.DiffDataCenterCount+1 > len(dcCounts) {
+		return false
+	}
+	if rp.DiffRackCount+1 > len(rackCounts) {
+		return false
+	}
+	if rp.SameRackCount > 0 {
+		for _, rackCount := range rackCounts {
+			if rackCount >= rp.SameRackCount+1 {
+				return true
+			}
+		}
+		return false
+	}
+	return true
+}
+
+// PickOneReplicaToCopyFrom returns the most recently modified replica, the
+// best source for adding a new replica.
+func PickOneReplicaToCopyFrom(replicas []Replica) int {
+	mostRecent := -1
+	for i, r := range replicas {
+		if mostRecent < 0 || r.ModifiedAtSecond > replicas[mostRecent].ModifiedAtSecond {
+			mostRecent = i
+		}
+	}
+	return mostRecent
+}
+
+// IsMisplaced reports whether any replica sits at a location inconsistent
+// with the replication policy given the other replicas' locations.
+func IsMisplaced(replicas []Replica, rp *super_block.ReplicaPlacement) bool {
+	for i := range replicas {
+		others := otherThan(replicas, i)
+		locs := make([]Location, len(others))
+		for j, r := range others {
+			locs[j] = r.Location
+		}
+		if !SatisfyReplicaPlacement(rp, locs, replicas[i].Location) {
+			return true
+		}
+	}
+	return false
+}
+
+// PickSmallestReplica returns the smallest replica (ties broken by oldest
+// then lowest compact revision).
+func PickSmallestReplica(replicas []Replica) int {
+	smallest := -1
+	for i := range replicas {
+		if smallest < 0 || lessReplica(replicas[i], replicas[smallest]) {
+			smallest = i
+		}
+	}
+	return smallest
+}
+
+// PickOneReplicaToDelete selects the replica to trim when over-replicated.
+// It only ever removes the smallest of multiple healthy writable replicas: a
+// ReadOnly/integrity-flagged replica is never chosen for deletion, and the
+// trim is refused (returns -1) when removing a writable replica would leave
+// only ReadOnly survivors. Among the writable replicas it prefers one whose
+// removal still satisfies placement, so the trim does not strip the only
+// replica in a required failure domain; it falls back to the smallest
+// writable if none keeps placement (a later misplaced cycle then
+// re-balances).
+func PickOneReplicaToDelete(replicas []Replica, rp *super_block.ReplicaPlacement) int {
+	var writable []int
+	for i, r := range replicas {
+		if !r.ReadOnly {
+			writable = append(writable, i)
+		}
+	}
+	if len(writable) < 2 {
+		return -1
+	}
+	var placementSafe []int
+	for _, i := range writable {
+		if !IsMisplaced(otherThan(replicas, i), rp) {
+			placementSafe = append(placementSafe, i)
+		}
+	}
+	if len(placementSafe) > 0 {
+		return pickSmallestAmong(replicas, placementSafe)
+	}
+	return pickSmallestAmong(replicas, writable)
+}
+
+// PickOneMisplacedVolume selects the replica to delete and recreate at a
+// correct placement. This is relocation, not over-replication: unlike
+// PickOneReplicaToDelete it must still act on read-only replicas (e.g. a full
+// but misplaced volume), so it does not use the writable-survivor guard. It
+// prefers a replica whose removal leaves the others well placed, and falls
+// back to the smallest replica.
+func PickOneMisplacedVolume(replicas []Replica, rp *super_block.ReplicaPlacement) int {
+	var deletionCandidates []int
+	for i := range replicas {
+		if !IsMisplaced(otherThan(replicas, i), rp) {
+			deletionCandidates = append(deletionCandidates, i)
+		}
+	}
+	if smallest := pickSmallestAmong(replicas, deletionCandidates); smallest >= 0 {
+		return smallest
+	}
+	return PickSmallestReplica(replicas)
+}
+
+func pickSmallestAmong(replicas []Replica, candidates []int) int {
+	smallest := -1
+	for _, i := range candidates {
+		if smallest < 0 || lessReplica(replicas[i], replicas[smallest]) {
+			smallest = i
+		}
+	}
+	return smallest
+}
+
+func lessReplica(a, b Replica) bool {
+	if a.Size != b.Size {
+		return a.Size < b.Size
+	}
+	if a.ModifiedAtSecond != b.ModifiedAtSecond {
+		return a.ModifiedAtSecond < b.ModifiedAtSecond
+	}
+	return a.CompactRevision < b.CompactRevision
+}
+
+func otherThan(replicas []Replica, index int) (others []Replica) {
+	for i := range replicas {
+		if index != i {
+			others = append(others, replicas[i])
+		}
+	}
+	return
+}

weed/topology/cluster_commands.go

@@ -0,0 +1,65 @@
+package topology
+
+import (
+	"encoding/json"
+	"fmt"
+
+	hashicorpRaft "github.com/hashicorp/raft"
+	"github.com/seaweedfs/raft"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+type MaxVolumeIdCommand struct {
+	MaxVolumeId needle.VolumeId `json:"maxVolumeId"`
+	TopologyId  string          `json:"topologyId"`
+}
+
+func NewMaxVolumeIdCommand(value needle.VolumeId, topologyId string) *MaxVolumeIdCommand {
+	return &MaxVolumeIdCommand{
+		MaxVolumeId: value,
+		TopologyId:  topologyId,
+	}
+}
+
+func (c *MaxVolumeIdCommand) CommandName() string {
+	return "MaxVolumeId"
+}
+
+// deprecatedCommandApply represents the old interface to apply a command to the server.
+func (c *MaxVolumeIdCommand) Apply(server raft.Server) (interface{}, error) {
+	topo := server.Context().(*Topology)
+	before := topo.GetMaxVolumeId()
+	topo.UpAdjustMaxVolumeId(c.MaxVolumeId)
+	if c.TopologyId != "" {
+		prevTopologyId := topo.GetTopologyId()
+		topo.SetTopologyId(c.TopologyId)
+		// Log when TopologyId is set for the first time, with different messages for leader and follower.
+		if prevTopologyId == "" {
+			if server.State() == raft.Leader {
+				glog.V(0).Infof("TopologyId generated and applied on leader: %s", c.TopologyId)
+			} else {
+				glog.V(0).Infof("TopologyId applied on follower: %s", c.TopologyId)
+			}
+		}
+	}
+	glog.V(1).Infoln("max volume id", before, "==>", topo.GetMaxVolumeId())
+
+	return nil, nil
+}
+
+func (s *MaxVolumeIdCommand) Persist(sink hashicorpRaft.SnapshotSink) error {
+	b, err := json.Marshal(s)
+	if err != nil {
+		return fmt.Errorf("marshal: %w", err)
+	}
+	_, err = sink.Write(b)
+	if err != nil {
+		sink.Cancel()
+		return fmt.Errorf("sink.Write(): %w", err)
+	}
+	return sink.Close()
+}
+
+func (s *MaxVolumeIdCommand) Release() {
+}

weed/topology/collection.go

@@ -0,0 +1,107 @@
+package topology
+
+import (
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type Collection struct {
+	Name                     string
+	volumeSizeLimit          uint64
+	replicationAsMin         bool
+	storageType2VolumeLayout *util.ConcurrentReadMap
+}
+
+func NewCollection(name string, volumeSizeLimit uint64, replicationAsMin bool) *Collection {
+	c := &Collection{
+		Name:             name,
+		volumeSizeLimit:  volumeSizeLimit,
+		replicationAsMin: replicationAsMin,
+	}
+	c.storageType2VolumeLayout = util.NewConcurrentReadMap()
+	return c
+}
+
+func (c *Collection) String() string {
+	return fmt.Sprintf("Name:%s, volumeSizeLimit:%d, storageType2VolumeLayout:%v", c.Name, c.volumeSizeLimit, c.storageType2VolumeLayout)
+}
+
+func (c *Collection) GetOrCreateVolumeLayout(rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType types.DiskType) *VolumeLayout {
+	keyString := rp.String()
+	if ttl != nil {
+		keyString += ttl.String()
+	}
+	if diskType != types.HardDriveType {
+		keyString += string(diskType)
+	}
+	vl := c.storageType2VolumeLayout.Get(keyString, func() interface{} {
+		return NewVolumeLayout(rp, ttl, diskType, c.volumeSizeLimit, c.replicationAsMin)
+	})
+	return vl.(*VolumeLayout)
+}
+
+func (c *Collection) GetVolumeLayout(rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType types.DiskType) (*VolumeLayout, bool) {
+	keyString := rp.String()
+	if ttl != nil {
+		keyString += ttl.String()
+	}
+	if diskType != types.HardDriveType {
+		keyString += string(diskType)
+	}
+	vl, ok := c.storageType2VolumeLayout.Find(keyString)
+	if !ok {
+		return nil, false
+	}
+	return vl.(*VolumeLayout), true
+}
+
+func (c *Collection) GetAllVolumeLayouts() []*VolumeLayout {
+	var vls []*VolumeLayout
+	for _, vl := range c.storageType2VolumeLayout.Items() {
+		if vl != nil {
+			vls = append(vls, vl.(*VolumeLayout))
+		}
+	}
+	return vls
+}
+
+func (c *Collection) DeleteVolumeLayout(rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType types.DiskType) {
+	keyString := rp.String()
+	if ttl != nil {
+		keyString += ttl.String()
+	}
+	if diskType != types.HardDriveType {
+		keyString += string(diskType)
+	}
+	// Unpublish first so a racing registration re-resolves into a fresh layout.
+	if vl, found := c.storageType2VolumeLayout.Delete(keyString); found {
+		vl.(*VolumeLayout).releaseLookupOwnership()
+	}
+}
+
+func (c *Collection) Lookup(vid needle.VolumeId) []*DataNode {
+	for _, vl := range c.storageType2VolumeLayout.Items() {
+		if vl != nil {
+			if list := vl.(*VolumeLayout).Lookup(vid); list != nil {
+				return list
+			}
+		}
+	}
+	return nil
+}
+
+func (c *Collection) ListVolumeServers() (nodes []*DataNode) {
+	for _, vl := range c.storageType2VolumeLayout.Items() {
+		if vl != nil {
+			if list := vl.(*VolumeLayout).ListVolumeServers(); list != nil {
+				nodes = append(nodes, list...)
+			}
+		}
+	}
+	return
+}

weed/topology/collection_layout_delete_test.go

@@ -0,0 +1,72 @@
+package topology
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func TestGetVolumeLayoutOfAbsentKey(t *testing.T) {
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	c := NewCollection("c", 32*1024, false)
+
+	vl, found := c.GetVolumeLayout(rp, needle.EMPTY_TTL, types.HardDriveType)
+	if found || vl != nil {
+		t.Fatalf("absent layout: got (%v, %v), want (nil, false)", vl, found)
+	}
+}
+
+// Volume servers dropping the last replica of several volumes that share one
+// layout all see the layout go empty and all delete it. The losers used to
+// crash the master on a nil type assertion.
+func TestConcurrentLastReplicaRemoval(t *testing.T) {
+	const nodeCount = 8
+	ttl, _ := needle.ReadTTL("5m")
+	for round := 0; round < 500; round++ {
+		topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+		rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
+
+		// A second layout keeps the collection alive, so every remover reaches
+		// the layout deletion instead of stopping at a vanished collection.
+		keeper := rack.GetOrCreateDataNode("127.0.0.1", 9000, 0, "", "", map[string]uint32{"": 100})
+		topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{{
+			Id: 999, Collection: "c", Version: uint32(needle.GetCurrentVersion()), Ttl: ttl.ToUint32(),
+		}}, keeper)
+
+		var wg sync.WaitGroup
+		start := make(chan struct{})
+		for i := 0; i < nodeCount; i++ {
+			dn := rack.GetOrCreateDataNode("127.0.0.1", 8080+i, 0, "", "", map[string]uint32{"": 100})
+			m := &master_pb.VolumeInformationMessage{
+				Id: uint32(i + 1), Collection: "c", Version: uint32(needle.GetCurrentVersion()),
+			}
+			topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{m}, dn)
+			vi, err := storage.NewVolumeInfo(m)
+			if err != nil {
+				t.Fatalf("NewVolumeInfo: %v", err)
+			}
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				<-start
+				topo.UnRegisterVolumeLayout(vi, dn)
+			}()
+		}
+		close(start)
+		wg.Wait()
+
+		c, found := topo.FindCollection("c")
+		if !found {
+			t.Fatalf("round %d: collection dropped while a layout still had volumes", round)
+		}
+		if layouts := c.GetAllVolumeLayouts(); len(layouts) != 1 {
+			t.Fatalf("round %d: got %d layouts, want only the one still holding a volume", round, len(layouts))
+		}
+	}
+}

weed/topology/collection_statistics.go

@@ -0,0 +1,148 @@
+package topology
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// CollectionStatistics is what a collection holds, summarised so that callers
+// tracking usage do not have to be sent every volume in the cluster to add it
+// up themselves.
+type CollectionStatistics struct {
+	Collection       string
+	FileCount        uint64
+	DeleteCount      uint64
+	DeletedByteCount uint64
+	// Size counts one copy of the data: a single replica of a regular volume,
+	// the data shards of an ec volume.
+	Size uint64
+	// PhysicalSize counts what is on disk: every replica, and parity shards.
+	PhysicalSize uint64
+	VolumeCount  uint64
+}
+
+type ecStatsKey struct {
+	collection string
+	volumeId   needle.VolumeId
+}
+
+// ecFileCounts holds the per-volume counts that can only be resolved once every
+// shard holder has been seen.
+type ecFileCounts struct {
+	collection  string
+	fileCount   uint64
+	deleteCount uint64
+}
+
+// CollectionStatistics summarises every collection in one pass over the
+// topology, allocating per collection rather than per volume.
+func (t *Topology) CollectionStatistics() []*CollectionStatistics {
+	byCollection := make(map[string]*CollectionStatistics)
+	statsFor := func(collection string) *CollectionStatistics {
+		stats, found := byCollection[collection]
+		if !found {
+			stats = &CollectionStatistics{Collection: collection}
+			byCollection[collection] = stats
+		}
+		return stats
+	}
+
+	// Regular volumes are counted once each for logical totals and once per
+	// replica for physical, which the lookup index gives without a set of seen
+	// ids: it is already keyed by volume.
+	for _, c := range t.collectionMap.Items() {
+		collection := c.(*Collection)
+		for _, vl := range collection.GetAllVolumeLayouts() {
+			vl.accessLock.RLock()
+			for vid, locations := range vl.vid2location {
+				stats := statsFor(collection.Name)
+				// Replicas of one volume can disagree while a write is landing
+				// or a heartbeat is late. Count the one holding the most live
+				// data, so the answer does not depend on which replica is
+				// looked at first and usage is never reported lower than some
+				// replica already holds. Quotas are enforced on size less
+				// deletions, so that is what has to be the largest -- a replica
+				// with the biggest raw size can be the one that has deleted the
+				// most, and picking it would leave an over-quota bucket
+				// writable.
+				var largest storage.VolumeInfo
+				var largestLive uint64
+				found := false
+				for _, dn := range locations.list {
+					v, err := dn.GetVolumesById(vid)
+					if err != nil {
+						continue
+					}
+					stats.PhysicalSize += v.Size
+					live := v.Size
+					if v.DeletedByteCount < live {
+						live -= v.DeletedByteCount
+					} else {
+						live = 0
+					}
+					if !found || live > largestLive {
+						largest, largestLive, found = v, live, true
+					}
+				}
+				if !found {
+					continue
+				}
+				stats.Size += largest.Size
+				stats.FileCount += uint64(largest.FileCount)
+				stats.DeleteCount += uint64(largest.DeleteCount)
+				// Never more deletions than the volume holds. Live usage is
+				// read as the collection's size less its deletions, so a volume
+				// reporting more deleted bytes than it has would cancel live
+				// bytes belonging to other volumes and report the bucket
+				// smaller than it is.
+				stats.DeletedByteCount += largest.Size - largestLive
+				stats.VolumeCount++
+			}
+			vl.accessLock.RUnlock()
+		}
+	}
+
+	// Ec shards are node-local rather than replicated, so their sizes sum
+	// across holders. The file and delete counts describe the volume rather
+	// than the shard, so they resolve once every holder has been seen.
+	perEcVolume := make(map[ecStatsKey]*ecFileCounts)
+	for _, dcNode := range t.Children() {
+		for _, rackNode := range dcNode.(*DataCenter).Children() {
+			for _, dnNode := range rackNode.(*Rack).Children() {
+				for _, ecInfo := range dnNode.(*DataNode).GetEcShards() {
+					message := ecInfo.ToVolumeEcShardInformationMessage()
+					stats := statsFor(ecInfo.Collection)
+					stats.PhysicalSize += uint64(erasure_coding.EcShardsTotalSize(message))
+					stats.Size += uint64(erasure_coding.EcShardsDataSize(message, 0))
+
+					key := ecStatsKey{collection: ecInfo.Collection, volumeId: ecInfo.VolumeId}
+					counts, found := perEcVolume[key]
+					if !found {
+						counts = &ecFileCounts{collection: ecInfo.Collection}
+						perEcVolume[key] = counts
+						stats.VolumeCount++
+					}
+					if message.FileCount > counts.fileCount {
+						counts.fileCount = message.FileCount
+					}
+					counts.deleteCount += message.DeleteCount
+				}
+			}
+		}
+	}
+	for _, counts := range perEcVolume {
+		stats := byCollection[counts.collection]
+		if stats == nil {
+			continue
+		}
+		stats.FileCount += counts.fileCount
+		stats.DeleteCount += counts.deleteCount
+	}
+
+	ret := make([]*CollectionStatistics, 0, len(byCollection))
+	for _, stats := range byCollection {
+		ret = append(ret, stats)
+	}
+	return ret
+}

weed/topology/collection_statistics_test.go

@@ -0,0 +1,309 @@
+package topology
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+// referenceCollectionStatistics is what callers computed for themselves from a
+// full topology listing, kept here so the summary can be held to producing the
+// same numbers. Quota enforcement reads these.
+func referenceCollectionStatistics(t *master_pb.TopologyInfo) map[string]*CollectionStatistics {
+	type volumeKey struct {
+		collection string
+		volumeId   uint32
+	}
+	out := map[string]*CollectionStatistics{}
+	seen := map[volumeKey]bool{}
+	ecCounts := map[volumeKey]*ecFileCounts{}
+	statsFor := func(c string) *CollectionStatistics {
+		s, ok := out[c]
+		if !ok {
+			s = &CollectionStatistics{Collection: c}
+			out[c] = s
+		}
+		return s
+	}
+
+	for _, dc := range t.DataCenterInfos {
+		for _, r := range dc.RackInfos {
+			for _, dn := range r.DataNodeInfos {
+				for _, diskInfo := range dn.DiskInfos {
+					for _, vi := range diskInfo.VolumeInfos {
+						s := statsFor(vi.Collection)
+						s.PhysicalSize += vi.Size
+						key := volumeKey{vi.Collection, vi.Id}
+						if seen[key] {
+							continue
+						}
+						seen[key] = true
+						s.Size += vi.Size
+						s.FileCount += vi.FileCount
+						s.DeleteCount += vi.DeleteCount
+						s.DeletedByteCount += vi.DeletedByteCount
+						s.VolumeCount++
+					}
+					for _, esi := range diskInfo.EcShardInfos {
+						s := statsFor(esi.Collection)
+						s.PhysicalSize += uint64(erasure_coding.EcShardsTotalSize(esi))
+						s.Size += uint64(erasure_coding.EcShardsDataSize(esi, 0))
+						key := volumeKey{esi.Collection, esi.Id}
+						agg, ok := ecCounts[key]
+						if !ok {
+							agg = &ecFileCounts{collection: esi.Collection}
+							ecCounts[key] = agg
+							s.VolumeCount++
+						}
+						if esi.FileCount > agg.fileCount {
+							agg.fileCount = esi.FileCount
+						}
+						agg.deleteCount += esi.DeleteCount
+					}
+				}
+			}
+		}
+	}
+	for _, agg := range ecCounts {
+		if s := out[agg.collection]; s != nil {
+			s.FileCount += agg.fileCount
+			s.DeleteCount += agg.deleteCount
+		}
+	}
+	return out
+}
+
+func statsTopology(t *testing.T) *Topology {
+	t.Helper()
+	topo := NewTopology("stats", nil, 32*1024*1024*1024, 5, false)
+	rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
+	nodes := make([]*DataNode, 3)
+	for i := range nodes {
+		nodes[i] = rack.GetOrCreateDataNode(
+			"10.0.0."+string(rune('1'+i)), 8080, 18080, "", "", map[string]uint32{"": 1000, "ssd": 1000})
+	}
+
+	// Two collections, replicated volumes, one volume on a second disk type.
+	// Replicas agree here; disagreement is covered on its own, where the
+	// listing this is compared against is itself order dependent.
+	report := func(dn *DataNode, msgs ...*master_pb.VolumeInformationMessage) {
+		topo.SyncDataNodeRegistration(msgs, dn)
+	}
+	vol := func(id uint32, collection string, size uint64, diskType string) *master_pb.VolumeInformationMessage {
+		return &master_pb.VolumeInformationMessage{
+			Id: id, Collection: collection, Size: size, FileCount: uint64(id) * 10,
+			DeleteCount: uint64(id), DeletedByteCount: uint64(id) * 100,
+			Version: 3, ReplicaPlacement: 1, DiskType: diskType,
+		}
+	}
+	report(nodes[0], vol(1, "bucket-a", 1000, ""), vol(2, "bucket-a", 2000, ""), vol(5, "bucket-b", 700, "ssd"))
+	report(nodes[1], vol(1, "bucket-a", 1000, ""), vol(3, "bucket-b", 3000, ""))
+	report(nodes[2], vol(2, "bucket-a", 2000, ""), vol(3, "bucket-b", 3000, ""))
+
+	// Ec shards for a third collection, spread over two nodes, with the file
+	// count reported differently by each holder.
+	topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{
+		{Id: 9, Collection: "bucket-c", EcIndexBits: 0x1f, ShardSizes: []int64{10, 20, 30, 40, 50}, FileCount: 40, DeleteCount: 2},
+	}, nodes[0])
+	topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{
+		{Id: 9, Collection: "bucket-c", EcIndexBits: 0x3e0, ShardSizes: []int64{60, 70, 80, 90, 100}, FileCount: 44, DeleteCount: 3},
+	}, nodes[1])
+	return topo
+}
+
+// The summary replaces callers adding up a full topology listing, so it has to
+// produce what that produced: these numbers enforce bucket quotas.
+func TestCollectionStatisticsMatchesAFullListing(t *testing.T) {
+	topo := statsTopology(t)
+
+	want := referenceCollectionStatistics(topo.ToTopologyInfo(VolumeFilter{}))
+	got := map[string]*CollectionStatistics{}
+	for _, s := range topo.CollectionStatistics() {
+		got[s.Collection] = s
+	}
+
+	if len(got) != len(want) {
+		t.Fatalf("summarised %d collections, the listing had %d: %v vs %v", len(got), len(want), got, want)
+	}
+	for name, expected := range want {
+		actual, found := got[name]
+		if !found {
+			t.Errorf("collection %s is missing from the summary", name)
+			continue
+		}
+		if *actual != *expected {
+			t.Errorf("collection %s:\n  summary %+v\n  listing %+v", name, *actual, *expected)
+		}
+	}
+}
+
+// Replicas disagree while a write is landing or a heartbeat is late. The full
+// listing walked the topology in map order and took whichever replica it
+// reached first, so its answer was not stable; the summary takes the largest.
+func TestCollectionStatisticsPicksTheLargestReplica(t *testing.T) {
+	topo := NewTopology("stats", nil, 32*1024*1024*1024, 5, false)
+	rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
+	small := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "a", map[string]uint32{"": 100})
+	large := rack.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "", "b", map[string]uint32{"": 100})
+
+	behind := &master_pb.VolumeInformationMessage{
+		Id: 1, Collection: "bucket-a", Size: 1000, FileCount: 10, Version: 3, ReplicaPlacement: 1,
+	}
+	ahead := &master_pb.VolumeInformationMessage{
+		Id: 1, Collection: "bucket-a", Size: 4000, FileCount: 40, Version: 3, ReplicaPlacement: 1,
+	}
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{behind}, small)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ahead}, large)
+
+	for i := 0; i < 8; i++ {
+		stats := topo.CollectionStatistics()
+		if len(stats) != 1 {
+			t.Fatalf("expected one collection, got %d", len(stats))
+		}
+		if stats[0].Size != 4000 || stats[0].FileCount != 40 {
+			t.Fatalf("run %d reported size %d files %d, want the larger replica's 4000 and 40",
+				i, stats[0].Size, stats[0].FileCount)
+		}
+		if stats[0].PhysicalSize != 5000 {
+			t.Fatalf("run %d reported physical size %d, want both replicas summed", i, stats[0].PhysicalSize)
+		}
+	}
+}
+
+func statsByCollection(topo *Topology) map[string]*CollectionStatistics {
+	out := map[string]*CollectionStatistics{}
+	for _, s := range topo.CollectionStatistics() {
+		out[s.Collection] = s
+	}
+	return out
+}
+
+// The .ecx and .ecj travel with the shards, so a holder still loading them
+// reports zero and must not pin the volume's count down.
+func TestCollectionStatisticsTakesTheLargestEcFileCount(t *testing.T) {
+	topo := NewTopology("stats", nil, 32*1024*1024*1024, 5, false)
+	rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
+	loading := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "a", map[string]uint32{"": 100})
+	loaded := rack.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "", "b", map[string]uint32{"": 100})
+
+	topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{
+		{Id: 11, Collection: "bucket-b", EcIndexBits: 0x7f, ShardSizes: []int64{1, 1, 1, 1, 1, 1, 1}, FileCount: 0},
+	}, loading)
+	topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{
+		{Id: 11, Collection: "bucket-b", EcIndexBits: 0x3f80, ShardSizes: []int64{1, 1, 1, 1, 1, 1, 1}, FileCount: 6},
+	}, loaded)
+
+	stats := statsByCollection(topo)["bucket-b"]
+	if stats == nil {
+		t.Fatal("expected bucket-b to be reported")
+	}
+	if stats.FileCount != 6 {
+		t.Errorf("file count %d, want the largest any holder reported (6)", stats.FileCount)
+	}
+	if stats.VolumeCount != 1 {
+		t.Errorf("volume count %d, want one volume however many holders report shards", stats.VolumeCount)
+	}
+	// 14 shards of 1 byte each; only the 10 data shards count as logical.
+	if stats.PhysicalSize != 14 {
+		t.Errorf("physical size %d, want every shard counted (14)", stats.PhysicalSize)
+	}
+	if stats.Size != 10 {
+		t.Errorf("size %d, want the data shards only (10)", stats.Size)
+	}
+}
+
+// A collection with both kinds has to have them added together.
+func TestCollectionStatisticsCountsRegularAndEcTogether(t *testing.T) {
+	topo := NewTopology("stats", nil, 32*1024*1024*1024, 5, false)
+	rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
+	dn := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "a", map[string]uint32{"": 100})
+
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		{Id: 1, Collection: "mixed", Size: 500, FileCount: 5, DeleteCount: 1, DeletedByteCount: 50, Version: 3},
+	}, dn)
+	topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{
+		{Id: 2, Collection: "mixed", EcIndexBits: 0x3fff,
+			ShardSizes: []int64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, FileCount: 7, DeleteCount: 2},
+	}, dn)
+
+	stats := statsByCollection(topo)["mixed"]
+	if stats == nil {
+		t.Fatal("expected the mixed collection to be reported")
+	}
+	if stats.VolumeCount != 2 {
+		t.Errorf("volume count %d, want the regular and the ec volume (2)", stats.VolumeCount)
+	}
+	if stats.FileCount != 12 {
+		t.Errorf("file count %d, want 5 regular plus 7 ec", stats.FileCount)
+	}
+	if stats.DeleteCount != 3 {
+		t.Errorf("delete count %d, want 1 regular plus 2 ec", stats.DeleteCount)
+	}
+	if stats.Size != 510 {
+		t.Errorf("size %d, want 500 regular plus 10 data shards", stats.Size)
+	}
+	if stats.PhysicalSize != 514 {
+		t.Errorf("physical size %d, want 500 regular plus all 14 shards", stats.PhysicalSize)
+	}
+}
+
+// Quotas are enforced on size less deletions, so the replica holding the most
+// live data is the one to count. The replica with the biggest raw size can be
+// the one that has deleted the most, and counting that one would report a
+// bucket smaller than it is and leave it writable over its quota.
+func TestCollectionStatisticsPicksTheReplicaHoldingTheMostLiveData(t *testing.T) {
+	topo := NewTopology("stats", nil, 32*1024*1024*1024, 5, false)
+	rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
+	bigMostlyDeleted := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "a", map[string]uint32{"": 100})
+	smallerButLive := rack.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "", "b", map[string]uint32{"": 100})
+
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{{
+		Id: 1, Collection: "bucket-a", Size: 1000, DeletedByteCount: 900,
+		Version: 3, ReplicaPlacement: 1,
+	}}, bigMostlyDeleted)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{{
+		Id: 1, Collection: "bucket-a", Size: 900, DeletedByteCount: 0,
+		Version: 3, ReplicaPlacement: 1,
+	}}, smallerButLive)
+
+	stats := statsByCollection(topo)["bucket-a"]
+	if stats == nil {
+		t.Fatal("expected bucket-a to be reported")
+	}
+	live := stats.Size - stats.DeletedByteCount
+	if live != 900 {
+		t.Errorf("reported %d bytes live (size %d less %d deleted), want the 900 one replica holds",
+			live, stats.Size, stats.DeletedByteCount)
+	}
+	if stats.PhysicalSize != 1900 {
+		t.Errorf("physical size %d, want both replicas summed", stats.PhysicalSize)
+	}
+}
+
+// Live usage is read as the collection's size less its deletions, so a volume
+// reporting more deleted bytes than it holds must not cancel live bytes
+// belonging to other volumes in the same bucket.
+func TestCollectionStatisticsDeletionsNeverExceedTheVolume(t *testing.T) {
+	topo := NewTopology("stats", nil, 32*1024*1024*1024, 5, false)
+	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "a", map[string]uint32{"": 100})
+
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		// More deleted bytes than the volume holds, which a compaction can
+		// leave behind, and a healthy volume beside it.
+		{Id: 1, Collection: "bucket-a", Size: 100, DeletedByteCount: 500, Version: 3},
+		{Id: 2, Collection: "bucket-a", Size: 1000, DeletedByteCount: 0, Version: 3},
+	}, dn)
+
+	stats := statsByCollection(topo)["bucket-a"]
+	if stats == nil {
+		t.Fatal("expected bucket-a to be reported")
+	}
+	if stats.DeletedByteCount > stats.Size {
+		t.Errorf("deletions %d exceed size %d, so live usage reads as zero", stats.DeletedByteCount, stats.Size)
+	}
+	if live := stats.Size - stats.DeletedByteCount; live != 1000 {
+		t.Errorf("reported %d bytes live, want the 1000 the second volume holds", live)
+	}
+}

weed/topology/configuration.go

@@ -0,0 +1,40 @@
+package topology
+
+import (
+	"encoding/xml"
+)
+
+type rack struct {
+	Name string   `xml:"name,attr"`
+	Ips  []string `xml:"Ip"`
+}
+type dataCenter struct {
+	Name  string `xml:"name,attr"`
+	Racks []rack `xml:"Rack"`
+}
+type topology struct {
+	DataCenters []dataCenter `xml:"DataCenter"`
+}
+type Configuration struct {
+	XMLName xml.Name `xml:"Configuration"`
+	Topo    topology `xml:"Topology"`
+}
+
+func (c *Configuration) String() string {
+	if b, e := xml.MarshalIndent(c, "  ", "  "); e == nil {
+		return string(b)
+	}
+	return ""
+}
+
+func (c *Configuration) Locate(ip string, dcName string, rackName string) (dc string, rack string) {
+	if dcName == "" {
+		dcName = "DefaultDataCenter"
+	}
+
+	if rackName == "" {
+		rackName = "DefaultRack"
+	}
+
+	return dcName, rackName
+}

weed/topology/crowded_writable_test.go

@@ -0,0 +1,125 @@
+package topology
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// Crowding asks for more room to write into, so a read-only volume has nothing
+// to say about it. Growth already discounts them by intersecting with the
+// writable list, so an entry for one is weight and nothing else.
+func TestCrowdedVolumesAreOnesThatCanTakeWrites(t *testing.T) {
+	topo := NewTopology("crowd", nil, 10000, 5, false)
+	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
+
+	// Both are past the growth threshold; only one can be written to.
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		{Id: 1, Size: 9500, Collection: "c", Version: 3},
+		{Id: 2, Size: 9500, Collection: "c", Version: 3, ReadOnly: true},
+	}, dn)
+	dn.LastSeen = time.Now().Unix()
+
+	// The channels are unbuffered and normally drained by the writable-refresh
+	// loop, which is not running here.
+	crowded := map[needle.VolumeId]bool{}
+	stop, done := make(chan struct{}), make(chan struct{})
+	go func() {
+		defer close(done)
+		for {
+			select {
+			case v := <-topo.chanCrowdedVolumes:
+				crowded[v.Id] = true
+			case <-topo.chanFullVolumes:
+			case <-stop:
+				return
+			}
+		}
+	}()
+	// The channels are unbuffered, so every send has been received by the time
+	// this returns; waiting for the collector to finish covers the recording.
+	topo.CollectDeadNodeAndFullVolumes(time.Now().Unix()-1, topo.volumeSizeLimit, 0.9)
+	close(stop)
+	<-done // the collector owns the map until it returns
+
+	if !crowded[needle.VolumeId(1)] {
+		t.Error("a writable volume past the threshold was not reported crowded")
+	}
+	if crowded[needle.VolumeId(2)] {
+		t.Error("a read-only volume was reported crowded, which only adds an entry growth then discounts")
+	}
+}
+
+// The count growth decides on has to be unchanged by this.
+func TestCrowdedCountStillMatchesWritables(t *testing.T) {
+	topo := NewTopology("crowd", nil, 10000, 5, false)
+	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		{Id: 1, Size: 9500, Collection: "c", Version: 3},
+	}, dn)
+
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
+	vl.SetVolumeCrowded(needle.VolumeId(1))
+
+	writable, crowded := vl.GetWritableVolumeCount()
+	if writable != 1 || crowded != 1 {
+		t.Errorf("writable=%d crowded=%d, want 1 and 1", writable, crowded)
+	}
+}
+
+// SetVolumeCrowded mutates the crowded map while GetWritableVolumeCount reads
+// it on the Assign hot path. Under RLock both could run at once and the Go
+// runtime aborts with "concurrent map read and map write". Run with -race to
+// reproduce the original bug; the write lock makes this pass.
+func TestSetVolumeCrowdedNoRaceWithGetWritableVolumeCount(t *testing.T) {
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := NewVolumeLayout(rp, needle.EMPTY_TTL, types.HardDriveType, 10000, false)
+	// GetWritableVolumeCount iterates vl.writables, so give it something to
+	// read while the crowded map is being mutated.
+	vl.writables = []needle.VolumeId{1, 2, 3}
+
+	var wg sync.WaitGroup
+	stop := make(chan struct{})
+	for range 8 {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case <-stop:
+					return
+				default:
+					vl.SetVolumeCrowded(needle.VolumeId(2))
+				}
+			}
+		}()
+	}
+	for range 8 {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case <-stop:
+					return
+				default:
+					vl.GetWritableVolumeCount()
+				}
+			}
+		}()
+	}
+
+	// The race detector needs only a brief window to flag a concurrent
+	// map read/write; let the goroutines hammer the lock briefly.
+	time.Sleep(50 * time.Millisecond)
+	close(stop)
+	wg.Wait()
+}

weed/topology/data_center.go

@@ -0,0 +1,69 @@
+package topology
+
+import (
+	"slices"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+type DataCenter struct {
+	NodeImpl
+}
+
+func NewDataCenter(id string) *DataCenter {
+	dc := &DataCenter{}
+	dc.id = NodeId(id)
+	dc.nodeType = "DataCenter"
+	dc.diskUsages = newDiskUsages()
+	dc.children = make(map[NodeId]Node)
+	dc.capacityReservations = newCapacityReservations()
+	dc.NodeImpl.value = dc
+	return dc
+}
+
+func (dc *DataCenter) GetOrCreateRack(rackName string) *Rack {
+	dc.Lock()
+	defer dc.Unlock()
+	for _, c := range dc.children {
+		rack := c.(*Rack)
+		if string(rack.Id()) == rackName {
+			return rack
+		}
+	}
+	rack := NewRack(rackName)
+	dc.doLinkChildNode(rack)
+	return rack
+}
+
+type DataCenterInfo struct {
+	Id    NodeId     `json:"Id"`
+	Racks []RackInfo `json:"Racks"`
+}
+
+func (dc *DataCenter) ToInfo() (info DataCenterInfo) {
+	info.Id = dc.Id()
+	var racks []RackInfo
+	for _, c := range dc.Children() {
+		rack := c.(*Rack)
+		racks = append(racks, rack.ToInfo())
+	}
+
+	slices.SortFunc(racks, func(a, b RackInfo) int {
+		return strings.Compare(string(a.Id), string(b.Id))
+	})
+	info.Racks = racks
+	return
+}
+
+func (dc *DataCenter) ToDataCenterInfo(filter VolumeFilter) *master_pb.DataCenterInfo {
+	m := &master_pb.DataCenterInfo{
+		Id:        string(dc.Id()),
+		DiskInfos: dc.diskUsages.ToDiskInfo(),
+	}
+	for _, c := range dc.Children() {
+		rack := c.(*Rack)
+		m.RackInfos = append(m.RackInfos, rack.ToRackInfo(filter))
+	}
+	return m
+}

weed/topology/data_node.go

@@ -0,0 +1,482 @@
+package topology
+
+import (
+	"fmt"
+	"sync/atomic"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type DataNode struct {
+	NodeImpl
+	Ip            string
+	Port          int
+	GrpcPort      int
+	PublicUrl     string
+	LastSeen      int64 // unix time in seconds
+	Counter       int   // in race condition, the previous dataNode was not dead
+	IsTerminating bool
+
+	// maintenanceMode mirrors the volume server's own flag, reported over the
+	// heartbeat. A server in maintenance is being drained: the master places
+	// no new volumes on it and assigns no writes to the volumes it holds. Read
+	// on the assign and volume-growth paths without the node lock.
+	maintenanceMode atomic.Bool
+	// lookupDigest covers the volumes reachable through this node in the volume
+	// layouts, for comparison against what its disks actually hold.
+	lookupDigest atomic.Uint64
+	// duplicateVolumeIds records that the node last reported one volume id more
+	// than once, which the master cannot represent.
+	duplicateVolumeIds atomic.Bool
+	// diskMetas holds each physical disk's tags, type, and capacity from the
+	// heartbeat DiskTags, including disks with no volumes or EC shards.
+	diskMetas map[uint32]diskMeta
+}
+
+type diskMeta struct {
+	tags           []string
+	diskType       types.DiskType
+	maxVolumeCount int64
+}
+
+func NewDataNode(id string) *DataNode {
+	dn := &DataNode{}
+	dn.id = NodeId(id)
+	dn.nodeType = "DataNode"
+	dn.diskUsages = newDiskUsages()
+	dn.children = make(map[NodeId]Node)
+	dn.capacityReservations = newCapacityReservations()
+	dn.NodeImpl.value = dn
+	return dn
+}
+
+func (dn *DataNode) String() string {
+	dn.RLock()
+	defer dn.RUnlock()
+	return fmt.Sprintf("Node:%s, Ip:%s, Port:%d, PublicUrl:%s", dn.NodeImpl.String(), dn.Ip, dn.Port, dn.PublicUrl)
+}
+
+// InMaintenanceMode reports whether the volume server asked to be left alone
+// for writes; see Topology.SetDataNodeMaintenanceMode for what that changes.
+func (dn *DataNode) InMaintenanceMode() bool {
+	return dn.maintenanceMode.Load()
+}
+
+// SetMaintenanceMode records the flag and reports whether it changed. Prefer
+// Topology.SetDataNodeMaintenanceMode, which also updates the writable lists.
+func (dn *DataNode) SetMaintenanceMode(on bool) (changed bool) {
+	return dn.maintenanceMode.Swap(on) != on
+}
+
+func (dn *DataNode) AddOrUpdateVolume(v storage.VolumeInfo) (isNew, isChangedRO, tierTransition bool) {
+	dn.Lock()
+	defer dn.Unlock()
+	return dn.doAddOrUpdateVolume(v)
+}
+
+// SetVolumeReadOnly records the read-only flag a volume server reported for
+// one of its volumes, ahead of the heartbeat that will repeat it.
+func (dn *DataNode) SetVolumeReadOnly(vid needle.VolumeId, readOnly bool) {
+	dn.Lock()
+	defer dn.Unlock()
+	for _, c := range dn.children {
+		if c.(*Disk).SetVolumeReadOnly(vid, readOnly) {
+			return
+		}
+	}
+}
+
+func (dn *DataNode) getOrCreateDisk(diskType string) *Disk {
+	c, found := dn.children[NodeId(diskType)]
+	if !found {
+		c = NewDisk(diskType)
+		dn.doLinkChildNode(c)
+	}
+	disk := c.(*Disk)
+	return disk
+}
+
+func (dn *DataNode) doAddOrUpdateVolume(v storage.VolumeInfo) (isNew, isChanged, tierTransition bool) {
+	disk := dn.getOrCreateDisk(v.DiskType)
+	return disk.AddOrUpdateVolume(v)
+}
+
+// AddProvisionalVolume records a volume the master registered on its own,
+// ahead of any server report naming it. See Disk.AddProvisionalVolume.
+func (dn *DataNode) AddProvisionalVolume(v storage.VolumeInfo) (isNew, isChanged, tierTransition bool) {
+	dn.Lock()
+	defer dn.Unlock()
+	disk := dn.getOrCreateDisk(v.DiskType)
+	return disk.AddProvisionalVolume(v)
+}
+
+// UpdateVolumes detects new/deleted/changed volumes on a volume server
+// used in master to notify master clients of these changes.
+//
+// changedVolumes covers every replica the disk already held whose
+// classification the new report altered in a way clients must learn about:
+// the ReadOnly flag flipped, or IsRemote() flipped on tier transition. The
+// latter is what lets the wdclient refresh DataInRemote after the digest
+// mismatch recovery path resends a full Volumes list -- that path is the
+// only way a re-tiered replica reaches the master without a separate
+// ChangedVolumes heartbeat.
+func (dn *DataNode) UpdateVolumes(actualVolumes []storage.VolumeInfo) (newVolumes, deletedVolumes, changedVolumes []storage.VolumeInfo) {
+
+	reported := newReportedVolumes(len(actualVolumes))
+	for _, v := range actualVolumes {
+		reported.add(v.Id, v.DiskType)
+	}
+
+	// A volume id mounted on two disks of one server -- a stale twin re-attached
+	// after a disk repair -- is reported twice, but the master keys volumes by
+	// id alone and keeps only the last copy. Its digest can then never equal the
+	// server's however often the list is resent, so record it and let the
+	// heartbeat fall back to the full list for this node.
+	dn.duplicateVolumeIds.Store(reported.duplicated)
+
+	dn.Lock()
+	defer dn.Unlock()
+
+	keptCount := 0
+	for _, c := range dn.children {
+		disk := c.(*Disk)
+		for _, v := range disk.RemoveVolumesNotIn(reported) {
+			glog.V(0).Infoln("Deleting volume id:", v.Id)
+			deletedVolumes = append(deletedVolumes, v)
+
+			deltaDiskUsage := &DiskUsageCounts{}
+			deltaDiskUsage.volumeCount = -1
+			if v.IsRemote() {
+				deltaDiskUsage.remoteVolumeCount = -1
+			}
+			if !v.ReadOnly {
+				deltaDiskUsage.activeVolumeCount = -1
+			}
+			disk.UpAdjustDiskUsageDelta(types.ToDiskType(v.DiskType), deltaDiskUsage)
+		}
+		keptCount += disk.VolumeCount()
+	}
+	// Everything still on the node is also in this heartbeat, so the remainder
+	// is what the node is about to gain. A steady-state heartbeat gains nothing
+	// and must not allocate here; a reconnecting server gains all of them.
+	if addedCount := reported.count() - keptCount; addedCount > 0 {
+		newVolumes = make([]storage.VolumeInfo, 0, addedCount)
+	}
+	for _, v := range actualVolumes {
+		isNew, isChanged, tierTransition := dn.doAddOrUpdateVolume(v)
+		if isNew {
+			newVolumes = append(newVolumes, v)
+		}
+		if isChanged || tierTransition {
+			changedVolumes = append(changedVolumes, v)
+		}
+	}
+	return
+}
+
+func (dn *DataNode) DeltaUpdateVolumes(newVolumes, deletedVolumes []storage.VolumeInfo) {
+	dn.Lock()
+	defer dn.Unlock()
+
+	for _, v := range deletedVolumes {
+		disk := dn.getOrCreateDisk(v.DiskType)
+
+		_, err := disk.GetVolumesById(v.Id)
+		if err != nil {
+			continue
+		}
+		disk.DeleteVolumeById(v.Id)
+
+		deltaDiskUsage := &DiskUsageCounts{}
+		deltaDiskUsage.volumeCount = -1
+		if v.IsRemote() {
+			deltaDiskUsage.remoteVolumeCount = -1
+		}
+		if !v.ReadOnly {
+			deltaDiskUsage.activeVolumeCount = -1
+		}
+		disk.UpAdjustDiskUsageDelta(types.ToDiskType(v.DiskType), deltaDiskUsage)
+	}
+	for _, v := range newVolumes {
+		dn.doAddOrUpdateVolume(v)
+	}
+	return
+}
+
+func (dn *DataNode) AdjustMaxVolumeCounts(maxVolumeCounts map[string]uint32) {
+	for diskType, maxVolumeCount := range maxVolumeCounts {
+		if maxVolumeCount == 0 {
+			// the volume server may have set the max to zero
+			continue
+		}
+		dt := types.ToDiskType(diskType)
+		currentDiskUsage := dn.diskUsages.getOrCreateDisk(dt)
+		currentDiskUsageMaxVolumeCount := atomic.LoadInt64(&currentDiskUsage.maxVolumeCount)
+		if currentDiskUsageMaxVolumeCount == int64(maxVolumeCount) {
+			continue
+		}
+		disk := dn.getOrCreateDisk(dt.String())
+		disk.UpAdjustDiskUsageDelta(dt, &DiskUsageCounts{
+			maxVolumeCount: int64(maxVolumeCount) - currentDiskUsageMaxVolumeCount,
+		})
+	}
+}
+
+// AdjustDiskUsageBytes records the physical filesystem capacity a volume server
+// reports per disk type, applied as a delta so it flows through the same
+// aggregation as the volume counts. Mirrors AdjustMaxVolumeCounts; entries with a
+// zero total are treated as "not reported" and skipped.
+func (dn *DataNode) AdjustDiskUsageBytes(diskTotalBytes, diskFreeBytes map[string]uint64) {
+	for diskType, totalBytes := range diskTotalBytes {
+		// Unlike maxVolumeCount, a 0 here is not "unset" but "not reported": let it
+		// flow through so a later heartbeat that drops physical-capacity reporting
+		// (e.g. statfs starts failing) clears the stale bytes and the gate falls
+		// back to slot-only instead of trusting outdated capacity.
+		dt := types.ToDiskType(diskType)
+		currentDiskUsage := dn.diskUsages.getOrCreateDisk(dt)
+		currentTotal := atomic.LoadInt64(&currentDiskUsage.diskTotalBytes)
+		currentFree := atomic.LoadInt64(&currentDiskUsage.diskFreeBytes)
+		newTotal := int64(totalBytes)
+		newFree := int64(diskFreeBytes[diskType])
+		if currentTotal == newTotal && currentFree == newFree {
+			continue
+		}
+		disk := dn.getOrCreateDisk(dt.String())
+		disk.UpAdjustDiskUsageDelta(dt, &DiskUsageCounts{
+			diskTotalBytes: newTotal - currentTotal,
+			diskFreeBytes:  newFree - currentFree,
+		})
+	}
+}
+
+// AppendVolumeIds appends the ids of this node's volumes to all, and repeats
+// the remote-tier ones on remote, without copying the volume records to read
+// them.
+func (dn *DataNode) AppendVolumeIds(all, remote, readOnly, readOnlyCanDelete []uint32) ([]uint32, []uint32, []uint32, []uint32) {
+	dn.RLock()
+	defer dn.RUnlock()
+	for _, c := range dn.children {
+		all, remote, readOnly, readOnlyCanDelete = c.(*Disk).AppendVolumeIds(all, remote, readOnly, readOnlyCanDelete)
+	}
+	return all, remote, readOnly, readOnlyCanDelete
+}
+
+func (dn *DataNode) GetVolumes() (ret []storage.VolumeInfo) {
+	dn.RLock()
+	defer dn.RUnlock()
+	total := 0
+	for _, c := range dn.children {
+		total += c.(*Disk).VolumeCount()
+	}
+	ret = make([]storage.VolumeInfo, 0, total)
+	for _, c := range dn.children {
+		ret = c.(*Disk).AppendVolumes(ret)
+	}
+	return ret
+}
+
+// HasDuplicateVolumeIds reports whether the node's last full report named one
+// volume id more than once. While it does, the node's digest is not meaningful.
+func (dn *DataNode) HasDuplicateVolumeIds() bool {
+	return dn.duplicateVolumeIds.Load()
+}
+
+// VolumeDigest summarises every volume the master believes this node holds. A
+// volume server that reports a different digest has drifted from the master and
+// needs to resend its volume list.
+func (dn *DataNode) VolumeDigest() uint64 {
+	dn.RLock()
+	defer dn.RUnlock()
+	var digest uint64
+	for _, c := range dn.children {
+		digest ^= c.(*Disk).VolumeDigest()
+	}
+	return digest
+}
+
+func (dn *DataNode) GetVolumesById(id needle.VolumeId) (vInfo storage.VolumeInfo, err error) {
+	dn.RLock()
+	defer dn.RUnlock()
+	found := false
+	for _, c := range dn.children {
+		disk := c.(*Disk)
+		vInfo, err = disk.GetVolumesById(id)
+		if err == nil {
+			found = true
+			break
+		}
+	}
+	if found {
+		return vInfo, nil
+	} else {
+		return storage.VolumeInfo{}, fmt.Errorf("volumeInfo not found")
+	}
+}
+
+func (dn *DataNode) GetDataCenter() *DataCenter {
+	rack := dn.Parent()
+	if rack == nil {
+		return nil
+	}
+	dcNode := rack.Parent()
+	if dcNode == nil {
+		return nil
+	}
+	dcValue := dcNode.GetValue()
+	return dcValue.(*DataCenter)
+}
+
+func (dn *DataNode) GetDataCenterId() string {
+	if dc := dn.GetDataCenter(); dc != nil {
+		return string(dc.Id())
+	}
+	return ""
+}
+
+func (dn *DataNode) GetRack() *Rack {
+	return dn.Parent().(*NodeImpl).value.(*Rack)
+}
+
+func (dn *DataNode) GetTopology() *Topology {
+	p := dn.Parent()
+	for p.Parent() != nil {
+		p = p.Parent()
+	}
+	t := p.(*Topology)
+	return t
+}
+
+func (dn *DataNode) MatchLocation(ip string, port int) bool {
+	return dn.Ip == ip && dn.Port == port
+}
+
+func (dn *DataNode) Url() string {
+	return util.JoinHostPort(dn.Ip, dn.Port)
+}
+
+func (dn *DataNode) ServerAddress() pb.ServerAddress {
+	return pb.NewServerAddress(dn.Ip, dn.Port, dn.GrpcPort)
+}
+
+type DataNodeInfo struct {
+	Url       string `json:"Url"`
+	PublicUrl string `json:"PublicUrl"`
+	Volumes   int64  `json:"Volumes"`
+	EcShards  int64  `json:"EcShards"`
+	Max       int64  `json:"Max"`
+	VolumeIds string `json:"VolumeIds"`
+}
+
+func (dn *DataNode) ToInfo() (info DataNodeInfo) {
+	info.Url = dn.Url()
+	info.PublicUrl = dn.PublicUrl
+
+	// aggregated volume info
+	var volumeCount, ecShardCount, maxVolumeCount int64
+	var volumeIds string
+	for _, diskUsage := range dn.diskUsages.usages {
+		volumeCount += diskUsage.volumeCount
+		ecShardCount += diskUsage.ecShardCount
+		maxVolumeCount += diskUsage.maxVolumeCount
+	}
+
+	for _, disk := range dn.Children() {
+		d := disk.(*Disk)
+		volumeIds += " " + d.GetVolumeIds()
+	}
+
+	info.Volumes = volumeCount
+	info.EcShards = ecShardCount
+	info.Max = maxVolumeCount
+	info.VolumeIds = volumeIds
+
+	return
+}
+
+func (dn *DataNode) ToDataNodeInfo(filter VolumeFilter) *master_pb.DataNodeInfo {
+	m := &master_pb.DataNodeInfo{
+		Id: string(dn.Id()),
+		// Start from disk usage counters so empty disks are still represented
+		// even when there are no volumes/EC shards on this data node yet.
+		DiskInfos: dn.diskUsages.ToDiskInfo(),
+		GrpcPort:  uint32(dn.GrpcPort),
+		Address:   dn.Url(), // ip:port for connecting to the volume server
+	}
+	if m.DiskInfos == nil {
+		m.DiskInfos = make(map[string]*master_pb.DiskInfo)
+	}
+	for diskType, diskInfo := range m.DiskInfos {
+		if diskInfo == nil {
+			m.DiskInfos[diskType] = &master_pb.DiskInfo{Type: diskType}
+			continue
+		}
+		diskInfo.Type = diskType
+	}
+
+	for _, c := range dn.Children() {
+		disk := c.(*Disk)
+		m.DiskInfos[string(disk.Id())] = disk.ToDiskInfo(filter)
+	}
+
+	dn.RLock()
+	metas := make(map[uint32]diskMeta, len(dn.diskMetas))
+	for diskID, meta := range dn.diskMetas {
+		metas[diskID] = meta
+	}
+	dn.RUnlock()
+	for _, diskInfo := range m.DiskInfos {
+		if diskInfo == nil {
+			continue
+		}
+		if meta, found := metas[diskInfo.DiskId]; found {
+			diskInfo.Tags = append([]string(nil), meta.tags...)
+		}
+		// Max per physical disk of this type, empty and unavailable (max 0) ones
+		// included. Emit only when some disk reports capacity, so an older server
+		// sending all zeros leaves the map nil and falls back.
+		diskType := types.ToDiskType(diskInfo.Type)
+		maxByDisk := make(map[uint32]int64)
+		anyCapacity := false
+		for diskID, meta := range metas {
+			if meta.diskType != diskType {
+				continue
+			}
+			if meta.maxVolumeCount > 0 {
+				anyCapacity = true
+			}
+			maxByDisk[diskID] = meta.maxVolumeCount
+		}
+		if anyCapacity {
+			diskInfo.MaxVolumeCountByDisk = maxByDisk
+		}
+	}
+	return m
+}
+
+func (dn *DataNode) UpdateDiskTags(tags []*master_pb.DiskTag) {
+	if len(tags) == 0 {
+		return
+	}
+	// DiskTags is the full list on each full heartbeat; rebuild fresh to drop
+	// removed disks.
+	metas := make(map[uint32]diskMeta, len(tags))
+	for _, tagInfo := range tags {
+		if tagInfo == nil {
+			continue
+		}
+		metas[tagInfo.DiskId] = diskMeta{
+			tags:           append([]string(nil), tagInfo.Tags...),
+			diskType:       types.ToDiskType(tagInfo.Type),
+			maxVolumeCount: tagInfo.MaxVolumeCount,
+		}
+	}
+	dn.Lock()
+	dn.diskMetas = metas
+	dn.Unlock()
+}

weed/topology/data_node_ec.go

@@ -0,0 +1,199 @@
+package topology
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func (dn *DataNode) GetEcShards() (ret []*erasure_coding.EcVolumeInfo) {
+	dn.RLock()
+	for _, c := range dn.children {
+		disk := c.(*Disk)
+		ret = append(ret, disk.GetEcShards()...)
+	}
+	dn.RUnlock()
+	return ret
+}
+
+// ecShardKey identifies a per-physical-disk EC shard entry on a DataNode.
+// A single volume's EC shards can live on multiple physical disks of one
+// DataNode (e.g. a 10+4 volume spread across 4 mount points), so entries
+// must be tracked per (volume, physical disk) rather than per volume alone.
+// See issue #9212.
+type ecShardKey struct {
+	vid    needle.VolumeId
+	diskId types.DiskId
+}
+
+func (dn *DataNode) UpdateEcShards(actualShards []*erasure_coding.EcVolumeInfo) (newShards, deletedShards []*erasure_coding.EcVolumeInfo) {
+	// Aggregate incoming messages by (volume, physical disk). Duplicates for
+	// the same (vid, diskId) are merged; entries for the same vid on
+	// different disks stay separate so per-physical-disk attribution is
+	// preserved all the way to DiskInfo.EcShardInfos — which the admin
+	// shell's ec.balance / ec.rebuild planners read via eci.DiskId.
+	actualByKey := make(map[ecShardKey]*erasure_coding.EcVolumeInfo, len(actualShards))
+	for _, ecShards := range actualShards {
+		key := ecShardKey{vid: ecShards.VolumeId, diskId: types.DiskId(ecShards.DiskId)}
+		if existing, ok := actualByKey[key]; ok {
+			existing.ShardsInfo.Add(ecShards.ShardsInfo)
+			continue
+		}
+		// Clone so subsequent merges for the same key don't mutate the
+		// caller's EcVolumeInfo, and so the diff below is stable.
+		clone := *ecShards
+		clone.ShardsInfo = ecShards.ShardsInfo.Copy()
+		actualByKey[key] = &clone
+	}
+
+	// Hold dn.Lock for the full read-diff-write cycle so concurrent heartbeats,
+	// getOrCreateDisk calls, and UpAdjustDiskUsageDelta updates on this data
+	// node are serialized with us — matching UpdateVolumes' locking model.
+	// Internal helpers below assume the caller holds dn.Lock.
+	dn.Lock()
+	defer dn.Unlock()
+
+	existingEcShards := dn.getEcShardsLocked()
+
+	// find out the newShards and deletedShards
+	for _, ecShards := range existingEcShards {
+
+		var newShardCount, deletedShardCount int
+		disk := dn.getOrCreateDisk(ecShards.DiskType)
+
+		key := ecShardKey{vid: ecShards.VolumeId, diskId: types.DiskId(ecShards.DiskId)}
+		if actualEcShards, ok := actualByKey[key]; !ok {
+			// dn registered ec shards not found in the new set of ec shards
+			deletedShards = append(deletedShards, ecShards)
+			deletedShardCount += ecShards.ShardsInfo.Count()
+		} else {
+			// found, but maybe the actual shard could be missing
+			a := actualEcShards.Minus(ecShards)
+			if a.ShardsInfo.Count() > 0 {
+				newShards = append(newShards, a)
+				newShardCount += a.ShardsInfo.Count()
+			}
+			d := ecShards.Minus(actualEcShards)
+			if d.ShardsInfo.Count() > 0 {
+				deletedShards = append(deletedShards, d)
+				deletedShardCount += d.ShardsInfo.Count()
+			}
+		}
+
+		if (newShardCount - deletedShardCount) != 0 {
+			disk.UpAdjustDiskUsageDelta(types.ToDiskType(ecShards.DiskType), &DiskUsageCounts{
+				ecShardCount: int64(newShardCount - deletedShardCount),
+			})
+		}
+
+	}
+
+	existingKeys := make(map[ecShardKey]struct{}, len(existingEcShards))
+	for _, ev := range existingEcShards {
+		existingKeys[ecShardKey{vid: ev.VolumeId, diskId: types.DiskId(ev.DiskId)}] = struct{}{}
+	}
+	for key, ecShards := range actualByKey {
+		if _, found := existingKeys[key]; found {
+			continue
+		}
+
+		newShards = append(newShards, ecShards)
+
+		disk := dn.getOrCreateDisk(ecShards.DiskType)
+		disk.UpAdjustDiskUsageDelta(types.ToDiskType(ecShards.DiskType), &DiskUsageCounts{
+			ecShardCount: int64(ecShards.ShardsInfo.Count()),
+		})
+	}
+
+	if len(newShards) > 0 || len(deletedShards) > 0 {
+		// if changed, set to the new ec shard map
+		dn.doUpdateEcShardsLocked(actualByKey)
+	}
+
+	return
+}
+
+// getEcShardsLocked returns the flat list of per-physical-disk EC shard
+// entries across all children. Caller MUST hold dn.Lock (or RLock); each
+// disk's ecShardsLock is taken internally via Disk.GetEcShards.
+func (dn *DataNode) getEcShardsLocked() (ret []*erasure_coding.EcVolumeInfo) {
+	for _, c := range dn.children {
+		disk := c.(*Disk)
+		ret = append(ret, disk.GetEcShards()...)
+	}
+	return ret
+}
+
+func (dn *DataNode) HasEcShards(volumeId needle.VolumeId) (found bool) {
+	dn.RLock()
+	defer dn.RUnlock()
+	for _, c := range dn.children {
+		disk := c.(*Disk)
+		disk.ecShardsLock.RLock()
+		byDisk, ok := disk.ecShards[volumeId]
+		has := ok && len(byDisk) > 0
+		disk.ecShardsLock.RUnlock()
+		if has {
+			return true
+		}
+	}
+	return
+}
+
+// doUpdateEcShardsLocked rewrites disk.ecShards from actualByKey. Caller
+// MUST hold dn.Lock; each disk's ecShardsLock is taken internally around
+// the read-modify-write of its ecShards map.
+func (dn *DataNode) doUpdateEcShardsLocked(actualByKey map[ecShardKey]*erasure_coding.EcVolumeInfo) {
+	for _, c := range dn.children {
+		disk := c.(*Disk)
+		disk.ecShardsLock.Lock()
+		disk.ecShards = make(map[needle.VolumeId]map[types.DiskId]*erasure_coding.EcVolumeInfo)
+		disk.ecShardsLock.Unlock()
+	}
+	for _, shard := range actualByKey {
+		disk := dn.getOrCreateDisk(shard.DiskType)
+		disk.ecShardsLock.Lock()
+		byDisk, ok := disk.ecShards[shard.VolumeId]
+		if !ok {
+			byDisk = make(map[types.DiskId]*erasure_coding.EcVolumeInfo, 1)
+			disk.ecShards[shard.VolumeId] = byDisk
+		}
+		byDisk[types.DiskId(shard.DiskId)] = shard
+		disk.ecShardsLock.Unlock()
+	}
+}
+
+func (dn *DataNode) DeltaUpdateEcShards(newShards, deletedShards []*erasure_coding.EcVolumeInfo) {
+	for _, newShard := range newShards {
+		dn.AddOrUpdateEcShard(newShard)
+	}
+
+	for _, deletedShard := range deletedShards {
+		dn.DeleteEcShard(deletedShard)
+	}
+
+}
+
+func (dn *DataNode) AddOrUpdateEcShard(s *erasure_coding.EcVolumeInfo) {
+	disk := dn.getOrCreateDisk(s.DiskType)
+	disk.AddOrUpdateEcShard(s)
+}
+
+func (dn *DataNode) DeleteEcShard(s *erasure_coding.EcVolumeInfo) {
+	disk := dn.getOrCreateDisk(s.DiskType)
+	disk.DeleteEcShard(s)
+}
+
+func (dn *DataNode) HasVolumesById(volumeId needle.VolumeId) (hasVolumeId bool) {
+
+	dn.RLock()
+	defer dn.RUnlock()
+	for _, c := range dn.children {
+		disk := c.(*Disk)
+		if disk.HasVolumesById(volumeId) {
+			return true
+		}
+	}
+	return false
+
+}

weed/topology/data_node_ec_multi_disk_test.go

@@ -0,0 +1,177 @@
+package topology
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// TestEcShardsAcrossMultipleDisksOnSameNode reproduces issue #9212.
+// When a volume server reports EC shards of the same volume spread across
+// multiple physical disks on the same node, the master must register ALL
+// shards, not only a subset.
+func TestEcShardsAcrossMultipleDisksOnSameNode(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	maxVolumeCounts := map[string]uint32{"": 100}
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
+
+	const vid = uint32(15)
+	const collection = "grafana-loki"
+	const diskType = "" // HDD / default type — all 4 disks share this type
+
+	// Volume 15 has its 14 EC shards spread across 3 physical disks on this node.
+	// Mirrors the "volume-2" row from issue #9212:
+	//   /data1 (diskId 0): ec02, ec06, ec10
+	//   /data2 (diskId 1): ec01, ec04, ec09
+	//   /data4 (diskId 3): ec08, ec12
+	disk0 := buildEcShardMessage(vid, collection, diskType, 0, []erasure_coding.ShardId{2, 6, 10})
+	disk1 := buildEcShardMessage(vid, collection, diskType, 1, []erasure_coding.ShardId{1, 4, 9})
+	disk3 := buildEcShardMessage(vid, collection, diskType, 3, []erasure_coding.ShardId{8, 12})
+
+	msgs := []*master_pb.VolumeEcShardInformationMessage{disk0, disk1, disk3}
+	topo.SyncDataNodeEcShards(msgs, dn)
+
+	locs, ok := topo.LookupEcShards(needle.VolumeId(vid))
+	if !ok {
+		t.Fatalf("volume %d: no ec shard locations registered at all", vid)
+	}
+
+	// All 8 shards should be visible to the master: 1,2,4,6,8,9,10,12.
+	wantShards := []erasure_coding.ShardId{1, 2, 4, 6, 8, 9, 10, 12}
+	var gotShards []erasure_coding.ShardId
+	for shardId, dataNodes := range locs.Locations {
+		if len(dataNodes) > 0 {
+			gotShards = append(gotShards, erasure_coding.ShardId(shardId))
+		}
+	}
+
+	if len(gotShards) != len(wantShards) {
+		t.Errorf("volume %d: topology.LookupEcShards sees %d shards %v, want %d shards %v",
+			vid, len(gotShards), gotShards, len(wantShards), wantShards)
+	}
+
+	for _, want := range wantShards {
+		if len(locs.Locations[want]) == 0 {
+			t.Errorf("volume %d: shard %d missing from topology (bug #9212)", vid, want)
+		}
+	}
+
+	// The DataNode's own view (what drives volume.list output, admin UI, and
+	// ec.rebuild dry-run diagnostics) must also see all shards across all disks.
+	dnShards := dn.GetEcShards()
+	dnShardCount := 0
+	dnShardBitmap := erasure_coding.ShardBits(0)
+	for _, ev := range dnShards {
+		if ev.VolumeId == needle.VolumeId(vid) {
+			dnShardCount += ev.ShardsInfo.Count()
+			dnShardBitmap |= erasure_coding.ShardBits(ev.ShardsInfo.Bitmap())
+		}
+	}
+	if dnShardCount != len(wantShards) {
+		t.Errorf("volume %d: DataNode.GetEcShards reports %d shards (bitmap=0b%b), want %d shards %v (bug #9212)",
+			vid, dnShardCount, dnShardBitmap, len(wantShards), wantShards)
+	}
+
+	// Per-physical-disk attribution must survive all the way to the protobuf
+	// DiskInfo.EcShardInfos consumed by ec.balance / ec.rebuild planners. The
+	// admin shell groups shards by eci.DiskId, so each physical disk needs a
+	// separate message with its own shard subset.
+	wantPerDisk := map[uint32][]erasure_coding.ShardId{
+		0: {2, 6, 10},
+		1: {1, 4, 9},
+		3: {8, 12},
+	}
+	dnInfo := dn.ToDataNodeInfo(VolumeFilter{})
+	gotPerDisk := map[uint32][]erasure_coding.ShardId{}
+	for _, diskInfo := range dnInfo.DiskInfos {
+		for _, eci := range diskInfo.EcShardInfos {
+			if eci.Id != vid {
+				continue
+			}
+			si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci)
+			gotPerDisk[eci.DiskId] = append(gotPerDisk[eci.DiskId], si.Ids()...)
+		}
+	}
+	for diskId, want := range wantPerDisk {
+		got := gotPerDisk[diskId]
+		if !shardSetEqual(got, want) {
+			t.Errorf("volume %d diskId %d: DiskInfo.EcShardInfos report shards %v, want %v (per-physical-disk attribution lost)",
+				vid, diskId, got, want)
+		}
+	}
+	for diskId := range gotPerDisk {
+		if _, ok := wantPerDisk[diskId]; !ok {
+			t.Errorf("volume %d: unexpected diskId %d in DiskInfo.EcShardInfos, shards=%v",
+				vid, diskId, gotPerDisk[diskId])
+		}
+	}
+}
+
+func shardSetEqual(a, b []erasure_coding.ShardId) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	seen := make(map[erasure_coding.ShardId]int, len(a))
+	for _, id := range a {
+		seen[id]++
+	}
+	for _, id := range b {
+		seen[id]--
+		if seen[id] < 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// TestEcShardsAfterRestartHeartbeat simulates the exact issue #9212 flow:
+// the volume server starts up, loads shards from each disk, and sends a
+// single full-sync heartbeat containing one VolumeEcShardInformationMessage
+// per (disk, volume). The master must end up with all shards visible per
+// DataNode, not just the subset belonging to whichever disk happened to be
+// iterated last.
+func TestEcShardsAfterRestartHeartbeat(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 100})
+
+	// Matches "volume-1" from the bug report: ec13 on /data1, ec03 on /data3.
+	msgs := []*master_pb.VolumeEcShardInformationMessage{
+		buildEcShardMessage(15, "grafana-loki", "", 0, []erasure_coding.ShardId{13}),
+		buildEcShardMessage(15, "grafana-loki", "", 2, []erasure_coding.ShardId{3}),
+	}
+	topo.SyncDataNodeEcShards(msgs, dn)
+
+	dnShards := dn.GetEcShards()
+	var combined erasure_coding.ShardBits
+	for _, ev := range dnShards {
+		if ev.VolumeId == 15 {
+			combined |= erasure_coding.ShardBits(ev.ShardsInfo.Bitmap())
+		}
+	}
+	if combined.Count() != 2 {
+		t.Errorf("volume 15: DataNode sees %d shards (bitmap=0b%b) after full sync of 2 disks; want both shards 3 and 13 visible (bug #9212)",
+			combined.Count(), combined)
+	}
+}
+
+func buildEcShardMessage(vid uint32, collection, diskType string, diskId uint32, shardIds []erasure_coding.ShardId) *master_pb.VolumeEcShardInformationMessage {
+	si := erasure_coding.NewShardsInfo()
+	for _, sid := range shardIds {
+		si.Set(erasure_coding.NewShardInfo(sid, erasure_coding.ShardSize(1024)))
+	}
+	return &master_pb.VolumeEcShardInformationMessage{
+		Id:          vid,
+		Collection:  collection,
+		DiskType:    diskType,
+		DiskId:      diskId,
+		EcIndexBits: si.Bitmap(),
+		ShardSizes:  si.SizesInt64(),
+	}
+}

weed/topology/data_node_empty_disk_test.go

@@ -0,0 +1,88 @@
+package topology
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+)
+
+// A disk with no volumes/shards surfaces via DiskTags.
+func TestToDataNodeInfoReportsEmptyPhysicalDisks(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 1000})
+	dn.AdjustMaxVolumeCounts(map[string]uint32{"": 1000})
+
+	dn.UpdateDiskTags([]*master_pb.DiskTag{
+		{DiskId: 0, Type: "", MaxVolumeCount: 350},
+		{DiskId: 1, Type: "", MaxVolumeCount: 350},
+		{DiskId: 2, Type: "", MaxVolumeCount: 300}, // empty disk, never held a volume
+	})
+
+	info := dn.ToDataNodeInfo(VolumeFilter{})
+	di, ok := info.DiskInfos[""]
+	if !ok {
+		t.Fatalf("missing HDD disk info")
+	}
+	if len(di.MaxVolumeCountByDisk) != 3 {
+		t.Fatalf("want 3 physical disks reported, got %d", len(di.MaxVolumeCountByDisk))
+	}
+
+	split := di.SplitByPhysicalDisk()
+	if len(split) != 3 {
+		t.Fatalf("SplitByPhysicalDisk: want 3 disks including the empty one, got %d", len(split))
+	}
+	byID := map[uint32]*master_pb.DiskInfo{}
+	for _, d := range split {
+		byID[d.DiskId] = d
+	}
+	if byID[2] == nil {
+		t.Fatalf("empty disk 2 not surfaced")
+	}
+	if byID[2].MaxVolumeCount != 300 || byID[2].VolumeCount != 0 {
+		t.Errorf("empty disk 2: want max 300 / 0 volumes, got max %d / %d volumes",
+			byID[2].MaxVolumeCount, byID[2].VolumeCount)
+	}
+}
+
+// A max-0 (unavailable) disk stays listed when the node reports capacity.
+func TestToDataNodeInfoKeepsZeroCapacityDisk(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 700})
+	dn.AdjustMaxVolumeCounts(map[string]uint32{"": 700})
+
+	dn.UpdateDiskTags([]*master_pb.DiskTag{
+		{DiskId: 0, Type: "", MaxVolumeCount: 350},
+		{DiskId: 1, Type: "", MaxVolumeCount: 350},
+		{DiskId: 2, Type: "", MaxVolumeCount: 0}, // unavailable disk
+	})
+
+	di := dn.ToDataNodeInfo(VolumeFilter{}).DiskInfos[""]
+	if len(di.MaxVolumeCountByDisk) != 3 {
+		t.Fatalf("want 3 physical disks (incl the zero-capacity one), got %d", len(di.MaxVolumeCountByDisk))
+	}
+}
+
+// An older server sending no per-disk capacity leaves the map empty.
+func TestToDataNodeInfoFallsBackWhenNoCapacityReported(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 700})
+	dn.AdjustMaxVolumeCounts(map[string]uint32{"": 700})
+
+	// Older server: DiskTags carry disk_id (+tags) but no type/max.
+	dn.UpdateDiskTags([]*master_pb.DiskTag{
+		{DiskId: 0},
+		{DiskId: 1},
+	})
+
+	di := dn.ToDataNodeInfo(VolumeFilter{}).DiskInfos[""]
+	if len(di.MaxVolumeCountByDisk) != 0 {
+		t.Fatalf("want no per-disk max (fallback), got %d", len(di.MaxVolumeCountByDisk))
+	}
+}

weed/topology/disk.go

@@ -0,0 +1,550 @@
+package topology
+
+import (
+	"fmt"
+	"slices"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+)
+
+type Disk struct {
+	NodeImpl
+	volumes map[needle.VolumeId]*storage.VolumeInfo
+	// ecShards is nested so the same volume can retain separate entries per
+	// physical disk id. A single topology Disk represents one DiskType on a
+	// DataNode and may front multiple physical disks of that type, so EC
+	// shards of one volume can legitimately live on several of them. The
+	// outer key is the volume id; the inner key is the physical disk id.
+	ecShards     map[needle.VolumeId]map[types.DiskId]*erasure_coding.EcVolumeInfo
+	ecShardsLock sync.RWMutex
+	// volumeDigest is the xor of every volume's ReportHash. Order-independent
+	// and its own inverse, so it stays current by xoring a volume out before
+	// its old state is dropped and back in after the new one lands.
+	volumeDigest uint64
+	// volumeIdDigest covers which volumes are on the disk, ignoring their
+	// state, so it can be compared against the lookup index the master serves
+	// reads from. The two indexes are maintained separately and have been seen
+	// to drift.
+	volumeIdDigest uint64
+	// volumeAddedAt remembers when each volume reached this view of the disk
+	// without a server report having confirmed it yet. Registration by the
+	// master itself -- volume growth -- races the heartbeat in flight, which
+	// cannot name a volume created after it was collected.
+	volumeAddedAt map[needle.VolumeId]time.Time
+}
+
+// volumeRemovalGracePeriod is how long an unconfirmed volume survives a report
+// that does not name it. Removing a just-grown volume strands its collection
+// without writable volumes, so the report that raced the grow does not get to
+// erase it; the cap keeps a registration that never materializes server-side
+// from lingering forever.
+const volumeRemovalGracePeriod = 10 * time.Second
+
+func NewDisk(diskType string) *Disk {
+	s := &Disk{}
+	s.id = NodeId(diskType)
+	s.nodeType = "Disk"
+	s.diskUsages = newDiskUsages()
+	s.volumes = make(map[needle.VolumeId]*storage.VolumeInfo, 2)
+	s.volumeAddedAt = make(map[needle.VolumeId]time.Time, 2)
+	s.ecShards = make(map[needle.VolumeId]map[types.DiskId]*erasure_coding.EcVolumeInfo, 2)
+	s.NodeImpl.value = s
+	return s
+}
+
+type DiskUsages struct {
+	sync.RWMutex
+	usages map[types.DiskType]*DiskUsageCounts
+}
+
+func newDiskUsages() *DiskUsages {
+	return &DiskUsages{
+		usages: make(map[types.DiskType]*DiskUsageCounts),
+	}
+}
+
+func (d *DiskUsages) negative() *DiskUsages {
+	d.RLock()
+	defer d.RUnlock()
+	t := newDiskUsages()
+	for diskType, b := range d.usages {
+		a := t.getOrCreateDisk(diskType)
+		a.volumeCount = -b.volumeCount
+		a.remoteVolumeCount = -b.remoteVolumeCount
+		a.activeVolumeCount = -b.activeVolumeCount
+		a.ecShardCount = -b.ecShardCount
+		a.maxVolumeCount = -b.maxVolumeCount
+		a.diskTotalBytes = -b.diskTotalBytes
+		a.diskFreeBytes = -b.diskFreeBytes
+
+	}
+	return t
+}
+
+func (d *DiskUsages) ToDiskInfo() map[string]*master_pb.DiskInfo {
+	d.RLock()
+	defer d.RUnlock()
+	ret := make(map[string]*master_pb.DiskInfo)
+	for diskType, diskUsageCounts := range d.usages {
+		usage := diskUsageCounts.snapshot()
+		m := &master_pb.DiskInfo{
+			VolumeCount:       usage.volumeCount,
+			MaxVolumeCount:    usage.maxVolumeCount,
+			FreeVolumeCount:   usage.maxVolumeCount - (usage.volumeCount - usage.remoteVolumeCount) - erasure_coding.VolumeSlots(usage.ecShardCount),
+			ActiveVolumeCount: usage.activeVolumeCount,
+			RemoteVolumeCount: usage.remoteVolumeCount,
+			DiskTotalBytes:    uint64(max(0, usage.diskTotalBytes)),
+			DiskFreeBytes:     uint64(max(0, usage.diskFreeBytes)),
+		}
+		ret[string(diskType)] = m
+	}
+	return ret
+}
+
+func (d *DiskUsages) FreeSpace() (freeSpace int64) {
+	d.RLock()
+	defer d.RUnlock()
+	for _, diskUsage := range d.usages {
+		freeSpace += diskUsage.FreeSpace()
+	}
+	return
+}
+
+func (d *DiskUsages) GetMaxVolumeCount() (maxVolumeCount int64) {
+	d.RLock()
+	defer d.RUnlock()
+	for _, diskUsage := range d.usages {
+		maxVolumeCount += diskUsage.maxVolumeCount
+	}
+	return
+}
+
+// FreeBytes sums the space one volume server reports as still free on its
+// filesystems. reported is false as soon as a disk holding volume slots says
+// nothing -- a volume server older than the field looks that way -- since
+// leaving its space out would understate the room the server has.
+func (d *DiskUsages) FreeBytes() (freeBytes uint64, reported bool) {
+	d.RLock()
+	defer d.RUnlock()
+	for _, diskUsageCounts := range d.usages {
+		usage := diskUsageCounts.snapshot()
+		if usage.diskTotalBytes <= 0 {
+			if usage.maxVolumeCount > 0 {
+				return 0, false
+			}
+			continue
+		}
+		freeBytes += uint64(max(0, usage.diskFreeBytes))
+	}
+	return freeBytes, true
+}
+
+type DiskUsageCounts struct {
+	volumeCount       int64
+	remoteVolumeCount int64
+	activeVolumeCount int64
+	ecShardCount      int64
+	maxVolumeCount    int64
+	// Physical filesystem capacity reported by the volume server, in bytes.
+	// 0 means the volume server did not report it (e.g. an older build).
+	diskTotalBytes int64
+	diskFreeBytes  int64
+}
+
+func (a *DiskUsageCounts) addDiskUsageCounts(b *DiskUsageCounts) {
+	atomic.AddInt64(&a.volumeCount, b.volumeCount)
+	atomic.AddInt64(&a.remoteVolumeCount, b.remoteVolumeCount)
+	atomic.AddInt64(&a.activeVolumeCount, b.activeVolumeCount)
+	atomic.AddInt64(&a.ecShardCount, b.ecShardCount)
+	atomic.AddInt64(&a.maxVolumeCount, b.maxVolumeCount)
+	atomic.AddInt64(&a.diskTotalBytes, b.diskTotalBytes)
+	atomic.AddInt64(&a.diskFreeBytes, b.diskFreeBytes)
+}
+
+// snapshot reads each counter atomically, so a reader sees whole values rather
+// than ones a concurrent heartbeat is halfway through writing. They are still
+// read one at a time, so they need not all describe the same instant.
+func (a *DiskUsageCounts) snapshot() DiskUsageCounts {
+	return DiskUsageCounts{
+		volumeCount:       atomic.LoadInt64(&a.volumeCount),
+		remoteVolumeCount: atomic.LoadInt64(&a.remoteVolumeCount),
+		activeVolumeCount: atomic.LoadInt64(&a.activeVolumeCount),
+		ecShardCount:      atomic.LoadInt64(&a.ecShardCount),
+		maxVolumeCount:    atomic.LoadInt64(&a.maxVolumeCount),
+		diskTotalBytes:    atomic.LoadInt64(&a.diskTotalBytes),
+		diskFreeBytes:     atomic.LoadInt64(&a.diskFreeBytes),
+	}
+}
+
+func (a *DiskUsageCounts) FreeSpace() int64 {
+	u := a.snapshot()
+	return u.maxVolumeCount + u.remoteVolumeCount - u.volumeCount - erasure_coding.VolumeSlots(u.ecShardCount)
+}
+
+func (du *DiskUsages) getOrCreateDisk(diskType types.DiskType) *DiskUsageCounts {
+	du.Lock()
+	defer du.Unlock()
+	t, found := du.usages[diskType]
+	if found {
+		return t
+	}
+	t = &DiskUsageCounts{}
+	du.usages[diskType] = t
+	return t
+}
+
+func (d *Disk) String() string {
+	d.RLock()
+	defer d.RUnlock()
+	return fmt.Sprintf("Disk:%s, volumes:%v, ecShards:%v", d.NodeImpl.String(), d.volumes, d.ecShards)
+}
+
+func (d *Disk) AddOrUpdateVolume(v storage.VolumeInfo) (isNew, isChanged, tierTransition bool) {
+	d.Lock()
+	defer d.Unlock()
+	return d.doAddOrUpdateVolume(v, true)
+}
+
+// AddProvisionalVolume records a volume the master registered on its own --
+// volume growth -- before any server report has named it. Until one does, the
+// volume is protected from removal by a report that raced its creation.
+func (d *Disk) AddProvisionalVolume(v storage.VolumeInfo) (isNew, isChanged, tierTransition bool) {
+	d.Lock()
+	defer d.Unlock()
+	return d.doAddOrUpdateVolume(v, false)
+}
+
+// SetVolumeReadOnly records the read-only flag the server reported for vid,
+// keeping the report digest and the active volume count in step, and reports
+// whether the disk holds vid. It is not a volume report: a provisional volume
+// stays protected from a stale report until one names it.
+func (d *Disk) SetVolumeReadOnly(vid needle.VolumeId, readOnly bool) (found bool) {
+	d.Lock()
+	defer d.Unlock()
+	v, found := d.volumes[vid]
+	if !found || v.ReadOnly == readOnly {
+		return found
+	}
+	d.volumeDigest ^= v.ReportHash()
+	v.ReadOnly = readOnly
+	d.volumeDigest ^= v.ReportHash()
+	delta := &DiskUsageCounts{activeVolumeCount: 1}
+	if readOnly {
+		delta.activeVolumeCount = -1
+	}
+	d.UpAdjustDiskUsageDelta(types.ToDiskType(v.DiskType), delta)
+	return true
+}
+
+// doAddOrUpdateVolume returns three signals about how v was installed against
+// any existing record:
+//
+//   - isNew: no record was held before; the volume arrived.
+//   - isChanged: the ReadOnly flag flipped (the only field isChanged currently
+//     tracks). Other fields changing without ReadOnly flipping leaves this
+//     false.
+//   - tierTransition: v's IsRemote() classification differs from the previous
+//     record. The volume was already known and remains servable, but every
+//     connected client needs to refresh its replica priority -- a remote-tier
+//     replica restored locally must jump the read order, and one newly tiered
+//     to remote storage must give way. Callers that broadcast volume changes
+//     to clients must include tier-transitioned volumes alongside arrivals.
+func (d *Disk) doAddOrUpdateVolume(v storage.VolumeInfo, fromReport bool) (isNew, isChanged, tierTransition bool) {
+	deltaDiskUsage := &DiskUsageCounts{}
+	if oldV, ok := d.volumes[v.Id]; !ok {
+		stored := v
+		d.volumes[v.Id] = &stored
+		if !fromReport {
+			d.volumeAddedAt[v.Id] = time.Now()
+		}
+		d.volumeDigest ^= v.ReportHash()
+		d.volumeIdDigest ^= VolumeIdDigestHash(v.Id)
+		deltaDiskUsage.volumeCount = 1
+		if v.IsRemote() {
+			deltaDiskUsage.remoteVolumeCount = 1
+		}
+		if !v.ReadOnly {
+			deltaDiskUsage.activeVolumeCount = 1
+		}
+		d.UpAdjustMaxVolumeId(v.Id)
+		d.UpAdjustDiskUsageDelta(types.ToDiskType(v.DiskType), deltaDiskUsage)
+		isNew = true
+	} else {
+		if !fromReport && v.DiskId == 0 && oldV.DiskId != 0 {
+			// A provisional (grow-time) record carries no disk id -- the
+			// master cannot know which directory the server chose. Keep the
+			// one the server's report already named, before the digest below
+			// is computed, or the stored record would drift from what the
+			// server keeps reporting.
+			v.DiskId = oldV.DiskId
+		}
+		tierTransition = oldV.IsRemote() != v.IsRemote()
+		if tierTransition {
+			if v.IsRemote() {
+				deltaDiskUsage.remoteVolumeCount = 1
+			}
+			if oldV.IsRemote() {
+				deltaDiskUsage.remoteVolumeCount = -1
+			}
+			d.UpAdjustDiskUsageDelta(types.ToDiskType(v.DiskType), deltaDiskUsage)
+		}
+		d.volumeDigest ^= oldV.ReportHash() ^ v.ReportHash()
+		if fromReport {
+			delete(d.volumeAddedAt, v.Id)
+		}
+		isChanged = oldV.ReadOnly != v.ReadOnly || oldV.ReadOnlyCanDelete != v.ReadOnlyCanDelete
+		if oldV.ReadOnly != v.ReadOnly {
+			// Adjust active volume count when ReadOnly status changes
+			// Use a separate delta object to avoid affecting other metric adjustments
+			readOnlyDelta := &DiskUsageCounts{}
+			if v.ReadOnly {
+				// Changed from writable to read-only
+				readOnlyDelta.activeVolumeCount = -1
+			} else {
+				// Changed from read-only to writable
+				readOnlyDelta.activeVolumeCount = 1
+			}
+			d.UpAdjustDiskUsageDelta(types.ToDiskType(v.DiskType), readOnlyDelta)
+		}
+		// Written through the pointer the map already holds, and only after
+		// everything above has read the old value off it.
+		*oldV = v
+	}
+	return
+}
+
+func (d *Disk) GetVolumes() []storage.VolumeInfo {
+	return d.AppendVolumes(make([]storage.VolumeInfo, 0, d.VolumeCount()))
+}
+
+// AppendVolumeIds appends the ids of the disk's volumes to all, and repeats
+// the remote-tier ones on remote. Callers that only need to name volumes use
+// this rather than AppendVolumes, which copies a whole record per volume to
+// be read for four bytes of it.
+func (d *Disk) AppendVolumeIds(all, remote, readOnly, readOnlyCanDelete []uint32) ([]uint32, []uint32, []uint32, []uint32) {
+	d.RLock()
+	defer d.RUnlock()
+	for id, v := range d.volumes {
+		all = append(all, uint32(id))
+		if v.IsRemote() {
+			remote = append(remote, uint32(id))
+		}
+		if v.ReadOnly {
+			readOnly = append(readOnly, uint32(id))
+			if v.ReadOnlyCanDelete {
+				readOnlyCanDelete = append(readOnlyCanDelete, uint32(id))
+			}
+		}
+	}
+	return all, remote, readOnly, readOnlyCanDelete
+}
+
+// AppendVolumes appends the disk's volumes to dst, so a caller gathering
+// several disks fills one slice instead of concatenating a copy per disk.
+func (d *Disk) AppendVolumes(dst []storage.VolumeInfo) []storage.VolumeInfo {
+	d.RLock()
+	defer d.RUnlock()
+	for _, v := range d.volumes {
+		dst = append(dst, *v)
+	}
+	return dst
+}
+
+func (d *Disk) VolumeCount() int {
+	d.RLock()
+	defer d.RUnlock()
+	return len(d.volumes)
+}
+
+// RemoveVolumesNotIn drops the volumes the heartbeat did not name on this disk
+// and returns them, so a heartbeat can be diffed without copying the volume map
+// out. A volume named on another disk has moved, and counts as absent here.
+func (d *Disk) RemoveVolumesNotIn(reported *reportedVolumes) (removed []storage.VolumeInfo) {
+	diskTypeIndex := reported.diskTypeIndex(string(d.Id()))
+	d.Lock()
+	defer d.Unlock()
+	now := time.Now()
+	for vid, v := range d.volumes {
+		if reported.namedOn(vid, diskTypeIndex) {
+			// The server confirmed this volume; from here on its absence from
+			// a report is meaningful.
+			delete(d.volumeAddedAt, vid)
+			continue
+		}
+		// A volume the master registered itself and no report has confirmed
+		// yet is likely racing the list being applied, which was collected
+		// before the grow finished. Explicitly reported deletions still
+		// remove immediately through DeleteVolumeById.
+		if addedAt, unconfirmed := d.volumeAddedAt[vid]; unconfirmed && now.Sub(addedAt) < volumeRemovalGracePeriod {
+			continue
+		}
+		removed = append(removed, *v)
+		delete(d.volumes, vid)
+		delete(d.volumeAddedAt, vid)
+		d.volumeDigest ^= v.ReportHash()
+		d.volumeIdDigest ^= VolumeIdDigestHash(vid)
+	}
+	return removed
+}
+
+func (d *Disk) GetVolumesById(id needle.VolumeId) (storage.VolumeInfo, error) {
+	d.RLock()
+	defer d.RUnlock()
+	vInfo, ok := d.volumes[id]
+	if ok {
+		return *vInfo, nil
+	} else {
+		return storage.VolumeInfo{}, fmt.Errorf("volumeInfo not found")
+	}
+}
+
+func (d *Disk) DeleteVolumeById(id needle.VolumeId) {
+	d.Lock()
+	defer d.Unlock()
+	if v, ok := d.volumes[id]; ok {
+		d.volumeDigest ^= v.ReportHash()
+		d.volumeIdDigest ^= VolumeIdDigestHash(id)
+		delete(d.volumes, id)
+		delete(d.volumeAddedAt, id)
+	}
+}
+
+// VolumeDigest returns the disk's running volume digest.
+func (d *Disk) VolumeDigest() uint64 {
+	d.RLock()
+	defer d.RUnlock()
+	return d.volumeDigest
+}
+
+// VolumeIdDigest returns the digest of which volumes the disk holds.
+func (d *Disk) VolumeIdDigest() uint64 {
+	d.RLock()
+	defer d.RUnlock()
+	return d.volumeIdDigest
+}
+
+func (d *Disk) GetDataCenter() *DataCenter {
+	dn := d.Parent()
+	rack := dn.Parent()
+	dcNode := rack.Parent()
+	dcValue := dcNode.GetValue()
+	return dcValue.(*DataCenter)
+}
+
+func (d *Disk) GetRack() *Rack {
+	return d.Parent().Parent().(*NodeImpl).value.(*Rack)
+}
+
+func (d *Disk) GetTopology() *Topology {
+	p := d.Parent()
+	for p.Parent() != nil {
+		p = p.Parent()
+	}
+	t := p.(*Topology)
+	return t
+}
+
+func (d *Disk) ToMap() interface{} {
+	ret := make(map[string]interface{})
+	diskUsage := d.diskUsages.getOrCreateDisk(types.ToDiskType(string(d.Id())))
+	ret["Volumes"] = diskUsage.volumeCount
+	ret["VolumeIds"] = d.GetVolumeIds()
+	ret["EcShards"] = diskUsage.ecShardCount
+	ret["Max"] = diskUsage.maxVolumeCount
+	ret["Free"] = d.FreeSpace()
+	return ret
+}
+
+func (d *Disk) FreeSpace() int64 {
+	t := d.diskUsages.getOrCreateDisk(types.ToDiskType(string(d.Id())))
+	return t.FreeSpace()
+}
+
+func (d *Disk) ToDiskInfo(filter VolumeFilter) *master_pb.DiskInfo {
+	diskUsage := d.diskUsages.getOrCreateDisk(types.ToDiskType(string(d.Id()))).snapshot()
+
+	// Built under the read lock rather than from a copy as large as the
+	// messages it fed. Nothing here re-enters the topology, so the hold is safe.
+	d.RLock()
+	// Reserving room for every volume would keep what a filter set out not to
+	// build.
+	capacity := 0
+	if filter.SelectsEverything() {
+		capacity = len(d.volumes)
+	}
+	volumeInfos := make([]*master_pb.VolumeInformationMessage, 0, capacity)
+	var diskId uint32
+	var haveDiskId bool
+	for _, v := range d.volumes {
+		// Any volume names the disk, including one filtered out. The smallest
+		// rather than whichever the map yields first, so that two listings of
+		// an unchanged disk agree when it fronts several physical disks.
+		if !haveDiskId || v.DiskId < diskId {
+			diskId, haveDiskId = v.DiskId, true
+		}
+		if !filter.matches(v) {
+			continue
+		}
+		volumeInfos = append(volumeInfos, v.ToVolumeInformationMessage())
+	}
+	d.RUnlock()
+
+	ecShards := d.GetEcShards()
+	if !haveDiskId {
+		for _, ecv := range ecShards {
+			if !haveDiskId || ecv.DiskId < diskId {
+				diskId, haveDiskId = ecv.DiskId, true
+			}
+		}
+	}
+
+	m := &master_pb.DiskInfo{
+		Type:              string(d.Id()),
+		VolumeCount:       diskUsage.volumeCount,
+		MaxVolumeCount:    diskUsage.maxVolumeCount,
+		FreeVolumeCount:   diskUsage.maxVolumeCount - (diskUsage.volumeCount - diskUsage.remoteVolumeCount) - erasure_coding.VolumeSlots(diskUsage.ecShardCount),
+		ActiveVolumeCount: diskUsage.activeVolumeCount,
+		RemoteVolumeCount: diskUsage.remoteVolumeCount,
+		DiskId:            diskId,
+		DiskTotalBytes:    uint64(max(0, diskUsage.diskTotalBytes)),
+		DiskFreeBytes:     uint64(max(0, diskUsage.diskFreeBytes)),
+	}
+	m.VolumeInfos = volumeInfos
+	ecCapacity := 0
+	if filter.SelectsEverything() {
+		ecCapacity = len(ecShards)
+	}
+	m.EcShardInfos = make([]*master_pb.VolumeEcShardInformationMessage, 0, ecCapacity)
+	for _, ecv := range ecShards {
+		if !filter.matches(ecv) {
+			continue
+		}
+		m.EcShardInfos = append(m.EcShardInfos, ecv.ToVolumeEcShardInformationMessage())
+	}
+	return m
+}
+
+// GetVolumeIds returns the human readable volume ids limited to count of max 100.
+func (d *Disk) GetVolumeIds() string {
+	d.RLock()
+	defer d.RUnlock()
+	ids := make([]int, 0, len(d.volumes))
+
+	for k := range d.volumes {
+		ids = append(ids, int(k))
+	}
+
+	slices.Sort(ids)
+
+	return util.HumanReadableIntsMax(100, ids...)
+}

weed/topology/disk_ec.go

@@ -0,0 +1,106 @@
+package topology
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func (d *Disk) GetEcShards() (ret []*erasure_coding.EcVolumeInfo) {
+	d.ecShardsLock.RLock()
+	defer d.ecShardsLock.RUnlock()
+	// Total entries = sum of per-volume per-disk entries, which typically
+	// exceeds the number of unique volumes once shards span multiple disks.
+	total := 0
+	for _, byDisk := range d.ecShards {
+		total += len(byDisk)
+	}
+	ret = make([]*erasure_coding.EcVolumeInfo, 0, total)
+	for _, byDisk := range d.ecShards {
+		for _, ecVolumeInfo := range byDisk {
+			ret = append(ret, ecVolumeInfo)
+		}
+	}
+	return ret
+}
+
+func (d *Disk) AddOrUpdateEcShard(s *erasure_coding.EcVolumeInfo) {
+	d.ecShardsLock.Lock()
+	defer d.ecShardsLock.Unlock()
+
+	byDisk, ok := d.ecShards[s.VolumeId]
+	if !ok {
+		byDisk = make(map[types.DiskId]*erasure_coding.EcVolumeInfo, 1)
+		d.ecShards[s.VolumeId] = byDisk
+	}
+
+	diskId := types.DiskId(s.DiskId)
+	delta := 0
+	if existing, ok := byDisk[diskId]; !ok {
+		byDisk[diskId] = s
+		delta = s.ShardsInfo.Count()
+	} else {
+		oldCount := existing.ShardsInfo.Count()
+		existing.ShardsInfo.Add(s.ShardsInfo)
+		delta = existing.ShardsInfo.Count() - oldCount
+	}
+
+	if delta != 0 {
+		d.UpAdjustDiskUsageDelta(types.ToDiskType(string(d.Id())), &DiskUsageCounts{
+			ecShardCount: int64(delta),
+		})
+	}
+}
+
+func (d *Disk) DeleteEcShard(s *erasure_coding.EcVolumeInfo) {
+	d.ecShardsLock.Lock()
+	defer d.ecShardsLock.Unlock()
+
+	byDisk, ok := d.ecShards[s.VolumeId]
+	if !ok {
+		return
+	}
+	diskId := types.DiskId(s.DiskId)
+	existing, ok := byDisk[diskId]
+	if !ok {
+		return
+	}
+	oldCount := existing.ShardsInfo.Count()
+	existing.ShardsInfo.Subtract(s.ShardsInfo)
+	delta := existing.ShardsInfo.Count() - oldCount
+
+	if delta != 0 {
+		d.UpAdjustDiskUsageDelta(types.ToDiskType(string(d.Id())), &DiskUsageCounts{
+			ecShardCount: int64(delta),
+		})
+	}
+	if existing.ShardsInfo.Count() == 0 {
+		delete(byDisk, diskId)
+		if len(byDisk) == 0 {
+			delete(d.ecShards, s.VolumeId)
+		}
+	}
+}
+
+func (d *Disk) HasVolumesById(id needle.VolumeId) (hasVolumeId bool) {
+	// check whether normal volumes has this volume id
+	d.RLock()
+	_, ok := d.volumes[id]
+	if ok {
+		hasVolumeId = true
+	}
+	d.RUnlock()
+
+	if hasVolumeId {
+		return
+	}
+
+	// check whether ec shards has this volume id
+	d.ecShardsLock.RLock()
+	if byDisk, ok := d.ecShards[id]; ok && len(byDisk) > 0 {
+		hasVolumeId = true
+	}
+	d.ecShardsLock.RUnlock()
+
+	return
+}

weed/topology/disk_info_test.go

@@ -0,0 +1,69 @@
+package topology
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func TestDiskInfoReportsEveryVolume(t *testing.T) {
+	topo := NewTopology("diskinfo", nil, 32*1024*1024*1024, 5, false)
+	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
+
+	volumes := []*master_pb.VolumeInformationMessage{
+		{Id: 1, Collection: "c", Size: 1000, FileCount: 10, Version: 3, DiskId: 2},
+		{Id: 2, Collection: "c", Size: 2000, FileCount: 20, Version: 3, DiskId: 2},
+		{Id: 3, Collection: "other", Size: 3000, FileCount: 30, Version: 3, DiskId: 2},
+	}
+	topo.SyncDataNodeRegistration(volumes, dn)
+	topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{
+		{Id: 9, Collection: "c", EcIndexBits: 0x3fff, DiskId: 2},
+	}, dn)
+
+	var info *master_pb.DiskInfo
+	for _, c := range dn.Children() {
+		info = c.(*Disk).ToDiskInfo(VolumeFilter{})
+	}
+	if info == nil {
+		t.Fatal("the node reported no disk")
+	}
+	if len(info.VolumeInfos) != len(volumes) {
+		t.Fatalf("reported %d volumes, want %d", len(info.VolumeInfos), len(volumes))
+	}
+	if len(info.EcShardInfos) != 1 {
+		t.Fatalf("reported %d ec volumes, want 1", len(info.EcShardInfos))
+	}
+	if info.DiskId != 2 {
+		t.Errorf("reported disk id %d, want the one its volumes are on", info.DiskId)
+	}
+
+	bySize := map[uint32]uint64{}
+	for _, v := range info.VolumeInfos {
+		bySize[v.Id] = v.Size
+	}
+	for _, want := range volumes {
+		if got := bySize[want.Id]; got != want.Size {
+			t.Errorf("volume %d reported size %d, want %d", want.Id, got, want.Size)
+		}
+	}
+}
+
+func TestDiskInfoReportsTheDiskIdOfEcOnlyDisks(t *testing.T) {
+	topo := NewTopology("diskinfo", nil, 32*1024*1024*1024, 5, false)
+	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
+	topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{
+		{Id: 9, Collection: "c", EcIndexBits: 0x3fff, DiskId: 5},
+	}, dn)
+
+	for _, c := range dn.Children() {
+		info := c.(*Disk).ToDiskInfo(VolumeFilter{})
+		if len(info.VolumeInfos) != 0 {
+			t.Fatalf("expected no regular volumes, got %d", len(info.VolumeInfos))
+		}
+		if info.DiskId != 5 {
+			t.Errorf("reported disk id %d, want the one its shards are on", info.DiskId)
+		}
+	}
+}

weed/topology/disk_provisional_disk_id_test.go

@@ -0,0 +1,68 @@
+package topology
+
+import (
+	"sync/atomic"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// A provisional (grow-time) registration carries no disk id; when it lands
+// after the server's own report -- the server pushes its report during the
+// AllocateVolume RPC, so either order happens -- it must not erase the disk id
+// the report recorded.
+func TestProvisionalUpdateKeepsReportedDiskId(t *testing.T) {
+	disk := NewDisk(types.HardDriveType.String())
+
+	disk.AddOrUpdateVolume(storage.VolumeInfo{Id: 1, DiskId: 2})
+	disk.AddProvisionalVolume(storage.VolumeInfo{Id: 1})
+
+	v, err := disk.GetVolumesById(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v.DiskId != 2 {
+		t.Fatalf("DiskId = %d, want 2 (provisional update clobbered the reported value)", v.DiskId)
+	}
+
+	// A later server report naming a different disk still wins.
+	disk.AddOrUpdateVolume(storage.VolumeInfo{Id: 1, DiskId: 1})
+	if v, _ = disk.GetVolumesById(1); v.DiskId != 1 {
+		t.Fatalf("DiskId = %d, want 1 (a real report must override)", v.DiskId)
+	}
+}
+
+// A read-only mark from the server is not a volume report: it must not end the
+// grace period that keeps a just-grown volume safe from a full report collected
+// before the grow, while still keeping the digest and active count in step.
+func TestSetVolumeReadOnlyKeepsProvisionalProtection(t *testing.T) {
+	dn := NewDataNode("dn1")
+	vi := storage.VolumeInfo{Id: 1, DiskType: types.HardDriveType.String()}
+	dn.AddProvisionalVolume(vi)
+
+	dn.SetVolumeReadOnly(1, true)
+
+	readOnly := vi
+	readOnly.ReadOnly = true
+	if got, want := dn.VolumeDigest(), readOnly.ReportHash(); got != want {
+		t.Errorf("digest = %x, want %x, the hash of the volume as the server will report it", got, want)
+	}
+	if got := atomic.LoadInt64(&dn.GetDiskUsages().getOrCreateDisk(types.HardDriveType).activeVolumeCount); got != 0 {
+		t.Errorf("activeVolumeCount = %d, want 0 after the read-only mark", got)
+	}
+
+	// the stale report, collected before the grow, does not name the volume
+	if _, deleted, _ := dn.UpdateVolumes(nil); len(deleted) != 0 {
+		t.Fatalf("a report that raced the grow removed the volume: %v", deleted)
+	}
+	if v, err := dn.GetVolumesById(1); err != nil || !v.ReadOnly {
+		t.Fatalf("GetVolumesById = %+v, %v; want the volume kept, read-only", v, err)
+	}
+
+	// the server's own report confirms it, and from then on absence counts
+	dn.UpdateVolumes([]storage.VolumeInfo{readOnly})
+	if _, deleted, _ := dn.UpdateVolumes(nil); len(deleted) != 1 {
+		t.Fatalf("a report after confirmation should remove the volume, got %v", deleted)
+	}
+}

weed/topology/maintenance_mode_test.go

@@ -0,0 +1,331 @@
+package topology
+
+import (
+	"slices"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// addMaintenanceTestNode links a data node with maxVol free slots under rack.
+func addMaintenanceTestNode(rack *Rack, id string, maxVol int64) *DataNode {
+	dn := NewDataNode(id)
+	dn.Ip = id
+	rack.LinkChildNode(dn)
+	dn.getOrCreateDisk("").UpAdjustDiskUsageDelta("", &DiskUsageCounts{maxVolumeCount: maxVol})
+	return dn
+}
+
+func newMaintenanceTestTopology() (*Topology, *DataCenter) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	dc := NewDataCenter("dc1")
+	topo.LinkChildNode(dc)
+	return topo, dc
+}
+
+func TestMaintenanceModeReportsNoFreeSlots(t *testing.T) {
+	rack := NewRack("rack1")
+	dn := addMaintenanceTestNode(rack, "server1", 10)
+	option := &VolumeGrowOption{DiskType: types.HardDriveType}
+
+	if got := dn.AvailableSpaceFor(option); got != 10 {
+		t.Fatalf("AvailableSpaceFor = %d, want 10", got)
+	}
+	if !dn.SetMaintenanceMode(true) {
+		t.Fatal("first switch should report a change")
+	}
+	if dn.SetMaintenanceMode(true) {
+		t.Fatal("repeating the same value should not report a change")
+	}
+	if !dn.InMaintenanceMode() {
+		t.Fatal("InMaintenanceMode = false after switching on")
+	}
+	if got := dn.AvailableSpaceFor(option); got != 0 {
+		t.Errorf("AvailableSpaceFor in maintenance = %d, want 0: no new volumes go to a server in maintenance", got)
+	}
+	if got := dn.AvailableSpaceForReservation(option); got != 0 {
+		t.Errorf("AvailableSpaceForReservation in maintenance = %d, want 0", got)
+	}
+	if _, ok := dn.TryReserveCapacity(types.HardDriveType, 1); ok {
+		t.Error("reserved capacity on a server in maintenance")
+	}
+	if got := rack.AvailableSpaceFor(option); got != 10 {
+		t.Errorf("rack AvailableSpaceFor = %d, want 10: the node's capacity still rolls up to its parents", got)
+	}
+
+	if !dn.SetMaintenanceMode(false) {
+		t.Fatal("switching off should report a change")
+	}
+	if got := dn.AvailableSpaceFor(option); got != 10 {
+		t.Errorf("AvailableSpaceFor after maintenance = %d, want 10", got)
+	}
+}
+
+// A server in maintenance mode is the one being evacuated, so volume growth in
+// its rack must land every replica on its siblings even when it has the most
+// free slots by far.
+func TestVolumeGrowthSkipsMaintenanceNodeInSameRack(t *testing.T) {
+	topo, dc := newMaintenanceTestTopology()
+	rack := NewRack("rack1")
+	dc.LinkChildNode(rack)
+	busy := addMaintenanceTestNode(rack, "busy", 1000)
+	busy.SetMaintenanceMode(true)
+	addMaintenanceTestNode(rack, "small1", 2)
+	addMaintenanceTestNode(rack, "small2", 2)
+
+	vg := NewDefaultVolumeGrowth()
+	rp, _ := super_block.NewReplicaPlacementFromString("001")
+	option := &VolumeGrowOption{ReplicaPlacement: rp, DiskType: types.HardDriveType}
+	for i := 0; i < 50; i++ {
+		servers, _, err := vg.findEmptySlotsForOneVolume(topo, option, false)
+		if err != nil {
+			t.Fatalf("iteration %d: %v", i, err)
+		}
+		if len(servers) != 2 {
+			t.Fatalf("iteration %d: got %d servers, want 2", i, len(servers))
+		}
+		for _, s := range servers {
+			if s.Id() == busy.Id() {
+				t.Fatalf("iteration %d: picked the server in maintenance mode", i)
+			}
+		}
+	}
+
+	// Pinning the request to the maintenance node must fail rather than
+	// create a volume there.
+	rp0, _ := super_block.NewReplicaPlacementFromString("000")
+	pinned := &VolumeGrowOption{ReplicaPlacement: rp0, DiskType: types.HardDriveType, DataNode: "busy"}
+	if _, _, err := vg.findEmptySlotsForOneVolume(topo, pinned, false); err == nil {
+		t.Fatal("a volume pinned to a server in maintenance mode must not be created")
+	}
+}
+
+// The replica on another rack is chosen by walking a random offset through the
+// rack's rolled-up free slots, which still count a node in maintenance. The walk
+// has to land on an eligible sibling every time, not fall off the end.
+func TestVolumeGrowthSkipsMaintenanceNodeInOtherRack(t *testing.T) {
+	topo, dc := newMaintenanceTestTopology()
+	rack1 := NewRack("rack1")
+	dc.LinkChildNode(rack1)
+	addMaintenanceTestNode(rack1, "r1n1", 100)
+	rack2 := NewRack("rack2")
+	dc.LinkChildNode(rack2)
+	busy := addMaintenanceTestNode(rack2, "r2busy", 1000)
+	busy.SetMaintenanceMode(true)
+	ok := addMaintenanceTestNode(rack2, "r2ok", 1)
+
+	vg := NewDefaultVolumeGrowth()
+	rp, _ := super_block.NewReplicaPlacementFromString("010")
+	option := &VolumeGrowOption{ReplicaPlacement: rp, DiskType: types.HardDriveType}
+	for i := 0; i < 50; i++ {
+		servers, _, err := vg.findEmptySlotsForOneVolume(topo, option, false)
+		if err != nil {
+			t.Fatalf("iteration %d: %v", i, err)
+		}
+		if len(servers) != 2 {
+			t.Fatalf("iteration %d: got %d servers, want 2", i, len(servers))
+		}
+		onSibling := 0
+		for _, s := range servers {
+			if s.Id() == busy.Id() {
+				t.Fatalf("iteration %d: picked the server in maintenance mode", i)
+			}
+			if s.Id() == ok.Id() {
+				onSibling++
+			}
+		}
+		if onSibling != 1 {
+			t.Fatalf("iteration %d: the other-rack replica must land on the eligible sibling, got %v", i, servers)
+		}
+	}
+}
+
+// A replica on a server in maintenance mode takes no writes, so its volume
+// leaves the writable list while the mode is on and returns once it is off.
+func TestMaintenanceModeTogglesVolumeWritability(t *testing.T) {
+	topo, dc := newMaintenanceTestTopology()
+	rack := NewRack("rack1")
+	dc.LinkChildNode(rack)
+	dn1 := addMaintenanceTestNode(rack, "dn1", 10)
+	dn2 := addMaintenanceTestNode(rack, "dn2", 10)
+
+	rp, _ := super_block.NewReplicaPlacementFromString("001")
+	shared := storage.VolumeInfo{Id: 1, Size: 100, ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, Version: needle.GetCurrentVersion()}
+	dn1.AddOrUpdateVolume(shared)
+	dn2.AddOrUpdateVolume(shared)
+	topo.RegisterVolumeLayout(shared, dn1)
+	topo.RegisterVolumeLayout(shared, dn2)
+
+	rp0, _ := super_block.NewReplicaPlacementFromString("000")
+	alone := storage.VolumeInfo{Id: 2, Size: 100, ReplicaPlacement: rp0, Ttl: needle.EMPTY_TTL, Version: needle.GetCurrentVersion()}
+	dn2.AddOrUpdateVolume(alone)
+	topo.RegisterVolumeLayout(alone, dn2)
+
+	vlShared := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
+	vlAlone := topo.GetVolumeLayout("", rp0, needle.EMPTY_TTL, types.HardDriveType)
+	expectWritables := func(step string, vl *VolumeLayout, want ...needle.VolumeId) {
+		t.Helper()
+		got := vl.CloneWritableVolumes()
+		slices.Sort(got)
+		if !slices.Equal(got, want) {
+			t.Errorf("%s: writables = %v, want %v", step, got, want)
+		}
+	}
+	expectWritables("initial shared", vlShared, 1)
+	expectWritables("initial alone", vlAlone, 2)
+
+	if !topo.SetDataNodeMaintenanceMode(dn1, true) {
+		t.Fatal("switching dn1 on should report a change")
+	}
+	expectWritables("dn1 in maintenance: a volume with a replica there takes no writes", vlShared)
+	expectWritables("dn1 in maintenance: volumes elsewhere are unaffected", vlAlone, 2)
+	if locations := vlShared.Lookup(1); len(locations) != 2 {
+		t.Errorf("reads must still see every replica, got %v", locations)
+	}
+
+	if topo.SetDataNodeMaintenanceMode(dn1, true) {
+		t.Error("repeating the same state should not report a change")
+	}
+
+	if !topo.SetDataNodeMaintenanceMode(dn1, false) {
+		t.Fatal("switching dn1 off should report a change")
+	}
+	expectWritables("dn1 back: the volume is writable again", vlShared, 1)
+
+	topo.SetDataNodeMaintenanceMode(dn2, true)
+	expectWritables("dn2 in maintenance: shared", vlShared)
+	expectWritables("dn2 in maintenance: alone", vlAlone)
+}
+
+// A vacuum commit or a mark-writable that lands after the server entered
+// maintenance mode must not put the volume back on the writable list. Both
+// used to check only the replica count, and since heartbeats carry only changed
+// volumes, nothing would have taken it off again.
+func TestMaintenanceModeHoldsThroughVacuumAndMarkWritable(t *testing.T) {
+	topo, dc := newMaintenanceTestTopology()
+	rack := NewRack("rack1")
+	dc.LinkChildNode(rack)
+	dn1 := addMaintenanceTestNode(rack, "dn1", 10)
+	dn2 := addMaintenanceTestNode(rack, "dn2", 10)
+
+	rp, _ := super_block.NewReplicaPlacementFromString("001")
+	vi := storage.VolumeInfo{Id: 1, Size: 100, ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, Version: needle.GetCurrentVersion()}
+	dn1.AddOrUpdateVolume(vi)
+	dn2.AddOrUpdateVolume(vi)
+	topo.RegisterVolumeLayout(vi, dn1)
+	topo.RegisterVolumeLayout(vi, dn2)
+	vl := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
+	expectWritableCount := func(step string, want int) {
+		t.Helper()
+		if got, _ := vl.GetWritableVolumeCount(); got != want {
+			t.Errorf("%s: writable count = %d, want %d", step, got, want)
+		}
+	}
+
+	// vacuum: taken off the writable list before compaction, dn1 enters
+	// maintenance meanwhile, then every replica commits
+	vl.DrainAndRemoveFromWritable(1)
+	topo.SetDataNodeMaintenanceMode(dn1, true)
+	for _, dn := range []*DataNode{dn1, dn2} {
+		if vl.SetVolumeAvailable(dn, 1, false, false) {
+			t.Errorf("vacuum commit on %s made the volume writable with a replica in maintenance", dn.Id())
+		}
+	}
+	expectWritableCount("after vacuum commit", 0)
+	topo.SetDataNodeMaintenanceMode(dn1, false)
+	expectWritableCount("after maintenance", 1)
+
+	// mark-writable from a vacuum worker, after dn1 entered maintenance
+	vl.SetVolumeReadOnly(dn2, 1)
+	topo.SetDataNodeMaintenanceMode(dn1, true)
+	if vl.SetVolumeWritable(dn2, 1) {
+		t.Error("mark-writable made the volume writable with a replica in maintenance")
+	}
+	expectWritableCount("after mark-writable", 0)
+	topo.SetDataNodeMaintenanceMode(dn1, false)
+	expectWritableCount("after maintenance", 1)
+}
+
+// A volume server notifies the master the moment it flips a volume between
+// read-only and writable, ahead of the heartbeat that repeats the flag. The
+// master must act on the notification, not on its heartbeat copy: mark-writable
+// takes effect at once, and a maintenance toggle landing between a mark-readonly
+// and its heartbeat must not put the volume back.
+func TestMarkReadOnlyAndWritableAheadOfHeartbeat(t *testing.T) {
+	topo, dc := newMaintenanceTestTopology()
+	rack := NewRack("rack1")
+	dc.LinkChildNode(rack)
+	dn1 := addMaintenanceTestNode(rack, "dn1", 10)
+	dn2 := addMaintenanceTestNode(rack, "dn2", 10)
+
+	rp, _ := super_block.NewReplicaPlacementFromString("001")
+	vi := storage.VolumeInfo{Id: 1, Size: 100, ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, Version: needle.GetCurrentVersion()}
+	dn1.AddOrUpdateVolume(vi)
+	dn2.AddOrUpdateVolume(vi)
+	topo.RegisterVolumeLayout(vi, dn1)
+	topo.RegisterVolumeLayout(vi, dn2)
+	vl := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
+	expectWritableCount := func(step string, want int) {
+		t.Helper()
+		if got, _ := vl.GetWritableVolumeCount(); got != want {
+			t.Errorf("%s: writable count = %d, want %d", step, got, want)
+		}
+	}
+
+	// the heartbeat has reported dn2's replica read-only
+	readOnly := vi
+	readOnly.ReadOnly = true
+	dn2.AddOrUpdateVolume(readOnly)
+	vl.EnsureCorrectWritables(&readOnly)
+	expectWritableCount("heartbeat says read-only", 0)
+
+	// dn2 marks it writable and notifies, before the next heartbeat
+	if !vl.SetVolumeWritable(dn2, 1) {
+		t.Error("mark-writable from the replica's own server should take effect at once")
+	}
+	expectWritableCount("after mark-writable", 1)
+	if v, err := dn2.GetVolumesById(1); err != nil || v.ReadOnly {
+		t.Errorf("the node's record should say writable, got %+v, %v", v, err)
+	}
+
+	// dn2 marks it read-only and notifies; dn1 leaves maintenance before the
+	// heartbeat repeats the flag
+	vl.SetVolumeReadOnly(dn2, 1)
+	expectWritableCount("after mark-readonly", 0)
+	topo.SetDataNodeMaintenanceMode(dn1, true)
+	topo.SetDataNodeMaintenanceMode(dn1, false)
+	expectWritableCount("re-evaluated before the heartbeat", 0)
+}
+
+// SetDataNodeMaintenanceMode walks a snapshot of the node's volumes, so a
+// concurrent disconnect can have removed one from its layout by the time it is
+// re-evaluated. That must be a no-op, not a crash or a resurrected entry.
+func TestMaintenanceModeAfterVolumeLeftLayout(t *testing.T) {
+	topo, dc := newMaintenanceTestTopology()
+	rack := NewRack("rack1")
+	dc.LinkChildNode(rack)
+	dn := addMaintenanceTestNode(rack, "dn1", 10)
+
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vi := storage.VolumeInfo{Id: 1, Size: 100, ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL, Version: needle.GetCurrentVersion()}
+	dn.AddOrUpdateVolume(vi)
+	topo.RegisterVolumeLayout(vi, dn)
+	vl := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
+	vl.UnRegisterVolume(&vi, dn)
+
+	for _, on := range []bool{true, false} {
+		if !topo.SetDataNodeMaintenanceMode(dn, on) {
+			t.Fatalf("SetDataNodeMaintenanceMode(%v) should report a change", on)
+		}
+		if got, _ := vl.GetWritableVolumeCount(); got != 0 {
+			t.Errorf("maintenance %v: writable count = %d, want 0", on, got)
+		}
+		if locations := vl.Lookup(1); locations != nil {
+			t.Errorf("maintenance %v: re-evaluating a departed volume re-created its location list: %v", on, locations)
+		}
+	}
+}

weed/topology/node.go

@@ -0,0 +1,602 @@
+package topology
+
+import (
+	"errors"
+	"fmt"
+	"math/rand/v2"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+type NodeId string
+
+// CapacityReservation represents a temporary reservation of capacity
+type CapacityReservation struct {
+	reservationId string
+	diskType      types.DiskType
+	count         int64
+	createdAt     time.Time
+}
+
+// CapacityReservations manages capacity reservations for a node
+type CapacityReservations struct {
+	sync.RWMutex
+	reservations   map[string]*CapacityReservation
+	reservedCounts map[types.DiskType]int64
+}
+
+func newCapacityReservations() *CapacityReservations {
+	return &CapacityReservations{
+		reservations:   make(map[string]*CapacityReservation),
+		reservedCounts: make(map[types.DiskType]int64),
+	}
+}
+
+func (cr *CapacityReservations) removeReservation(reservationId string) bool {
+	cr.Lock()
+	defer cr.Unlock()
+
+	if reservation, exists := cr.reservations[reservationId]; exists {
+		delete(cr.reservations, reservationId)
+		cr.decrementCount(reservation.diskType, reservation.count)
+		return true
+	}
+	return false
+}
+
+func (cr *CapacityReservations) getReservedCount(diskType types.DiskType) int64 {
+	cr.RLock()
+	defer cr.RUnlock()
+
+	return cr.reservedCounts[diskType]
+}
+
+// decrementCount is a helper to decrement reserved count and clean up zero entries
+func (cr *CapacityReservations) decrementCount(diskType types.DiskType, count int64) {
+	cr.reservedCounts[diskType] -= count
+	// Clean up zero counts to prevent map growth
+	if cr.reservedCounts[diskType] <= 0 {
+		delete(cr.reservedCounts, diskType)
+	}
+}
+
+// doAddReservation is a helper to add a reservation, assuming the lock is already held
+func (cr *CapacityReservations) doAddReservation(diskType types.DiskType, count int64) string {
+	now := time.Now()
+	reservationId := fmt.Sprintf("%s-%d-%d-%d", diskType, count, now.UnixNano(), rand.Int64())
+	cr.reservations[reservationId] = &CapacityReservation{
+		reservationId: reservationId,
+		diskType:      diskType,
+		count:         count,
+		createdAt:     now,
+	}
+	cr.reservedCounts[diskType] += count
+	return reservationId
+}
+
+// tryReserveAtomic atomically checks available space and reserves if possible
+func (cr *CapacityReservations) tryReserveAtomic(diskType types.DiskType, count int64, availableSpaceFunc func() int64) (reservationId string, success bool) {
+	cr.Lock()
+	defer cr.Unlock()
+
+	// Check available space under lock
+	currentReserved := cr.reservedCounts[diskType]
+	availableSpace := availableSpaceFunc() - currentReserved
+
+	if availableSpace >= count {
+		// Create and add reservation atomically
+		return cr.doAddReservation(diskType, count), true
+	}
+
+	return "", false
+}
+
+func (cr *CapacityReservations) cleanExpiredReservations(expirationDuration time.Duration) {
+	cr.Lock()
+	defer cr.Unlock()
+
+	now := time.Now()
+	for id, reservation := range cr.reservations {
+		if now.Sub(reservation.createdAt) > expirationDuration {
+			delete(cr.reservations, id)
+			cr.decrementCount(reservation.diskType, reservation.count)
+			glog.V(1).Infof("Cleaned up expired capacity reservation: %s", id)
+		}
+	}
+}
+
+type Node interface {
+	Id() NodeId
+	String() string
+	AvailableSpaceFor(option *VolumeGrowOption) int64
+	ReserveOneVolume(r int64, option *VolumeGrowOption) (*DataNode, error)
+	ReserveOneVolumeForReservation(r int64, option *VolumeGrowOption) (*DataNode, error)
+	UpAdjustDiskUsageDelta(diskType types.DiskType, diskUsage *DiskUsageCounts)
+	UpAdjustMaxVolumeId(vid needle.VolumeId)
+	GetDiskUsages() *DiskUsages
+
+	// Capacity reservation methods for avoiding race conditions
+	TryReserveCapacity(diskType types.DiskType, count int64) (reservationId string, success bool)
+	ReleaseReservedCapacity(reservationId string)
+	AvailableSpaceForReservation(option *VolumeGrowOption) int64
+
+	GetMaxVolumeId() needle.VolumeId
+	SetParent(Node)
+	LinkChildNode(node Node)
+	UnlinkChildNode(nodeId NodeId)
+	CollectDeadNodeAndFullVolumes(freshThreshHold int64, volumeSizeLimit uint64, growThreshold float64)
+
+	IsDataNode() bool
+	IsRack() bool
+	IsDataCenter() bool
+	IsLocked() bool
+	Children() []Node
+	Parent() Node
+
+	GetValue() interface{} //get reference to the topology,dc,rack,datanode
+}
+
+type NodeImpl struct {
+	diskUsages   *DiskUsages
+	id           NodeId
+	parent       Node
+	sync.RWMutex // lock children
+	children     map[NodeId]Node
+	// maxVolumeId uses atomic ops so UpAdjustMaxVolumeId (called from the
+	// volume server heartbeat path) and GetMaxVolumeId (called from the
+	// master's assign / warmup checks) can run concurrently without a
+	// data race on NodeImpl.
+	maxVolumeId atomic.Uint32
+
+	//for rack, data center, topology
+	nodeType string
+	value    interface{}
+
+	// capacity reservations to prevent race conditions during volume creation
+	capacityReservations *CapacityReservations
+}
+
+func (n *NodeImpl) GetDiskUsages() *DiskUsages {
+	return n.diskUsages
+}
+
+// nodeHost returns the host a node runs on, or "" for non-data-node tiers (data
+// centers, racks).
+func nodeHost(node Node) string {
+	if dn, ok := node.(*DataNode); ok {
+		return dn.Ip
+	}
+	return ""
+}
+
+// preferDistinctHosts reorders candidates so the first node of each not-yet-used
+// host comes first (preserving weighted order), then the same-host leftovers, so a
+// prefix covers the most distinct machines. usedHost seeds the set. No-op when
+// hosts are empty (non-data-node tiers).
+func preferDistinctHosts(usedHost string, candidates []Node) []Node {
+	used := map[string]bool{}
+	if usedHost != "" {
+		used[usedHost] = true
+	}
+	distinct := make([]Node, 0, len(candidates))
+	dup := make([]Node, 0, len(candidates))
+	for _, node := range candidates {
+		h := nodeHost(node)
+		if h != "" && !used[h] {
+			used[h] = true
+			distinct = append(distinct, node)
+		} else {
+			dup = append(dup, node)
+		}
+	}
+	return append(distinct, dup...)
+}
+
+// the first node must satisfy filterFirstNodeFn(), the rest nodes must have one free slot
+func (n *NodeImpl) PickNodesByWeight(numberOfNodes int, option *VolumeGrowOption, filterFirstNodeFn func(dn Node) error) (firstNode Node, restNodes []Node, err error) {
+	var totalWeights int64
+	var errs []string
+	n.RLock()
+	candidates := make([]Node, 0, len(n.children))
+	candidatesWeights := make([]int64, 0, len(n.children))
+	//pick nodes which has enough free volumes as candidates, and use free volumes number as node weight.
+	for _, node := range n.children {
+		if node.AvailableSpaceFor(option) <= 0 {
+			continue
+		}
+		totalWeights += node.AvailableSpaceFor(option)
+		candidates = append(candidates, node)
+		candidatesWeights = append(candidatesWeights, node.AvailableSpaceFor(option))
+	}
+	n.RUnlock()
+	if len(candidates) < numberOfNodes {
+		glog.V(0).Infoln(n.Id(), "failed to pick", numberOfNodes, "from ", len(candidates), "node candidates")
+		return nil, nil, errors.New("Not enough data nodes found!")
+	}
+
+	//pick nodes randomly by weights, the node picked earlier has higher final weights
+	sortedCandidates := make([]Node, 0, len(candidates))
+	for i := 0; i < len(candidates); i++ {
+		// Break if no more weights available to prevent panic in rand.Int64N
+		if totalWeights <= 0 {
+			break
+		}
+		weightsInterval := rand.Int64N(totalWeights)
+		lastWeights := int64(0)
+		for k, weights := range candidatesWeights {
+			if (weightsInterval >= lastWeights) && (weightsInterval < lastWeights+weights) {
+				sortedCandidates = append(sortedCandidates, candidates[k])
+				candidatesWeights[k] = 0
+				totalWeights -= weights
+				break
+			}
+			lastWeights += weights
+		}
+	}
+
+	restNodes = make([]Node, 0, numberOfNodes-1)
+	ret := false
+	n.RLock()
+	for k, node := range sortedCandidates {
+		if err := filterFirstNodeFn(node); err == nil {
+			firstNode = node
+			// Fill the rest preferring not-yet-used hosts, so replicas spread across
+			// machines; falls back to same-host when too few. No-op for dc/rack tiers
+			// (empty host), which keep the weighted order.
+			pool := make([]Node, 0, len(sortedCandidates)-1)
+			pool = append(pool, sortedCandidates[:k]...)
+			pool = append(pool, sortedCandidates[k+1:]...)
+			pool = preferDistinctHosts(nodeHost(firstNode), pool)
+			if len(pool) > numberOfNodes-1 {
+				pool = pool[:numberOfNodes-1]
+			}
+			restNodes = pool
+			ret = true
+			break
+		} else {
+			errs = append(errs, string(node.Id())+":"+err.Error())
+		}
+	}
+	n.RUnlock()
+	if !ret {
+		return nil, nil, errors.New("No matching data node found! \n" + strings.Join(errs, "\n"))
+	}
+	return
+}
+
+func (n *NodeImpl) IsDataNode() bool {
+	return n.nodeType == "DataNode"
+}
+
+func (n *NodeImpl) IsRack() bool {
+	return n.nodeType == "Rack"
+}
+
+func (n *NodeImpl) IsDataCenter() bool {
+	return n.nodeType == "DataCenter"
+}
+
+func (n *NodeImpl) IsLocked() (isTryLock bool) {
+	if isTryLock = n.TryRLock(); isTryLock {
+		n.RUnlock()
+	}
+	return !isTryLock
+}
+
+func (n *NodeImpl) String() string {
+	if n.parent != nil {
+		return n.parent.String() + ":" + string(n.id)
+	}
+	return string(n.id)
+}
+
+func (n *NodeImpl) Id() NodeId {
+	return n.id
+}
+
+func (n *NodeImpl) getOrCreateDisk(diskType types.DiskType) *DiskUsageCounts {
+	return n.diskUsages.getOrCreateDisk(diskType)
+}
+
+// inMaintenanceMode is true for a data node whose volume server is in
+// maintenance mode. Racks and data centers never are: their rolled-up counters
+// still include such a node, so their free-slot totals may exceed what their
+// children will actually hand out; see reserveOneVolumeInternal.
+func (n *NodeImpl) inMaintenanceMode() bool {
+	dn, ok := n.value.(*DataNode)
+	return ok && dn.InMaintenanceMode()
+}
+
+// AvailableSpaceFor is the free volume slots on this node for the option's disk
+// type. A data node in maintenance mode reports none: it is being drained, so
+// it is neither a volume-growth candidate nor reservable.
+func (n *NodeImpl) AvailableSpaceFor(option *VolumeGrowOption) int64 {
+	if n.inMaintenanceMode() {
+		return 0
+	}
+	t := n.getOrCreateDisk(option.DiskType)
+	freeVolumeSlotCount := atomic.LoadInt64(&t.maxVolumeCount) + atomic.LoadInt64(&t.remoteVolumeCount) - atomic.LoadInt64(&t.volumeCount)
+	freeVolumeSlotCount -= erasure_coding.VolumeSlots(atomic.LoadInt64(&t.ecShardCount))
+	return freeVolumeSlotCount
+}
+
+// CapacityFor is the total registered volume slots for the option's disk type;
+// zero means no volume server has reported capacity for it yet.
+func (n *NodeImpl) CapacityFor(option *VolumeGrowOption) int64 {
+	t := n.getOrCreateDisk(option.DiskType)
+	return atomic.LoadInt64(&t.maxVolumeCount) + atomic.LoadInt64(&t.remoteVolumeCount)
+}
+
+// CapacityForAnyDisk is the total registered volume slots across every disk
+// type. CapacityFor answers zero both while a cluster is still starting and
+// when it never serves the option's medium; this tells the two apart.
+func (n *NodeImpl) CapacityForAnyDisk() (total int64) {
+	n.diskUsages.RLock()
+	defer n.diskUsages.RUnlock()
+	for _, t := range n.diskUsages.usages {
+		total += atomic.LoadInt64(&t.maxVolumeCount) + atomic.LoadInt64(&t.remoteVolumeCount)
+	}
+	return
+}
+
+// AvailableSpaceForReservation returns available space considering existing reservations
+func (n *NodeImpl) AvailableSpaceForReservation(option *VolumeGrowOption) int64 {
+	baseAvailable := n.AvailableSpaceFor(option)
+	reservedCount := n.capacityReservations.getReservedCount(option.DiskType)
+	return baseAvailable - reservedCount
+}
+
+// TryReserveCapacity attempts to atomically reserve capacity for volume creation
+func (n *NodeImpl) TryReserveCapacity(diskType types.DiskType, count int64) (reservationId string, success bool) {
+	const reservationTimeout = 5 * time.Minute // TODO: make this configurable
+
+	// Clean up any expired reservations first
+	n.capacityReservations.cleanExpiredReservations(reservationTimeout)
+
+	// Atomically check and reserve space
+	option := &VolumeGrowOption{DiskType: diskType}
+	reservationId, success = n.capacityReservations.tryReserveAtomic(diskType, count, func() int64 {
+		return n.AvailableSpaceFor(option)
+	})
+
+	if success {
+		glog.V(1).Infof("Reserved %d capacity for diskType %s on node %s: %s", count, diskType, n.Id(), reservationId)
+	}
+
+	return reservationId, success
+}
+
+// ReleaseReservedCapacity releases a previously reserved capacity
+func (n *NodeImpl) ReleaseReservedCapacity(reservationId string) {
+	if n.capacityReservations.removeReservation(reservationId) {
+		glog.V(1).Infof("Released capacity reservation on node %s: %s", n.Id(), reservationId)
+	} else {
+		glog.V(1).Infof("Attempted to release non-existent reservation on node %s: %s", n.Id(), reservationId)
+	}
+}
+func (n *NodeImpl) SetParent(node Node) {
+	n.parent = node
+}
+
+func (n *NodeImpl) Children() (ret []Node) {
+	n.RLock()
+	defer n.RUnlock()
+	for _, c := range n.children {
+		ret = append(ret, c)
+	}
+	return ret
+}
+
+func (n *NodeImpl) Parent() Node {
+	return n.parent
+}
+
+func (n *NodeImpl) GetValue() interface{} {
+	return n.value
+}
+
+func (n *NodeImpl) ReserveOneVolume(r int64, option *VolumeGrowOption) (assignedNode *DataNode, err error) {
+	return n.reserveOneVolumeInternal(r, option, false)
+}
+
+// ReserveOneVolumeForReservation selects a node using reservation-aware capacity checks
+func (n *NodeImpl) ReserveOneVolumeForReservation(r int64, option *VolumeGrowOption) (assignedNode *DataNode, err error) {
+	return n.reserveOneVolumeInternal(r, option, true)
+}
+
+func (n *NodeImpl) reserveOneVolumeInternal(r int64, option *VolumeGrowOption, useReservations bool) (assignedNode *DataNode, err error) {
+	n.RLock()
+	defer n.RUnlock()
+	freeSpaceOf := func(node Node) int64 {
+		if useReservations {
+			return node.AvailableSpaceForReservation(option)
+		}
+		return node.AvailableSpaceFor(option)
+	}
+	// The caller draws r from this node's rolled-up free slots, which still
+	// count children the walk below skips: a data node in maintenance mode
+	// reports no space, and an over-committed one reports less than zero. Fold
+	// r into the space that is actually on offer so it cannot walk off the end
+	// and fail with slots still free.
+	var eligible int64
+	for _, node := range n.children {
+		if freeSpace := freeSpaceOf(node); freeSpace > 0 {
+			eligible += freeSpace
+		}
+	}
+	if eligible <= 0 {
+		return nil, errors.New("No free volume slot found!")
+	}
+	r %= eligible
+	for _, node := range n.children {
+		freeSpace := freeSpaceOf(node)
+		// fmt.Println("r =", r, ", node =", node, ", freeSpace =", freeSpace)
+		if freeSpace <= 0 {
+			continue
+		}
+		if r >= freeSpace {
+			r -= freeSpace
+		} else {
+			var hasSpace bool
+			if useReservations {
+				hasSpace = node.IsDataNode() && node.AvailableSpaceForReservation(option) > 0
+			} else {
+				hasSpace = node.IsDataNode() && node.AvailableSpaceFor(option) > 0
+			}
+			if hasSpace {
+				// fmt.Println("vid =", vid, " assigned to node =", node, ", freeSpace =", node.FreeSpace())
+				dn := node.(*DataNode)
+				if dn.IsTerminating {
+					continue
+				}
+				return dn, nil
+			}
+			if useReservations {
+				assignedNode, err = node.ReserveOneVolumeForReservation(r, option)
+			} else {
+				assignedNode, err = node.ReserveOneVolume(r, option)
+			}
+			if err == nil {
+				return
+			}
+		}
+	}
+	return nil, errors.New("No free volume slot found!")
+}
+
+func (n *NodeImpl) UpAdjustDiskUsageDelta(diskType types.DiskType, diskUsage *DiskUsageCounts) { //can be negative
+	existingDisk := n.getOrCreateDisk(diskType)
+	existingDisk.addDiskUsageCounts(diskUsage)
+	if n.parent != nil {
+		n.parent.UpAdjustDiskUsageDelta(diskType, diskUsage)
+	}
+}
+func (n *NodeImpl) UpAdjustMaxVolumeId(vid needle.VolumeId) {
+	target := uint32(vid)
+	for {
+		current := n.maxVolumeId.Load()
+		if current >= target {
+			return
+		}
+		if n.maxVolumeId.CompareAndSwap(current, target) {
+			break
+		}
+	}
+	if n.parent != nil {
+		n.parent.UpAdjustMaxVolumeId(vid)
+	}
+}
+func (n *NodeImpl) GetMaxVolumeId() needle.VolumeId {
+	return needle.VolumeId(n.maxVolumeId.Load())
+}
+
+func (n *NodeImpl) LinkChildNode(node Node) {
+	n.Lock()
+	defer n.Unlock()
+	n.doLinkChildNode(node)
+}
+
+func (n *NodeImpl) doLinkChildNode(node Node) {
+	if n.children[node.Id()] == nil {
+		n.children[node.Id()] = node
+		for dt, du := range node.GetDiskUsages().usages {
+			n.UpAdjustDiskUsageDelta(dt, du)
+		}
+		n.UpAdjustMaxVolumeId(node.GetMaxVolumeId())
+		node.SetParent(n)
+		// Maintain the topology's address index so Ping admission and other
+		// callers can resolve a data node from its address in O(1).
+		if dn, ok := node.GetValue().(*DataNode); ok {
+			if topo := n.GetTopology(); topo != nil {
+				topo.registerDataNodeAddress(dn)
+			}
+		}
+		glog.V(0).Infoln(n, "adds child", node.Id())
+	}
+}
+
+func (n *NodeImpl) UnlinkChildNode(nodeId NodeId) {
+	n.Lock()
+	defer n.Unlock()
+	node := n.children[nodeId]
+	if node != nil {
+		// Drop the topology address index before clearing the parent pointer
+		// so GetTopology() can still walk up to the root.
+		if dn, ok := node.GetValue().(*DataNode); ok {
+			if topo := n.GetTopology(); topo != nil {
+				topo.unregisterDataNodeAddress(dn.ServerAddress(), dn)
+			}
+		}
+		node.SetParent(nil)
+		delete(n.children, node.Id())
+		for dt, du := range node.GetDiskUsages().negative().usages {
+			n.UpAdjustDiskUsageDelta(dt, du)
+		}
+		glog.V(0).Infoln(n, "removes", node.Id())
+	}
+}
+
+func (n *NodeImpl) CollectDeadNodeAndFullVolumes(freshThreshHoldUnixTime int64, volumeSizeLimit uint64, growThreshold float64) {
+	if n.IsRack() {
+		for _, c := range n.Children() {
+			dn := c.(*DataNode) //can not cast n to DataNode
+			for _, v := range dn.GetVolumes() {
+				topo := n.GetTopology()
+				diskType := types.ToDiskType(v.DiskType)
+				vl := topo.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
+
+				if v.Size >= volumeSizeLimit {
+					vl.accessLock.RLock()
+					vacuumTime, ok := vl.vacuumedVolumes[v.Id]
+					vl.accessLock.RUnlock()
+
+					// If a volume has been vacuumed in the past 20 seconds, we do not check whether it has reached full capacity.
+					// After 20s(grpc timeout), theoretically all the heartbeats of the volume server have reached the master,
+					// the volume size should be correct, not the size before the vacuum.
+					if !ok || time.Now().Add(-20*time.Second).After(vacuumTime) {
+						//fmt.Println("volume",v.Id,"size",v.Size,">",volumeSizeLimit)
+						topo.chanFullVolumes <- v
+					}
+				} else if !v.ReadOnly && float64(v.Size) > float64(volumeSizeLimit)*growThreshold {
+					// Crowding asks for more room to write into, which a
+					// read-only volume can never provide. Growth already
+					// discounts them by intersecting with the writable list, so
+					// marking one only costs the entry.
+					topo.chanCrowdedVolumes <- v
+				}
+				copyCount := v.ReplicaPlacement.GetCopyCount()
+				if copyCount > 1 {
+					if copyCount > len(topo.Lookup(v.Collection, v.Id)) {
+						stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1)
+					} else {
+						stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(0)
+					}
+				}
+			}
+		}
+	} else {
+		for _, c := range n.Children() {
+			c.CollectDeadNodeAndFullVolumes(freshThreshHoldUnixTime, volumeSizeLimit, growThreshold)
+		}
+	}
+}
+
+func (n *NodeImpl) GetTopology() *Topology {
+	var p Node = n
+	for p.Parent() != nil {
+		p = p.Parent()
+	}
+	// A detached subtree (no Topology root in scope) must not panic; the
+	// callers above check the returned value for nil and skip the
+	// address-index maintenance in that case.
+	topo, _ := p.GetValue().(*Topology)
+	return topo
+}

weed/topology/race_condition_stress_test.go

@@ -0,0 +1,350 @@
+package topology
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// TestRaceConditionStress simulates the original issue scenario:
+// High concurrent writes causing capacity misjudgment
+func TestRaceConditionStress(t *testing.T) {
+	// Create a cluster similar to the issue description:
+	// 3 volume servers, 200GB each, 5GB volume limit = 40 volumes max per server
+	const (
+		numServers          = 3
+		volumeLimitMB       = 5000                                      // 5GB in MB
+		storagePerServerGB  = 200                                       // 200GB per server
+		maxVolumesPerServer = storagePerServerGB * 1024 / volumeLimitMB // 200*1024/5000 = 40
+		concurrentRequests  = 50                                        // High concurrency like the issue
+	)
+
+	// Create test topology
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), uint64(volumeLimitMB)*1024*1024, 5, false)
+
+	dc := NewDataCenter("dc1")
+	topo.LinkChildNode(dc)
+	rack := NewRack("rack1")
+	dc.LinkChildNode(rack)
+
+	// Create 3 volume servers with realistic capacity
+	servers := make([]*DataNode, numServers)
+	for i := 0; i < numServers; i++ {
+		dn := NewDataNode(fmt.Sprintf("server%d", i+1))
+		rack.LinkChildNode(dn)
+
+		// Set up disk with capacity for 40 volumes
+		disk := NewDisk(types.HardDriveType.String())
+		disk.diskUsages.getOrCreateDisk(types.HardDriveType).maxVolumeCount = maxVolumesPerServer
+		dn.LinkChildNode(disk)
+
+		servers[i] = dn
+	}
+
+	vg := NewDefaultVolumeGrowth()
+	rp, _ := super_block.NewReplicaPlacementFromString("000") // Single replica like the issue
+
+	option := &VolumeGrowOption{
+		Collection:       "test-bucket-large", // Same collection name as issue
+		ReplicaPlacement: rp,
+		DiskType:         types.HardDriveType,
+	}
+
+	// Track results
+	var successfulAllocations int64
+	var failedAllocations int64
+	var totalVolumesCreated int64
+
+	var wg sync.WaitGroup
+
+	// Launch concurrent volume creation requests
+	startTime := time.Now()
+	for i := 0; i < concurrentRequests; i++ {
+		wg.Add(1)
+		go func(requestId int) {
+			defer wg.Done()
+
+			// This is the critical test: multiple threads trying to allocate simultaneously
+			servers, reservation, err := vg.findEmptySlotsForOneVolume(topo, option, true)
+
+			if err != nil {
+				atomic.AddInt64(&failedAllocations, 1)
+				t.Logf("Request %d failed: %v", requestId, err)
+				return
+			}
+
+			// Simulate volume creation delay (like in real scenario)
+			time.Sleep(time.Millisecond * 50)
+
+			// Simulate successful volume creation
+			for _, server := range servers {
+				disk := server.children[NodeId(types.HardDriveType.String())].(*Disk)
+				deltaDiskUsage := &DiskUsageCounts{
+					volumeCount: 1,
+				}
+				disk.UpAdjustDiskUsageDelta(types.HardDriveType, deltaDiskUsage)
+				atomic.AddInt64(&totalVolumesCreated, 1)
+			}
+
+			// Release reservations (simulates successful registration)
+			reservation.releaseAllReservations()
+			atomic.AddInt64(&successfulAllocations, 1)
+
+		}(i)
+	}
+
+	wg.Wait()
+	duration := time.Since(startTime)
+
+	// Verify results
+	t.Logf("Test completed in %v", duration)
+	t.Logf("Successful allocations: %d", successfulAllocations)
+	t.Logf("Failed allocations: %d", failedAllocations)
+	t.Logf("Total volumes created: %d", totalVolumesCreated)
+
+	// Check capacity limits are respected
+	totalCapacityUsed := int64(0)
+	for i, server := range servers {
+		disk := server.children[NodeId(types.HardDriveType.String())].(*Disk)
+		volumeCount := disk.diskUsages.getOrCreateDisk(types.HardDriveType).volumeCount
+		totalCapacityUsed += volumeCount
+
+		t.Logf("Server %d: %d volumes (max: %d)", i+1, volumeCount, maxVolumesPerServer)
+
+		// Critical test: No server should exceed its capacity
+		if volumeCount > maxVolumesPerServer {
+			t.Errorf("RACE CONDITION DETECTED: Server %d exceeded capacity: %d > %d",
+				i+1, volumeCount, maxVolumesPerServer)
+		}
+	}
+
+	// Verify totals make sense
+	if totalVolumesCreated != totalCapacityUsed {
+		t.Errorf("Volume count mismatch: created=%d, actual=%d", totalVolumesCreated, totalCapacityUsed)
+	}
+
+	// The total should never exceed the cluster capacity (120 volumes for 3 servers × 40 each)
+	maxClusterCapacity := int64(numServers * maxVolumesPerServer)
+	if totalCapacityUsed > maxClusterCapacity {
+		t.Errorf("RACE CONDITION DETECTED: Cluster capacity exceeded: %d > %d",
+			totalCapacityUsed, maxClusterCapacity)
+	}
+
+	// With reservations, we should have controlled allocation
+	// Total requests = successful + failed should equal concurrentRequests
+	if successfulAllocations+failedAllocations != concurrentRequests {
+		t.Errorf("Request count mismatch: success=%d + failed=%d != total=%d",
+			successfulAllocations, failedAllocations, concurrentRequests)
+	}
+
+	t.Logf("Race condition test passed: Capacity limits respected with %d concurrent requests",
+		concurrentRequests)
+}
+
+// TestCapacityJudgmentAccuracy verifies that the capacity calculation is accurate
+// under various load conditions
+func TestCapacityJudgmentAccuracy(t *testing.T) {
+	// Create a single server with known capacity
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 5*1024*1024*1024, 5, false)
+
+	dc := NewDataCenter("dc1")
+	topo.LinkChildNode(dc)
+	rack := NewRack("rack1")
+	dc.LinkChildNode(rack)
+
+	dn := NewDataNode("server1")
+	rack.LinkChildNode(dn)
+
+	// Server with capacity for exactly 10 volumes
+	disk := NewDisk(types.HardDriveType.String())
+	diskUsage := disk.diskUsages.getOrCreateDisk(types.HardDriveType)
+	diskUsage.maxVolumeCount = 10
+	dn.LinkChildNode(disk)
+
+	// Also set max volume count on the DataNode level (gets propagated up)
+	dn.diskUsages.getOrCreateDisk(types.HardDriveType).maxVolumeCount = 10
+
+	vg := NewDefaultVolumeGrowth()
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+
+	option := &VolumeGrowOption{
+		Collection:       "test",
+		ReplicaPlacement: rp,
+		DiskType:         types.HardDriveType,
+	}
+
+	// Test accurate capacity reporting at each step
+	for i := 0; i < 10; i++ {
+		// Check available space before reservation
+		availableBefore := dn.AvailableSpaceFor(option)
+		availableForReservation := dn.AvailableSpaceForReservation(option)
+
+		expectedAvailable := int64(10 - i)
+		if availableBefore != expectedAvailable {
+			t.Errorf("Step %d: Expected %d available, got %d", i, expectedAvailable, availableBefore)
+		}
+
+		if availableForReservation != expectedAvailable {
+			t.Errorf("Step %d: Expected %d available for reservation, got %d", i, expectedAvailable, availableForReservation)
+		}
+
+		// Try to reserve and allocate
+		_, reservation, err := vg.findEmptySlotsForOneVolume(topo, option, true)
+		if err != nil {
+			t.Fatalf("Step %d: Unexpected reservation failure: %v", i, err)
+		}
+
+		// Check that available space for reservation decreased
+		availableAfterReservation := dn.AvailableSpaceForReservation(option)
+		if availableAfterReservation != expectedAvailable-1 {
+			t.Errorf("Step %d: Expected %d available after reservation, got %d",
+				i, expectedAvailable-1, availableAfterReservation)
+		}
+
+		// Simulate successful volume creation by properly updating disk usage hierarchy
+		disk := dn.children[NodeId(types.HardDriveType.String())].(*Disk)
+
+		// Create a volume usage delta to simulate volume creation
+		deltaDiskUsage := &DiskUsageCounts{
+			volumeCount: 1,
+		}
+
+		// Properly propagate the usage up the hierarchy
+		disk.UpAdjustDiskUsageDelta(types.HardDriveType, deltaDiskUsage)
+
+		// Debug: Check the volume count after update
+		diskUsageOnNode := dn.diskUsages.getOrCreateDisk(types.HardDriveType)
+		currentVolumeCount := atomic.LoadInt64(&diskUsageOnNode.volumeCount)
+		t.Logf("Step %d: Volume count after update: %d", i, currentVolumeCount)
+
+		// Release reservation
+		reservation.releaseAllReservations()
+
+		// Verify final state
+		availableAfter := dn.AvailableSpaceFor(option)
+		expectedAfter := int64(10 - i - 1)
+		if availableAfter != expectedAfter {
+			t.Errorf("Step %d: Expected %d available after creation, got %d",
+				i, expectedAfter, availableAfter)
+			// More debugging
+			diskUsageOnNode := dn.diskUsages.getOrCreateDisk(types.HardDriveType)
+			maxVolumes := atomic.LoadInt64(&diskUsageOnNode.maxVolumeCount)
+			remoteVolumes := atomic.LoadInt64(&diskUsageOnNode.remoteVolumeCount)
+			actualVolumeCount := atomic.LoadInt64(&diskUsageOnNode.volumeCount)
+			t.Logf("Debug Step %d: max=%d, volume=%d, remote=%d", i, maxVolumes, actualVolumeCount, remoteVolumes)
+		}
+	}
+
+	// At this point, no more reservations should succeed
+	_, _, err := vg.findEmptySlotsForOneVolume(topo, option, true)
+	if err == nil {
+		t.Error("Expected reservation to fail when at capacity")
+	}
+
+	t.Logf("Capacity judgment accuracy test passed")
+}
+
+// TestReservationSystemPerformance measures the performance impact of reservations
+func TestReservationSystemPerformance(t *testing.T) {
+	// Create topology
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	dc := NewDataCenter("dc1")
+	topo.LinkChildNode(dc)
+	rack := NewRack("rack1")
+	dc.LinkChildNode(rack)
+
+	dn := NewDataNode("server1")
+	rack.LinkChildNode(dn)
+
+	disk := NewDisk(types.HardDriveType.String())
+	disk.diskUsages.getOrCreateDisk(types.HardDriveType).maxVolumeCount = 1000
+	dn.LinkChildNode(disk)
+
+	vg := NewDefaultVolumeGrowth()
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+
+	option := &VolumeGrowOption{
+		Collection:       "test",
+		ReplicaPlacement: rp,
+		DiskType:         types.HardDriveType,
+	}
+
+	// Benchmark reservation operations
+	const iterations = 1000
+
+	startTime := time.Now()
+	for i := 0; i < iterations; i++ {
+		_, reservation, err := vg.findEmptySlotsForOneVolume(topo, option, true)
+		if err != nil {
+			t.Fatalf("Iteration %d failed: %v", i, err)
+		}
+		reservation.releaseAllReservations()
+
+		// Simulate volume creation
+		diskUsage := dn.diskUsages.getOrCreateDisk(types.HardDriveType)
+		atomic.AddInt64(&diskUsage.volumeCount, 1)
+	}
+	duration := time.Since(startTime)
+
+	avgDuration := duration / iterations
+	t.Logf("Performance: %d reservations in %v (avg: %v per reservation)",
+		iterations, duration, avgDuration)
+
+	// Performance should be reasonable (less than 1ms per reservation on average)
+	if avgDuration > time.Millisecond {
+		t.Errorf("Reservation system performance concern: %v per reservation", avgDuration)
+	} else {
+		t.Logf("Performance test passed: %v per reservation", avgDuration)
+	}
+}
+
+func TestDisk_GetEcShards_Race(t *testing.T) {
+	d := NewDisk("hdd")
+
+	// Pre-populate with one shard
+	initialShard := &erasure_coding.EcVolumeInfo{
+		VolumeId:   needle.VolumeId(1),
+		ShardsInfo: erasure_coding.NewShardsInfo(),
+	}
+	initialShard.ShardsInfo.Set(erasure_coding.ShardInfo{Id: 0, Size: 100})
+	d.AddOrUpdateEcShard(initialShard)
+
+	var wg sync.WaitGroup
+	wg.Add(10)
+
+	// Goroutine 1-5: Continuously read shards
+	for j := 0; j < 5; j++ {
+		go func() {
+			defer wg.Done()
+			for i := 0; i < 10000; i++ {
+				d.GetEcShards()
+			}
+		}()
+	}
+
+	// Goroutine 6-10: Continuously update shards
+	for j := 0; j < 5; j++ {
+		go func() {
+			defer wg.Done()
+			for i := 0; i < 10000; i++ {
+				shard := &erasure_coding.EcVolumeInfo{
+					VolumeId:   needle.VolumeId(i % 100),
+					ShardsInfo: erasure_coding.NewShardsInfo(),
+				}
+				shard.ShardsInfo.Set(erasure_coding.ShardInfo{Id: erasure_coding.ShardId(i % 14), Size: 100})
+				d.AddOrUpdateEcShard(shard)
+			}
+		}()
+	}
+
+	wg.Wait()
+}

weed/topology/rack.go

@@ -0,0 +1,161 @@
+package topology
+
+import (
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type Rack struct {
+	NodeImpl
+}
+
+func NewRack(id string) *Rack {
+	r := &Rack{}
+	r.id = NodeId(id)
+	r.nodeType = "Rack"
+	r.diskUsages = newDiskUsages()
+	r.children = make(map[NodeId]Node)
+	r.capacityReservations = newCapacityReservations()
+	r.NodeImpl.value = r
+	return r
+}
+
+func (r *Rack) FindDataNode(ip string, port int) *DataNode {
+	for _, c := range r.Children() {
+		dn := c.(*DataNode)
+		if dn.MatchLocation(ip, port) {
+			return dn
+		}
+	}
+	return nil
+}
+
+// FindDataNodeById finds a DataNode by its ID using O(1) map lookup
+func (r *Rack) FindDataNodeById(id string) *DataNode {
+	r.RLock()
+	defer r.RUnlock()
+	if c, ok := r.children[NodeId(id)]; ok {
+		return c.(*DataNode)
+	}
+	return nil
+}
+
+func (r *Rack) GetOrCreateDataNode(ip string, port int, grpcPort int, publicUrl string, id string, maxVolumeCounts map[string]uint32) *DataNode {
+	r.Lock()
+	defer r.Unlock()
+
+	// Normalize the id parameter (trim whitespace)
+	id = strings.TrimSpace(id)
+
+	// Determine the node ID: use provided id, or fall back to ip:port for backward compatibility
+	nodeId := util.GetVolumeServerId(id, ip, port)
+
+	// First, try to find by node ID using O(1) map lookup (stable identity)
+	if c, ok := r.children[NodeId(nodeId)]; ok {
+		dn := c.(*DataNode)
+		// Log if IP or Port changed (e.g., pod rescheduled in K8s)
+		addrChanged := dn.Ip != ip || dn.Port != port
+		var oldAddr pb.ServerAddress
+		if addrChanged {
+			oldAddr = dn.ServerAddress()
+			glog.V(0).Infof("DataNode %s address changed from %s:%d to %s:%d", nodeId, dn.Ip, dn.Port, ip, port)
+		}
+		// Update the IP/Port in case they changed
+		dn.Ip = ip
+		dn.Port = port
+		dn.GrpcPort = grpcPort
+		dn.PublicUrl = publicUrl
+		dn.LastSeen = time.Now().Unix()
+		if addrChanged {
+			if topo := r.GetTopology(); topo != nil {
+				topo.unregisterDataNodeAddress(oldAddr, dn)
+				topo.registerDataNodeAddress(dn)
+			}
+		}
+		return dn
+	}
+
+	// For backward compatibility: if explicit id was provided, also check by ip:port
+	// to handle transition from old (ip:port) to new (explicit id) behavior
+	ipPortId := util.JoinHostPort(ip, port)
+	if nodeId != ipPortId {
+		for oldId, c := range r.children {
+			dn := c.(*DataNode)
+			if dn.MatchLocation(ip, port) {
+				// Only transition if the oldId exactly matches ip:port (legacy identification).
+				// If oldId is different, this is a node with an explicit id that happens to
+				// reuse the same ip:port - don't incorrectly merge them.
+				if string(oldId) != ipPortId {
+					glog.Warningf("Volume server with id %s has ip:port %s which is used by node %s", nodeId, ipPortId, oldId)
+					continue
+				}
+				// Found a legacy node identified by ip:port, transition it to use the new explicit id
+				glog.V(0).Infof("Volume server %s transitioning id from %s to %s", dn.Url(), oldId, nodeId)
+				// Re-key the node in the children map with the new id
+				delete(r.children, oldId)
+				dn.id = NodeId(nodeId)
+				r.children[NodeId(nodeId)] = dn
+				// Update connection info in case they changed; address itself is
+				// unchanged on legacy transition, so the index entry stays valid.
+				dn.GrpcPort = grpcPort
+				dn.PublicUrl = publicUrl
+				dn.LastSeen = time.Now().Unix()
+				return dn
+			}
+		}
+	}
+
+	dn := NewDataNode(nodeId)
+	dn.Ip = ip
+	dn.Port = port
+	dn.GrpcPort = grpcPort
+	dn.PublicUrl = publicUrl
+	dn.LastSeen = time.Now().Unix()
+	r.doLinkChildNode(dn)
+	for diskType, maxVolumeCount := range maxVolumeCounts {
+		disk := NewDisk(diskType)
+		disk.diskUsages.getOrCreateDisk(types.ToDiskType(diskType)).maxVolumeCount = int64(maxVolumeCount)
+		dn.LinkChildNode(disk)
+	}
+	return dn
+}
+
+type RackInfo struct {
+	Id        NodeId         `json:"Id"`
+	DataNodes []DataNodeInfo `json:"DataNodes"`
+}
+
+func (r *Rack) ToInfo() (info RackInfo) {
+	info.Id = r.Id()
+	var dns []DataNodeInfo
+	for _, c := range r.Children() {
+		dn := c.(*DataNode)
+		dns = append(dns, dn.ToInfo())
+	}
+
+	slices.SortFunc(dns, func(a, b DataNodeInfo) int {
+		return strings.Compare(a.Url, b.Url)
+	})
+
+	info.DataNodes = dns
+	return
+}
+
+func (r *Rack) ToRackInfo(filter VolumeFilter) *master_pb.RackInfo {
+	m := &master_pb.RackInfo{
+		Id:        string(r.Id()),
+		DiskInfos: r.diskUsages.ToDiskInfo(),
+	}
+	for _, c := range r.Children() {
+		dn := c.(*DataNode)
+		m.DataNodeInfos = append(m.DataNodeInfos, dn.ToDataNodeInfo(filter))
+	}
+	return m
+}

weed/topology/replication_metrics_test.go

@@ -0,0 +1,31 @@
+package topology
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+)
+
+func TestClassifyReplicationError(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+		want string
+	}{
+		{"deadline exceeded", context.DeadlineExceeded, stats.FailureTimeout},
+		{"context cancelled", context.Canceled, stats.FailureContextCancelled},
+		{"connection refused", errors.New("connection refused"), stats.FailureConnectionRefused},
+		{"generic error", errors.New("internal server error"), stats.FailureServerError},
+		{"nil error", nil, ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := classifyReplicationError(tt.err)
+			if got != tt.want {
+				t.Errorf("classifyReplicationError() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}

weed/topology/reported_volumes.go

@@ -0,0 +1,79 @@
+package topology
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// reportedVolumes records which disk types a heartbeat named each volume on, so
+// a volume that moved between a server's disks is dropped from the one it left.
+// Disk types are interned because a string header per volume would cost more
+// than the map holding them.
+type reportedVolumes struct {
+	diskTypes []string
+	byVolume  map[needle.VolumeId]int32
+	// extra covers volumes named on several disk types, a stale twin rather than
+	// a move. Nil otherwise.
+	extra map[needle.VolumeId][]int32
+	// duplicated: named twice on one disk type, which the master stores once.
+	duplicated bool
+}
+
+func newReportedVolumes(size int) *reportedVolumes {
+	return &reportedVolumes{byVolume: make(map[needle.VolumeId]int32, size)}
+}
+
+func (r *reportedVolumes) add(vid needle.VolumeId, diskType string) {
+	index := r.internDiskType(diskType)
+	existing, seen := r.byVolume[vid]
+	if !seen {
+		r.byVolume[vid] = index
+		return
+	}
+	if existing == index || r.hasExtra(vid, index) {
+		r.duplicated = true
+		return
+	}
+	if r.extra == nil {
+		r.extra = make(map[needle.VolumeId][]int32)
+	}
+	r.extra[vid] = append(r.extra[vid], index)
+}
+
+func (r *reportedVolumes) internDiskType(diskType string) int32 {
+	if index := r.diskTypeIndex(diskType); index >= 0 {
+		return index
+	}
+	r.diskTypes = append(r.diskTypes, diskType)
+	return int32(len(r.diskTypes) - 1)
+}
+
+// diskTypeIndex returns -1 when the heartbeat named no volume on the type.
+func (r *reportedVolumes) diskTypeIndex(diskType string) int32 {
+	for i, known := range r.diskTypes {
+		if known == diskType {
+			return int32(i)
+		}
+	}
+	return -1
+}
+
+func (r *reportedVolumes) namedOn(vid needle.VolumeId, index int32) bool {
+	if index < 0 {
+		return false
+	}
+	if stored, ok := r.byVolume[vid]; ok && stored == index {
+		return true
+	}
+	return r.hasExtra(vid, index)
+}
+
+func (r *reportedVolumes) hasExtra(vid needle.VolumeId, index int32) bool {
+	for _, other := range r.extra[vid] {
+		if other == index {
+			return true
+		}
+	}
+	return false
+}
+
+func (r *reportedVolumes) count() int { return len(r.byVolume) }

weed/topology/store_replicate.go

@@ -0,0 +1,357 @@
+package topology
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/buffer_pool"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	"google.golang.org/grpc"
+)
+
+// ReplicatedWrite writes a needle to the local volume and fans it out to all
+// remote replica locations. When type=replicate is set, the request is itself
+// a forwarded replication and no further remote lookups are performed.
+// Returns isUnchanged=true when the local write determined the needle content
+// was already present.
+func ReplicatedWrite(ctx context.Context, masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption, s *storage.Store, volumeId needle.VolumeId, n *needle.Needle, r *http.Request, contentMd5 string) (isUnchanged bool, err error) {
+
+	//check JWT
+	jwt := security.GetJwt(r)
+
+	// check whether this is a replicated write request
+	var remoteLocations []operation.Location
+	if r.FormValue("type") != "replicate" {
+		// this is the initial request
+		remoteLocations, err = GetWritableRemoteReplications(s, grpcDialOption, volumeId, masterFn)
+		if err != nil {
+			glog.V(0).Infoln(err)
+			return
+		}
+	}
+
+	// read fsync value
+	fsync := false
+	if r.FormValue("fsync") == "true" {
+		fsync = true
+	}
+
+	replicaCount := len(remoteLocations)
+
+	if replicaCount > 0 {
+		// Record replication duration histogram for the overall write operation
+		defer func(t time.Time) {
+			stats.VolumeServerReplicationHistogram.WithLabelValues(stats.ReplicationOpWrite).Observe(time.Since(t).Seconds())
+		}(time.Now())
+	}
+
+	if s.GetVolume(volumeId) != nil {
+		start := time.Now()
+
+		inFlightGauge := stats.VolumeServerInFlightRequestsGauge.WithLabelValues(stats.WriteToLocalDisk)
+		inFlightGauge.Inc()
+		defer inFlightGauge.Dec()
+
+		isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, true, fsync)
+		stats.VolumeServerRequestHistogram.WithLabelValues(stats.WriteToLocalDisk).Observe(time.Since(start).Seconds())
+		if err != nil {
+			stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorWriteToLocalDisk).Inc()
+			err = fmt.Errorf("failed to write to local disk: %w", err)
+			glog.V(0).Infoln(err)
+			return
+		}
+	}
+
+	// Observe replication targets histogram for all operations (including zero)
+	stats.VolumeServerReplicationTargets.Observe(float64(replicaCount))
+
+	if replicaCount > 0 { //send to other replica locations
+		start := time.Now()
+
+		inFlightGauge := stats.VolumeServerInFlightRequestsGauge.WithLabelValues(stats.WriteToReplicas)
+		inFlightGauge.Inc()
+		defer inFlightGauge.Dec()
+
+		err = DistributedOperation(ctx, remoteLocations, func(ctx context.Context, location operation.Location) error {
+			u := url.URL{
+				Scheme: "http",
+				Host:   location.Url,
+				Path:   r.URL.Path,
+			}
+			q := url.Values{
+				"type": {"replicate"},
+				"ttl":  {n.Ttl.String()},
+			}
+			if fsync {
+				q.Set("fsync", "true")
+			}
+			if n.LastModified > 0 {
+				q.Set("ts", strconv.FormatUint(n.LastModified, 10))
+			}
+			if n.IsChunkedManifest() {
+				q.Set("cm", "true")
+			}
+			u.RawQuery = q.Encode()
+
+			pairMap := make(map[string]string)
+			if n.HasPairs() {
+				tmpMap := make(map[string]string)
+				err := json.Unmarshal(n.Pairs, &tmpMap)
+				if err != nil {
+					stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorUnmarshalPairs).Inc()
+					glog.V(0).Infoln("Unmarshal pairs error:", err)
+				}
+				for k, v := range tmpMap {
+					pairMap[needle.PairNamePrefix+k] = v
+				}
+			}
+			bytesBuffer := buffer_pool.SyncPoolGetBuffer()
+			defer buffer_pool.SyncPoolPutBuffer(bytesBuffer)
+
+			// volume server do not know about encryption
+			// TODO optimize here to compress data only once
+			uploadOption := &operation.UploadOption{
+				UploadUrl:         u.String(),
+				Filename:          string(n.Name),
+				Cipher:            false,
+				IsInputCompressed: n.IsCompressed(),
+				IsReplication:     true,
+				MimeType:          string(n.Mime),
+				PairMap:           pairMap,
+				Jwt:               jwt,
+				Md5:               contentMd5,
+				BytesBuffer:       bytesBuffer,
+				MaxAttempts:       1, // fail fast on a dead replica; the client write retries
+			}
+
+			uploader, err := operation.NewUploader()
+			if err != nil {
+				glog.Errorf("replication-UploadData, err:%v, url:%s", err, u.String())
+				return err
+			}
+			_, err = uploader.UploadData(ctx, n.Data, uploadOption)
+			if err != nil {
+				glog.Errorf("replication-UploadData, err:%v, url:%s", err, u.String())
+			}
+			return err
+		})
+		stats.VolumeServerRequestHistogram.WithLabelValues(stats.WriteToReplicas).Observe(time.Since(start).Seconds())
+		if err != nil {
+			stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc()
+			stats.VolumeServerReplicationCounter.WithLabelValues(stats.ReplicationOpWrite, stats.ReplicationFailure).Inc()
+			reason := classifyReplicationError(err)
+			stats.VolumeServerReplicationFailures.WithLabelValues(stats.ReplicationOpWrite, reason).Inc()
+			err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err)
+			glog.V(0).Infoln(err)
+			return false, err
+		}
+		stats.VolumeServerReplicationCounter.WithLabelValues(stats.ReplicationOpWrite, stats.ReplicationSuccess).Inc()
+	}
+	return
+}
+
+// ReplicatedDelete deletes a needle from the local volume and sends delete
+// requests to all remote replica locations. Replica deletes use
+// context.Background() so that a client disconnect does not orphan replica
+// deletes.
+func ReplicatedDelete(masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption, store *storage.Store, volumeId needle.VolumeId, n *needle.Needle, r *http.Request) (size types.Size, err error) {
+
+	//check JWT
+	jwt := security.GetJwt(r)
+
+	var remoteLocations []operation.Location
+	if r.FormValue("type") != "replicate" {
+		remoteLocations, err = GetRemoteReplications(store, grpcDialOption, volumeId, masterFn)
+		if err != nil {
+			glog.V(0).Infoln(err)
+			return
+		}
+	}
+
+	replicaCount := len(remoteLocations)
+
+	if replicaCount > 0 {
+		// Record replication duration and operation counter for delete
+		defer func(t time.Time) {
+			stats.VolumeServerReplicationHistogram.WithLabelValues(stats.ReplicationOpDelete).Observe(time.Since(t).Seconds())
+			if err != nil {
+				stats.VolumeServerReplicationCounter.WithLabelValues(stats.ReplicationOpDelete, stats.ReplicationFailure).Inc()
+			} else {
+				stats.VolumeServerReplicationCounter.WithLabelValues(stats.ReplicationOpDelete, stats.ReplicationSuccess).Inc()
+			}
+		}(time.Now())
+	}
+
+	size, err = store.DeleteVolumeNeedle(volumeId, n)
+	if err != nil {
+		glog.V(0).Infoln("delete error:", err)
+		return
+	}
+
+	// Observe replication targets histogram for all operations (including zero)
+	stats.VolumeServerReplicationTargets.Observe(float64(replicaCount))
+
+	if replicaCount > 0 { //send to other replica locations
+		// background, not r.Context(): a client disconnect must not orphan replica deletes
+		if err = DistributedOperation(context.Background(), remoteLocations, func(ctx context.Context, location operation.Location) error {
+			url, normalizeErr := util_http.GetGlobalHttpClient().NormalizeHttpScheme(location.Url + r.URL.Path + "?type=replicate")
+			if normalizeErr != nil {
+				return normalizeErr
+			}
+			if jwt != "" && (!strings.HasPrefix(url, "https://") || !util_http.GetGlobalHttpClient().IsTLSVerified()) {
+				return fmt.Errorf("refusing to forward delete authorization to %s without HTTPS", location.Url)
+			}
+			return util_http.Delete(url, string(jwt))
+		}); err != nil {
+			reason := classifyReplicationError(err)
+			stats.VolumeServerReplicationFailures.WithLabelValues(stats.ReplicationOpDelete, reason).Inc()
+			size = 0
+		}
+	}
+	return
+}
+
+type DistributedOperationResult map[string]error
+
+func (dr DistributedOperationResult) Error() error {
+	var errs []string
+	for k, v := range dr {
+		if v != nil {
+			errs = append(errs, fmt.Sprintf("[%s]: %v", k, v))
+		}
+	}
+	if len(errs) == 0 {
+		return nil
+	}
+	return errors.New(strings.Join(errs, "\n"))
+}
+
+type RemoteResult struct {
+	Host  string
+	Error error
+}
+
+func DistributedOperation(ctx context.Context, locations []operation.Location, op func(ctx context.Context, location operation.Location) error) error {
+	length := len(locations)
+	if length == 0 {
+		return nil
+	}
+
+	// cancel outstanding replica ops once the outcome is decided
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	// buffered so a straggler (e.g. a replica stalled on a TCP dial timeout) can
+	// still deliver its result and exit after we have already returned.
+	resultCh := make(chan RemoteResult, length)
+	for _, location := range locations {
+		go func(location operation.Location) {
+			resultCh <- RemoteResult{location.Url, op(ctx, location)}
+		}(location)
+	}
+
+	ret := DistributedOperationResult(make(map[string]error))
+	for i := 0; i < length; i++ {
+		result := <-resultCh
+		ret[result.Host] = result.Error
+		if result.Error != nil {
+			// fail fast on the first error instead of waiting for slow replicas
+			return ret.Error()
+		}
+	}
+	return ret.Error()
+}
+
+func GetRemoteReplications(s *storage.Store, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, masterFn operation.GetMasterFn) ([]operation.Location, error) {
+	return getRemoteReplications(s, grpcDialOption, volumeId, masterFn, false)
+}
+
+func GetWritableRemoteReplications(s *storage.Store, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, masterFn operation.GetMasterFn) ([]operation.Location, error) {
+	return getRemoteReplications(s, grpcDialOption, volumeId, masterFn, true)
+}
+
+func getRemoteReplications(s *storage.Store, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, masterFn operation.GetMasterFn, writableOnly bool) (remoteLocations []operation.Location, err error) {
+
+	v := s.GetVolume(volumeId)
+	if v != nil && v.ReplicaPlacement.GetCopyCount() == 1 {
+		return
+	}
+
+	// not on local store, or has replications
+	lookupResults, lookupErr := operation.LookupVolumeIds(masterFn, grpcDialOption, []string{volumeId.String()}, false)
+	lookupResult := lookupResults[volumeId.String()]
+	writableLocations := 0
+	if lookupErr == nil {
+		if lookupResult == nil {
+			err = fmt.Errorf("replicating lookup returned no result for %d", volumeId)
+			return
+		}
+		selfUrl := util.JoinHostPort(s.Ip, s.Port)
+		for _, location := range lookupResult.Locations {
+			if writableOnly && location.ReadOnly {
+				continue
+			}
+			if !writableOnly && location.ReadOnly && !location.ReadOnlyCanDelete {
+				continue
+			}
+			writableLocations++
+			if location.Url != selfUrl {
+				remoteLocations = append(remoteLocations, location)
+			}
+		}
+	} else {
+		err = fmt.Errorf("replicating lookup failed for %d: %v", volumeId, lookupErr)
+		return
+	}
+
+	if v != nil {
+		// has one local and has remote replications
+		copyCount := v.ReplicaPlacement.GetCopyCount()
+		if writableLocations < copyCount {
+			operation.InvalidateVolumeIdLocationCache(volumeId.String())
+			label := "replication"
+			if writableOnly {
+				label = "writable replication"
+			}
+			err = fmt.Errorf("%s locations [%d] is less than volume %d replication copy count [%d]",
+				label, writableLocations, volumeId, copyCount)
+		}
+	}
+
+	return
+}
+
+// classifyReplicationError maps a Go error to a bounded-cardinality failure
+// reason label for the replication_failures_total metric. Returns an empty
+// string for nil errors.
+func classifyReplicationError(err error) string {
+	if err == nil {
+		return ""
+	}
+	if errors.Is(err, context.DeadlineExceeded) {
+		return stats.FailureTimeout
+	}
+	if errors.Is(err, context.Canceled) {
+		return stats.FailureContextCancelled
+	}
+	errStr := err.Error()
+	if strings.Contains(errStr, "connection refused") {
+		return stats.FailureConnectionRefused
+	}
+	return stats.FailureServerError
+}

weed/topology/store_replicate_test.go

@@ -0,0 +1,227 @@
+package topology
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// TestDistributedOperationCancelsSiblingsOnFirstError verifies that once one
+// replica fails, an outstanding replica still stalled in a dial timeout is
+// cancelled rather than gating the caller until it times out.
+func TestDistributedOperationCancelsSiblingsOnFirstError(t *testing.T) {
+	locations := []operation.Location{{Url: "fast"}, {Url: "slow"}}
+	cancelled := make(chan struct{}, 1)
+
+	start := time.Now()
+	err := DistributedOperation(context.Background(), locations, func(ctx context.Context, location operation.Location) error {
+		if location.Url == "fast" {
+			return errors.New("connection refused")
+		}
+		// slow: a replica stalled in a dial timeout
+		select {
+		case <-ctx.Done():
+			cancelled <- struct{}{}
+			return ctx.Err()
+		case <-time.After(10 * time.Second):
+			return nil
+		}
+	})
+
+	if err == nil {
+		t.Fatal("expected an error from the fast-failing replica")
+	}
+	if elapsed := time.Since(start); elapsed > 2*time.Second {
+		t.Fatalf("did not fail fast: took %v", elapsed)
+	}
+	select {
+	case <-cancelled:
+	case <-time.After(2 * time.Second):
+		t.Fatal("slow replica was not cancelled after the first error")
+	}
+}
+
+func TestDistributedOperationEmpty(t *testing.T) {
+	err := DistributedOperation(context.Background(), nil, func(ctx context.Context, location operation.Location) error {
+		t.Fatal("op should not be called when there are no locations")
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("expected nil for no locations, got %v", err)
+	}
+}
+
+type mockMasterServer struct {
+	master_pb.UnimplementedSeaweedServer
+	mu        sync.Mutex
+	calls     int
+	locations []*master_pb.Location
+}
+
+func (m *mockMasterServer) LookupVolume(ctx context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.calls++
+	var vls []*master_pb.LookupVolumeResponse_VolumeIdLocation
+	for _, vid := range req.VolumeOrFileIds {
+		vls = append(vls, &master_pb.LookupVolumeResponse_VolumeIdLocation{
+			VolumeOrFileId: vid,
+			Locations:      m.locations,
+		})
+	}
+	return &master_pb.LookupVolumeResponse{VolumeIdLocations: vls}, nil
+}
+
+func startMockMasterServer(t *testing.T, master *mockMasterServer) (operation.GetMasterFn, grpc.DialOption) {
+	t.Helper()
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	grpcServer := grpc.NewServer()
+	master_pb.RegisterSeaweedServer(grpcServer, master)
+	serveErr := make(chan error, 1)
+	go func() { serveErr <- grpcServer.Serve(lis) }()
+	t.Cleanup(func() {
+		grpcServer.Stop()
+		if err := <-serveErr; err != nil && !errors.Is(err, grpc.ErrServerStopped) {
+			t.Errorf("mock master serve: %v", err)
+		}
+	})
+
+	grpcPort := lis.Addr().(*net.TCPAddr).Port
+	return func(_ context.Context) pb.ServerAddress {
+		return pb.NewServerAddressWithGrpcPort(fmt.Sprintf("127.0.0.1:%d", grpcPort), grpcPort)
+	}, grpc.WithTransportCredentials(insecure.NewCredentials())
+}
+
+func TestGetWritableRemoteReplicationsRefreshesReadOnlyReplicas(t *testing.T) {
+	master := &mockMasterServer{locations: []*master_pb.Location{
+		{Url: "127.0.0.1:8080"},
+		{Url: "127.0.0.2:8080"},
+	}}
+	masterFn, dialOption := startMockMasterServer(t, master)
+	store := &storage.Store{Ip: "127.0.0.1", Port: 8080}
+	volumeId := needle.VolumeId(1)
+	operation.InvalidateVolumeIdLocationCache(volumeId.String())
+
+	locations, err := GetWritableRemoteReplications(store, dialOption, volumeId, masterFn)
+	if err != nil {
+		t.Fatalf("first lookup: %v", err)
+	}
+	if len(locations) != 1 || locations[0].Url != "127.0.0.2:8080" {
+		t.Fatalf("first lookup locations = %v", locations)
+	}
+
+	master.mu.Lock()
+	master.locations = []*master_pb.Location{
+		{Url: "127.0.0.1:8080"},
+		{Url: "127.0.0.2:8080", ReadOnly: true, ReadOnlyCanDelete: true},
+	}
+	master.mu.Unlock()
+
+	locations, err = GetWritableRemoteReplications(store, dialOption, volumeId, masterFn)
+	if err != nil {
+		t.Fatalf("second lookup: %v", err)
+	}
+	if len(locations) != 0 {
+		t.Fatalf("read-only replica remained a write target: %v", locations)
+	}
+	master.mu.Lock()
+	calls := master.calls
+	master.mu.Unlock()
+	if calls != 2 {
+		t.Fatalf("master lookup calls = %d, want 2", calls)
+	}
+
+	locations, err = GetRemoteReplications(store, dialOption, volumeId, masterFn)
+	if err != nil {
+		t.Fatalf("delete lookup: %v", err)
+	}
+	if len(locations) != 1 || !locations[0].ReadOnly || !locations[0].ReadOnlyCanDelete {
+		t.Fatalf("read-only-can-delete target was dropped: %v", locations)
+	}
+}
+
+// TestReplicatedWriteForwardsFsyncToReplicas verifies that the fsync=true
+// request parameter is forwarded to replica volume servers in the fan-out
+// request, so a durable write means every replica has flushed to disk.
+func TestReplicatedWriteForwardsFsyncToReplicas(t *testing.T) {
+	replicaQueries := make(chan url.Values, 4)
+	replica := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		replicaQueries <- r.URL.Query()
+		w.WriteHeader(http.StatusCreated)
+		if _, err := w.Write([]byte(`{"size":1}`)); err != nil {
+			t.Errorf("replica write response: %v", err)
+		}
+	}))
+	defer replica.Close()
+	replicaHost := strings.TrimPrefix(replica.URL, "http://")
+
+	masterFn, dialOption := startMockMasterServer(t, &mockMasterServer{
+		locations: []*master_pb.Location{{Url: replicaHost}},
+	})
+
+	store := &storage.Store{}
+	volumeId := needle.VolumeId(1)
+
+	for _, tc := range []struct {
+		name      string
+		fsync     string
+		wantFsync bool
+	}{
+		{name: "fsync requested", fsync: "true", wantFsync: true},
+		{name: "no fsync requested", fsync: "", wantFsync: false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			operation.InvalidateVolumeIdLocationCache(volumeId.String())
+
+			path := "http://127.0.0.1:8080/1,01637037d6"
+			if tc.fsync != "" {
+				path += "?fsync=" + tc.fsync
+			}
+			r := httptest.NewRequest(http.MethodPost, path, nil)
+			if err := r.ParseForm(); err != nil {
+				t.Fatal(err)
+			}
+
+			n := &needle.Needle{
+				Id:   1,
+				Data: []byte("test data"),
+				Ttl:  needle.EMPTY_TTL,
+			}
+			if _, err := ReplicatedWrite(context.Background(), masterFn, dialOption, store, volumeId, n, r, ""); err != nil {
+				t.Fatalf("ReplicatedWrite: %v", err)
+			}
+
+			select {
+			case q := <-replicaQueries:
+				got := q.Get("fsync")
+				if tc.wantFsync && got != "true" {
+					t.Errorf("expected fsync=true in replica query, got %q (query: %v)", got, q)
+				}
+				if !tc.wantFsync && got != "" {
+					t.Errorf("expected no fsync in replica query, got %q (query: %v)", got, q)
+				}
+			case <-time.After(5 * time.Second):
+				t.Fatal("replica never received the fan-out request")
+			}
+		})
+	}
+}

weed/topology/topology.go

@@ -0,0 +1,867 @@
+package topology
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math"
+	"math/rand/v2"
+	"slices"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+
+	backoff "github.com/cenkalti/backoff/v4"
+
+	hashicorpRaft "github.com/hashicorp/raft"
+	"github.com/seaweedfs/raft"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const (
+	// WarmupPulseMultiplier is the number of heartbeat intervals to wait after
+	// a leader change before treating volume lookup misses as definitive.
+	WarmupPulseMultiplier = 3
+)
+
+type Topology struct {
+	vacuumLockCounter int64
+	NodeImpl
+
+	collectionMap  *util.ConcurrentReadMap
+	ecShardMap     map[needle.VolumeId]*EcShardLocations
+	ecShardMapLock sync.RWMutex
+
+	pulse int64
+
+	volumeSizeLimit          uint64
+	replicationAsMin         bool
+	vacuumDisabledByOperator atomic.Bool // true when operator manually disables vacuum
+	vacuumDisabledByPlugin   atomic.Bool // true when disabled by the vacuum plugin monitor
+	adminServerConnectedFunc func() bool // optional callback to check admin server presence
+
+	Sequence sequence.Sequencer
+
+	chanFullVolumes    chan storage.VolumeInfo
+	chanCrowdedVolumes chan storage.VolumeInfo
+
+	Configuration *Configuration
+
+	RaftServer           raft.Server
+	RaftServerAccessLock sync.RWMutex
+	HashicorpRaft        *hashicorpRaft.Raft
+	barrierLock          sync.Mutex
+	barrierDone          bool
+
+	UuidAccessLock sync.RWMutex
+	UuidMap        map[string][]string
+
+	topologyId     string
+	topologyIdLock sync.RWMutex
+
+	lastLeaderChangeTime     time.Time
+	hadVolumesAtLeaderChange bool
+	lastLeaderChangeTimeLock sync.RWMutex
+
+	// dataNodeIndex is an address -> *DataNode lookup so callers (e.g. the
+	// Ping admission gate) do not have to walk every dc/rack/node tier on
+	// every request. Keys use the canonical http form returned by
+	// pb.ServerAddress.ToHttpAddress so a target like "1.2.3.4:8080" finds
+	// the same node whether or not the grpc port suffix is present.
+	dataNodeIndex     map[string]*DataNode
+	dataNodeIndexLock sync.RWMutex
+}
+
+func NewTopology(id string, seq sequence.Sequencer, volumeSizeLimit uint64, pulse int, replicationAsMin bool) *Topology {
+	t := &Topology{}
+	t.id = NodeId(id)
+	t.nodeType = "Topology"
+	t.NodeImpl.value = t
+	t.diskUsages = newDiskUsages()
+	t.children = make(map[NodeId]Node)
+	t.capacityReservations = newCapacityReservations()
+	t.collectionMap = util.NewConcurrentReadMap()
+	t.ecShardMap = make(map[needle.VolumeId]*EcShardLocations)
+	t.pulse = int64(pulse)
+	t.volumeSizeLimit = volumeSizeLimit
+	t.replicationAsMin = replicationAsMin
+
+	t.Sequence = seq
+
+	t.chanFullVolumes = make(chan storage.VolumeInfo)
+	t.chanCrowdedVolumes = make(chan storage.VolumeInfo)
+
+	t.Configuration = &Configuration{}
+	t.dataNodeIndex = make(map[string]*DataNode)
+
+	return t
+}
+
+// LookupDataNodeByAddress returns the registered DataNode that serves addr,
+// or nil if no such node has been observed. Lookup is O(1) and uses the
+// canonical http form of the address so callers that pass either
+// "host:port" or "host:port.grpc" find the same node.
+func (t *Topology) LookupDataNodeByAddress(addr pb.ServerAddress) *DataNode {
+	if addr == "" {
+		return nil
+	}
+	t.dataNodeIndexLock.RLock()
+	defer t.dataNodeIndexLock.RUnlock()
+	if t.dataNodeIndex == nil {
+		return nil
+	}
+	return t.dataNodeIndex[addr.ToHttpAddress()]
+}
+
+// registerDataNodeAddress records dn in the address index under its current
+// http address. Callers must invoke unregisterDataNodeAddress with the prior
+// address whenever a node's Ip or Port changes (e.g. k8s pod reschedule).
+func (t *Topology) registerDataNodeAddress(dn *DataNode) {
+	if dn == nil {
+		return
+	}
+	key := dn.ServerAddress().ToHttpAddress()
+	if key == "" {
+		return
+	}
+	t.dataNodeIndexLock.Lock()
+	defer t.dataNodeIndexLock.Unlock()
+	if t.dataNodeIndex == nil {
+		t.dataNodeIndex = make(map[string]*DataNode)
+	}
+	t.dataNodeIndex[key] = dn
+}
+
+// unregisterDataNodeAddress removes the index entry for addr, but only when
+// the entry still points at dn. The conditional guard avoids dropping a
+// freshly re-registered node whose address happens to alias the one being
+// removed (e.g. legacy id transitions or a fast restart).
+func (t *Topology) unregisterDataNodeAddress(addr pb.ServerAddress, dn *DataNode) {
+	if addr == "" {
+		return
+	}
+	key := addr.ToHttpAddress()
+	if key == "" {
+		return
+	}
+	t.dataNodeIndexLock.Lock()
+	defer t.dataNodeIndexLock.Unlock()
+	if existing, ok := t.dataNodeIndex[key]; ok && (dn == nil || existing == dn) {
+		delete(t.dataNodeIndex, key)
+	}
+}
+
+// FreeBytes sums what every volume server reports as free on its filesystems.
+// reported is false unless all of them answered: the one that stayed quiet may
+// be the one holding the room, and a partial sum would read as a cluster with
+// none left.
+func (t *Topology) FreeBytes() (freeBytes uint64, reported bool) {
+	for _, dcNode := range t.Children() {
+		for _, rackNode := range dcNode.Children() {
+			for _, dataNode := range rackNode.Children() {
+				nodeFreeBytes, nodeReported := dataNode.GetDiskUsages().FreeBytes()
+				if !nodeReported {
+					return 0, false
+				}
+				freeBytes += nodeFreeBytes
+			}
+		}
+	}
+	return freeBytes, true
+}
+
+func (t *Topology) IsChildLocked() (bool, error) {
+	if t.IsLocked() {
+		return true, errors.New("topology is locked")
+	}
+	for _, dcNode := range t.Children() {
+		if dcNode.IsLocked() {
+			return true, fmt.Errorf("topology child %s is locked", dcNode.String())
+		}
+		for _, rackNode := range dcNode.Children() {
+			if rackNode.IsLocked() {
+				return true, fmt.Errorf("dc %s child %s is locked", dcNode.String(), rackNode.String())
+			}
+			for _, dataNode := range rackNode.Children() {
+				if dataNode.IsLocked() {
+					return true, fmt.Errorf("rack %s child %s is locked", rackNode.String(), dataNode.Id())
+				}
+			}
+		}
+	}
+	return false, nil
+}
+
+// SetLastLeaderChangeTime records the time of the most recent leader transition.
+// It also snapshots whether the topology already had known volumes at that
+// moment. IsWarmingUp uses the snapshot instead of the live MaxVolumeId so a
+// fresh cluster that happens to grow its first volume inside the warmup window
+// does not retroactively flip into "warming up" state — there is no prior
+// topology to wait for on a bootstrap.
+func (t *Topology) SetLastLeaderChangeTime(ts time.Time) {
+	hadVolumes := t.GetMaxVolumeId() > 0
+	t.lastLeaderChangeTimeLock.Lock()
+	defer t.lastLeaderChangeTimeLock.Unlock()
+	t.lastLeaderChangeTime = ts
+	t.hadVolumesAtLeaderChange = hadVolumes
+}
+
+// GetLastLeaderChangeTime returns the time of the most recent leader transition.
+func (t *Topology) GetLastLeaderChangeTime() time.Time {
+	t.lastLeaderChangeTimeLock.RLock()
+	defer t.lastLeaderChangeTimeLock.RUnlock()
+	return t.lastLeaderChangeTime
+}
+
+// IsWarmingUp returns true if the master recently became leader and may not yet
+// have a complete topology. After a leader change or restart, volume servers need
+// up to WarmupPulseMultiplier heartbeat intervals to reconnect and report their volumes.
+// Returns false on a fresh cluster start — i.e. when no volumes existed at the
+// time of the leader change — since there is no prior topology state to wait for.
+// Checking the *live* MaxVolumeId here would make a bootstrapping cluster flip
+// into warming-up the moment its first volume is grown, which manifested as a
+// 15-second window of spurious Unavailable errors on AssignVolume for workloads
+// that start writing immediately (see #8777).
+func (t *Topology) IsWarmingUp() bool {
+	t.lastLeaderChangeTimeLock.RLock()
+	lastChange := t.lastLeaderChangeTime
+	hadVolumes := t.hadVolumesAtLeaderChange
+	t.lastLeaderChangeTimeLock.RUnlock()
+	if !hadVolumes || lastChange.IsZero() {
+		return false
+	}
+	return time.Since(lastChange) < t.WarmupDuration()
+}
+
+// WarmupDuration returns the configured warmup duration based on pulse interval.
+func (t *Topology) WarmupDuration() time.Duration {
+	return time.Duration(t.pulse*WarmupPulseMultiplier) * time.Second
+}
+
+// RemainingWarmupDuration returns how much warmup time is left, or 0 if not warming up.
+func (t *Topology) RemainingWarmupDuration() time.Duration {
+	if !t.IsWarmingUp() {
+		return 0
+	}
+	remaining := t.WarmupDuration() - time.Since(t.GetLastLeaderChangeTime())
+	if remaining < 0 {
+		return 0
+	}
+	return remaining
+}
+
+func (t *Topology) IsLeader() bool {
+	t.RaftServerAccessLock.RLock()
+	defer t.RaftServerAccessLock.RUnlock()
+
+	if t.RaftServer != nil {
+		if t.RaftServer.State() == raft.Leader {
+			return true
+		}
+		// Directly check leader to avoid re-acquiring lock via MaybeLeader()
+		leader := pb.ServerAddress(t.RaftServer.Leader())
+		if leader != "" {
+			if pb.ServerAddress(t.RaftServer.Name()).Equals(leader) {
+				return true
+			}
+		}
+	} else if t.HashicorpRaft != nil {
+		if t.HashicorpRaft.State() == hashicorpRaft.Leader {
+			return true
+		}
+	}
+	return false
+}
+
+func (t *Topology) IsLeaderAndCanRead() bool {
+	if t.RaftServer != nil {
+		return t.IsLeader()
+	} else if t.HashicorpRaft != nil {
+		return t.IsLeader() && t.DoBarrier()
+	} else {
+		return false
+	}
+}
+
+func (t *Topology) DoBarrier() bool {
+	t.barrierLock.Lock()
+	defer t.barrierLock.Unlock()
+	if t.barrierDone {
+		return true
+	}
+
+	glog.V(0).Infof("raft do barrier")
+	barrier := t.HashicorpRaft.Barrier(2 * time.Minute)
+	if err := barrier.Error(); err != nil {
+		glog.Errorf("failed to wait for barrier, error %s", err)
+		return false
+
+	}
+
+	t.barrierDone = true
+	glog.V(0).Infof("raft do barrier success")
+	return true
+}
+
+func (t *Topology) BarrierReset() {
+	t.barrierLock.Lock()
+	defer t.barrierLock.Unlock()
+	t.barrierDone = false
+}
+
+func (t *Topology) Leader() (l pb.ServerAddress, err error) {
+	exponentialBackoff := backoff.NewExponentialBackOff()
+	exponentialBackoff.InitialInterval = 100 * time.Millisecond
+	exponentialBackoff.MaxElapsedTime = 20 * time.Second
+	leaderNotSelected := errors.New("leader not selected yet")
+	l, err = backoff.RetryWithData(
+		func() (l pb.ServerAddress, err error) {
+			l, err = t.MaybeLeader()
+			if err == nil && l == "" {
+				err = leaderNotSelected
+			}
+			return l, err
+		},
+		exponentialBackoff)
+	if err == leaderNotSelected {
+		l = ""
+	}
+	return l, err
+}
+
+func (t *Topology) MaybeLeader() (l pb.ServerAddress, err error) {
+	t.RaftServerAccessLock.RLock()
+	defer t.RaftServerAccessLock.RUnlock()
+
+	if t.RaftServer != nil {
+		l = pb.ServerAddress(t.RaftServer.Leader())
+		if l == "" && t.RaftServer.State() == raft.Leader {
+			l = pb.ServerAddress(t.RaftServer.Name())
+		}
+	} else if t.HashicorpRaft != nil {
+		l = pb.ServerAddress(t.HashicorpRaft.Leader())
+	} else {
+		err = errors.New("Raft Server not ready yet!")
+	}
+
+	return
+}
+
+func (t *Topology) Lookup(collection string, vid needle.VolumeId) (dataNodes []*DataNode) {
+	// maybe an issue if lots of collections?
+	if collection == "" {
+		for _, c := range t.collectionMap.Items() {
+			if list := c.(*Collection).Lookup(vid); list != nil {
+				return list
+			}
+		}
+	} else {
+		if c, ok := t.collectionMap.Find(collection); ok {
+			return c.(*Collection).Lookup(vid)
+		}
+	}
+
+	if locations, found := t.LookupEcShards(vid); found {
+		for _, loc := range locations.Locations {
+			dataNodes = append(dataNodes, loc...)
+		}
+		return dataNodes
+	}
+
+	return nil
+}
+
+func (t *Topology) NextVolumeId() (needle.VolumeId, error) {
+	if !t.IsLeaderAndCanRead() {
+		return 0, fmt.Errorf("as leader can not read yet")
+
+	}
+	vid := t.GetMaxVolumeId()
+	next := vid.Next()
+
+	t.RaftServerAccessLock.RLock()
+	defer t.RaftServerAccessLock.RUnlock()
+
+	if t.RaftServer != nil {
+		if _, err := t.RaftServer.Do(NewMaxVolumeIdCommand(next, t.GetTopologyId())); err != nil {
+			return 0, err
+		}
+	} else if t.HashicorpRaft != nil {
+		b, err := json.Marshal(NewMaxVolumeIdCommand(next, t.GetTopologyId()))
+		if err != nil {
+			return 0, fmt.Errorf("failed marshal NewMaxVolumeIdCommand: %+v", err)
+		}
+		if future := t.HashicorpRaft.Apply(b, time.Second); future.Error() != nil {
+			return 0, future.Error()
+		}
+	}
+	return next, nil
+}
+
+// DefaultNeedleSizeEstimate is the fallback per-file-ID size estimate when
+// the client does not provide an expected data size.
+const DefaultNeedleSizeEstimate uint64 = 1024 * 1024 // 1 MB
+
+func (t *Topology) PickForWrite(requestedCount uint64, option *VolumeGrowOption, volumeLayout *VolumeLayout, expectedDataSize uint64) (fileId string, count uint64, volumeLocationList *VolumeLocationList, shouldGrow bool, err error) {
+	var vid needle.VolumeId
+	vid, count, volumeLocationList, shouldGrow, err = volumeLayout.PickForWrite(requestedCount, option)
+	if err != nil {
+		return "", 0, nil, shouldGrow, fmt.Errorf("failed to find writable volumes for collection:%s replication:%s ttl:%s error: %v", option.Collection, option.ReplicaPlacement.String(), option.Ttl.String(), err)
+	}
+	if volumeLocationList == nil || volumeLocationList.Length() == 0 {
+		return "", 0, nil, shouldGrow, fmt.Errorf("%s available for collection:%s replication:%s ttl:%s", NoWritableVolumes, option.Collection, option.ReplicaPlacement.String(), option.Ttl.String())
+	}
+	// Track estimated assigned bytes to spread load between heartbeats. A flat
+	// fallback overcharges a small-file workload enough to mark near-empty
+	// volumes full, so prefer the volume's own average.
+	sizePerFile := DefaultNeedleSizeEstimate
+	if expectedDataSize > 0 {
+		sizePerFile = expectedDataSize
+	} else if vi, infoErr := volumeLocationList.Head().GetVolumesById(vid); infoErr == nil && vi.FileCount > 0 {
+		if avg := vi.Size / uint64(vi.FileCount); avg > 0 {
+			sizePerFile = avg
+		}
+	}
+	pendingBytes := min(uint64(count)*sizePerFile, uint64(math.MaxInt64))
+	if volumeLayout.RecordAssign(vid, int64(pendingBytes)) {
+		volumeLayout.AdjustActiveVolumeCountForFull(vid)
+	}
+	nextFileId := t.Sequence.NextFileId(requestedCount)
+	fileId = needle.NewFileId(vid, nextFileId, rand.Uint32()).String()
+	return fileId, count, volumeLocationList, shouldGrow, nil
+}
+
+func (t *Topology) GetVolumeLayout(collectionName string, rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType types.DiskType) *VolumeLayout {
+	return t.collectionMap.Get(collectionName, func() interface{} {
+		return NewCollection(collectionName, t.volumeSizeLimit, t.replicationAsMin)
+	}).(*Collection).GetOrCreateVolumeLayout(rp, ttl, diskType)
+}
+
+// DecayQuietVolumeSizes decays pending assign estimates across every layout.
+// A volume that changed reports within a pulse, so two quiet pulses mean the
+// size on record is the size there is.
+func (t *Topology) DecayQuietVolumeSizes() {
+	quietCutoff := time.Duration(2*t.pulse) * time.Second
+	for _, c := range t.collectionMap.Items() {
+		for _, vl := range c.(*Collection).GetAllVolumeLayouts() {
+			vl.DecayQuietVolumeSizes(quietCutoff)
+		}
+	}
+}
+
+// CollectionVolumeStats aggregates stats across all volume layouts and EC
+// volumes of one collection, or across every collection when collectionName is
+// empty.
+func (t *Topology) CollectionVolumeStats(collectionName string) *VolumeLayoutStats {
+	ret := &VolumeLayoutStats{}
+	var collections []*Collection
+	if collectionName == "" {
+		for _, c := range t.collectionMap.Items() {
+			collections = append(collections, c.(*Collection))
+		}
+	} else if c, found := t.FindCollection(collectionName); found {
+		collections = append(collections, c)
+	}
+	for _, c := range collections {
+		for _, vl := range c.GetAllVolumeLayouts() {
+			stats := vl.Stats()
+			ret.TotalSize += stats.TotalSize
+			ret.UsedSize += stats.UsedSize
+			ret.LogicalUsedSize += stats.LogicalUsedSize
+			ret.FileCount += stats.FileCount
+		}
+	}
+	// EC volumes live outside collectionMap, so a collection whose volumes are
+	// all encoded has no layout left to report them
+	ecStats := t.CollectionEcVolumeStats(collectionName)
+	ret.TotalSize += ecStats.TotalSize
+	ret.UsedSize += ecStats.UsedSize
+	ret.LogicalUsedSize += ecStats.LogicalUsedSize
+	ret.FileCount += ecStats.FileCount
+	return ret
+}
+
+func (t *Topology) ListCollections(includeNormalVolumes, includeEcVolumes bool) (ret []string) {
+	found := make(map[string]bool)
+
+	if includeNormalVolumes {
+		t.collectionMap.RLock()
+		for _, c := range t.collectionMap.Items() {
+			found[c.(*Collection).Name] = true
+		}
+		t.collectionMap.RUnlock()
+	}
+
+	if includeEcVolumes {
+		t.ecShardMapLock.RLock()
+		for _, ecVolumeLocation := range t.ecShardMap {
+			found[ecVolumeLocation.Collection] = true
+		}
+		t.ecShardMapLock.RUnlock()
+	}
+
+	for k := range found {
+		ret = append(ret, k)
+	}
+	slices.Sort(ret)
+
+	return ret
+}
+
+func (t *Topology) FindCollection(collectionName string) (*Collection, bool) {
+	c, hasCollection := t.collectionMap.Find(collectionName)
+	if !hasCollection {
+		return nil, false
+	}
+	return c.(*Collection), hasCollection
+}
+
+func (t *Topology) DeleteCollection(collectionName string) {
+	// The layouts vanish with the collection, but every location they served
+	// holds a bit in its node's lookup digest. Left in place, those bits keep
+	// the node's held and servable digests apart forever, and the master asks
+	// for the full volume list on every heartbeat from then on.
+	// Unpublish first so a racing registration re-resolves into a fresh collection.
+	collection, found := t.collectionMap.Delete(collectionName)
+	if !found {
+		return
+	}
+	for _, vl := range collection.(*Collection).GetAllVolumeLayouts() {
+		vl.releaseLookupOwnership()
+	}
+}
+
+func (t *Topology) DeleteLayout(collectionName string, rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType types.DiskType) {
+	collection, found := t.FindCollection(collectionName)
+	if !found {
+		return
+	}
+	collection.DeleteVolumeLayout(rp, ttl, diskType)
+	if len(collection.storageType2VolumeLayout.Items()) == 0 {
+		t.DeleteCollection(collectionName)
+	}
+}
+
+func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) {
+	diskType := types.ToDiskType(v.DiskType)
+	for {
+		vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
+		if vl.RegisterVolume(&v, dn) {
+			vl.EnsureCorrectWritables(&v)
+			return
+		}
+		// Dropped with its collection; the next lookup creates a fresh one.
+	}
+}
+
+func (t *Topology) UnRegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) {
+	glog.Infof("removing volume info: %+v from %v", v, dn.id)
+	diskType := types.ToDiskType(v.DiskType)
+	volumeLayout := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
+	volumeLayout.UnRegisterVolume(&v, dn)
+	// Drop the series only after the last placement is gone. Deleting while
+	// another data node still holds v would hide under-replication until the
+	// next CollectDeadNodeAndFullVolumes cycle recreates the label.
+	if v.ReplicaPlacement.GetCopyCount() > 1 && len(t.Lookup(v.Collection, v.Id)) == 0 {
+		stats.MasterReplicaPlacementMismatch.DeleteLabelValues(v.Collection, v.Id.String())
+	}
+	if volumeLayout.isEmpty() {
+		t.DeleteLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
+	}
+}
+
+func (t *Topology) DataCenterExists(dcName string) bool {
+	return dcName == "" || t.GetDataCenter(dcName) != nil
+}
+
+func (t *Topology) GetDataCenter(dcName string) (dc *DataCenter) {
+	t.RLock()
+	defer t.RUnlock()
+	for _, c := range t.children {
+		dc = c.(*DataCenter)
+		if string(dc.Id()) == dcName {
+			return dc
+		}
+	}
+	return dc
+}
+
+func (t *Topology) GetOrCreateDataCenter(dcName string) *DataCenter {
+	t.Lock()
+	defer t.Unlock()
+	for _, c := range t.children {
+		dc := c.(*DataCenter)
+		if string(dc.Id()) == dcName {
+			return dc
+		}
+	}
+	dc := NewDataCenter(dcName)
+	t.doLinkChildNode(dc)
+	return dc
+}
+
+func (t *Topology) ListDataCenters() (dcs []string) {
+	t.RLock()
+	defer t.RUnlock()
+	for _, c := range t.children {
+		dcs = append(dcs, string(c.(*DataCenter).Id()))
+	}
+	return dcs
+}
+
+func (t *Topology) ListDCAndRacks() (dcs map[NodeId][]NodeId) {
+	t.RLock()
+	defer t.RUnlock()
+	dcs = make(map[NodeId][]NodeId)
+	for _, dcNode := range t.children {
+		dcNodeId := dcNode.(*DataCenter).Id()
+		for _, rackNode := range dcNode.Children() {
+			dcs[dcNodeId] = append(dcs[dcNodeId], rackNode.(*Rack).Id())
+		}
+	}
+	return dcs
+}
+
+func (t *Topology) SyncDataNodeRegistration(volumes []*master_pb.VolumeInformationMessage, dn *DataNode) (newVolumes, deletedVolumes, changedVolumes []storage.VolumeInfo) {
+	// convert into in memory struct storage.VolumeInfo
+	volumeInfos := make([]storage.VolumeInfo, 0, len(volumes))
+	for _, v := range volumes {
+		if vi, err := storage.NewVolumeInfo(v); err == nil {
+			volumeInfos = append(volumeInfos, vi)
+		} else {
+			glog.V(0).Infof("Fail to convert joined volume information: %v", err)
+		}
+	}
+	// find out the delta volumes
+	newVolumes, deletedVolumes, changedVolumes = dn.UpdateVolumes(volumeInfos)
+	for _, v := range newVolumes {
+		t.RegisterVolumeLayout(v, dn)
+	}
+	for _, v := range deletedVolumes {
+		t.UnRegisterVolumeLayout(v, dn)
+	}
+	// Update effective sizes for all reported volumes (decay pending estimates).
+	// If decay brings a volume eagerly removed by RecordAssign back under the
+	// writable threshold, restore the matching activeVolumeCount.
+	for _, v := range volumeInfos {
+		if v.ReplicaPlacement == nil {
+			continue
+		}
+		diskType := types.ToDiskType(v.DiskType)
+		vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
+		// Self-heal: a volume reported by the data node but missing from the
+		// lookup index is re-registered. This repairs the split left by a
+		// disconnect/reconnect race, where UnRegisterDataNode dropped the volume
+		// from vid2location but the reconnecting full heartbeat skipped it
+		// (still in the disk map, so UpdateVolumes did not report it as new).
+		// Without this, the volume stays visible in volume.list/admin UI yet
+		// LookupVolume returns "volume id not found".
+		if !vl.HasDataNode(v.Id, dn) {
+			for !vl.RegisterVolume(&v, dn) {
+				// Dropped with its collection; the next lookup creates a fresh
+				// one, which the calls below must use too.
+				vl = t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
+			}
+			// Volumes new to the disk map were registered above, so reaching
+			// here means only the lookup index had lost it. Clients were told
+			// it went when the node dropped out, so the repair has to tell them
+			// it is back.
+			newVolumes = append(newVolumes, v)
+		}
+		vl.UpdateOversizedState(&v, dn)
+		if vl.UpdateVolumeSize(v.Id, v.Size, v.CompactRevision, true) {
+			vl.AdjustActiveVolumeCountAfterRecovery(v.Id)
+		}
+		vl.EnsureCorrectWritables(&v)
+	}
+	return
+}
+
+func (t *Topology) IncrementalSyncDataNodeRegistration(newVolumes, deletedVolumes []*master_pb.VolumeShortInformationMessage, dn *DataNode) {
+	var newVis, oldVis []storage.VolumeInfo
+	for _, v := range newVolumes {
+		vi, err := storage.NewVolumeInfoFromShort(v)
+		if err != nil {
+			glog.V(0).Infof("NewVolumeInfoFromShort %v: %v", v, err)
+			continue
+		}
+		newVis = append(newVis, vi)
+	}
+	for _, v := range deletedVolumes {
+		vi, err := storage.NewVolumeInfoFromShort(v)
+		if err != nil {
+			glog.V(0).Infof("NewVolumeInfoFromShort %v: %v", v, err)
+			continue
+		}
+		oldVis = append(oldVis, vi)
+	}
+	dn.DeltaUpdateVolumes(newVis, oldVis)
+
+	type layoutKey struct {
+		id               needle.VolumeId
+		collection       string
+		replicaPlacement byte
+		ttl              uint32
+		diskType         types.DiskType
+	}
+	key := func(vi storage.VolumeInfo) layoutKey {
+		return layoutKey{
+			id:               vi.Id,
+			collection:       vi.Collection,
+			replicaPlacement: vi.ReplicaPlacement.Byte(),
+			ttl:              vi.Ttl.ToUint32(),
+			diskType:         types.ToDiskType(vi.DiskType),
+		}
+	}
+	replacements := make(map[layoutKey]struct{}, len(newVis))
+	for _, vi := range newVis {
+		replacements[key(vi)] = struct{}{}
+		t.RegisterVolumeLayout(vi, dn)
+	}
+	for _, oldVi := range oldVis {
+		if _, replaced := replacements[key(oldVi)]; !replaced {
+			t.UnRegisterVolumeLayout(oldVi, dn)
+		}
+	}
+
+	return
+}
+
+// ApplyVolumeChanges records the volumes a heartbeat reported as changed and
+// returns the ones the node did not already have. Only the named volumes are
+// touched: unlike a full report, silence about a volume says nothing about
+// whether the server still has it.
+//
+// Most changes are a volume growing, which moves no location, so returning
+// only the arrivals keeps a busy cluster from telling every client about
+// volumes they can already reach. The arrival set also includes replicas whose
+// IsRemote() classification flipped on tier transition: the volume is still
+// servable from the same node, but every connected client has stale replica
+// priority and must be told to refresh.
+func (t *Topology) ApplyVolumeChanges(changed []*master_pb.VolumeInformationMessage, dn *DataNode) (newVolumes []storage.VolumeInfo) {
+	volumeInfos := make([]storage.VolumeInfo, 0, len(changed))
+	for _, v := range changed {
+		vi, err := storage.NewVolumeInfo(v)
+		if err != nil {
+			glog.V(0).Infof("Fail to convert changed volume information: %v", err)
+			continue
+		}
+		volumeInfos = append(volumeInfos, vi)
+	}
+
+	for _, vi := range volumeInfos {
+		isNew, isChanged, tierTransition := dn.AddOrUpdateVolume(vi)
+		if vi.ReplicaPlacement == nil {
+			if isNew {
+				newVolumes = append(newVolumes, vi)
+			}
+			continue
+		}
+		vl := t.GetVolumeLayout(vi.Collection, vi.ReplicaPlacement, vi.Ttl, types.ToDiskType(vi.DiskType))
+		// Reaching the lookup index is what makes a volume servable, so a
+		// volume only that index had lost is an arrival as far as clients are
+		// concerned: they were told it went when the node dropped out.
+		becameServable := !vl.HasDataNode(vi.Id, dn)
+		for becameServable && !vl.RegisterVolume(&vi, dn) {
+			// Dropped with its collection; the next lookup creates a fresh one.
+			vl = t.GetVolumeLayout(vi.Collection, vi.ReplicaPlacement, vi.Ttl, types.ToDiskType(vi.DiskType))
+		}
+		if isNew || becameServable || tierTransition || isChanged {
+			newVolumes = append(newVolumes, vi)
+		}
+		vl.UpdateOversizedState(&vi, dn)
+		if vl.UpdateVolumeSize(vi.Id, vi.Size, vi.CompactRevision, true) {
+			vl.AdjustActiveVolumeCountAfterRecovery(vi.Id)
+		}
+		vl.EnsureCorrectWritables(&vi)
+	}
+	return newVolumes
+}
+
+func (t *Topology) DataNodeRegistration(dcName, rackName string, dn *DataNode) {
+	if dn.Parent() != nil {
+		return
+	}
+	// registration to topo
+	dc := t.GetOrCreateDataCenter(dcName)
+	rack := dc.GetOrCreateRack(rackName)
+	rack.LinkChildNode(dn)
+	glog.Infof("[%s] reLink To topo  ", dn.Id())
+}
+
+// IsVacuumDisabled returns true if vacuum is disabled by either the
+// operator or the plugin monitor.
+func (t *Topology) IsVacuumDisabled() bool {
+	return t.vacuumDisabledByOperator.Load() || t.vacuumDisabledByPlugin.Load()
+}
+
+// DisableVacuum is called by the operator (shell command / manual RPC).
+// Only sets the operator flag; does not affect the plugin flag.
+func (t *Topology) DisableVacuum() {
+	glog.V(0).Infof("DisableVacuum (by operator)")
+	t.vacuumDisabledByOperator.Store(true)
+}
+
+// EnableVacuum is called by the operator (shell command / manual RPC).
+// Only clears the operator flag; does not affect the plugin flag.
+func (t *Topology) EnableVacuum() {
+	glog.V(0).Infof("EnableVacuum (by operator)")
+	t.vacuumDisabledByOperator.Store(false)
+}
+
+// DisableVacuumByPlugin is called by the admin server's vacuum monitor
+// when a vacuum plugin worker connects. Only sets the plugin flag.
+func (t *Topology) DisableVacuumByPlugin() {
+	glog.V(0).Infof("DisableVacuum (by plugin worker)")
+	t.vacuumDisabledByPlugin.Store(true)
+}
+
+// EnableVacuumByPlugin is called by the admin server's vacuum monitor
+// when a vacuum plugin worker disconnects. Only clears the plugin flag.
+func (t *Topology) EnableVacuumByPlugin() {
+	glog.V(0).Infof("EnableVacuum (by plugin worker)")
+	t.vacuumDisabledByPlugin.Store(false)
+}
+
+// IsVacuumDisabledByPlugin returns whether the plugin monitor has disabled vacuum.
+func (t *Topology) IsVacuumDisabledByPlugin() bool {
+	return t.vacuumDisabledByPlugin.Load()
+}
+
+// SetAdminServerConnectedFunc sets an optional callback used by the vacuum
+// safety net to detect when the admin server has disconnected.
+func (t *Topology) SetAdminServerConnectedFunc(f func() bool) {
+	t.adminServerConnectedFunc = f
+}
+
+func (t *Topology) GetTopologyId() string {
+	t.topologyIdLock.RLock()
+	defer t.topologyIdLock.RUnlock()
+	return t.topologyId
+}
+
+func (t *Topology) SetTopologyId(topologyId string) {
+	t.topologyIdLock.Lock()
+	defer t.topologyIdLock.Unlock()
+	if topologyId == "" {
+		return
+	}
+	if t.topologyId == "" {
+		t.topologyId = topologyId
+		return
+	}
+	if t.topologyId != topologyId {
+		glog.Fatalf("Split-brain detected! Current TopologyId is %s, but received %s. Stopping to prevent data corruption.", t.topologyId, topologyId)
+	}
+}

weed/topology/topology_ec.go

@@ -0,0 +1,281 @@
+package topology
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+type EcShardLocations struct {
+	Collection string
+	// Use MaxShardCount (32) to support custom EC ratios
+	Locations [erasure_coding.MaxShardCount][]*DataNode
+}
+
+func (t *Topology) SyncDataNodeEcShards(shardInfos []*master_pb.VolumeEcShardInformationMessage, dn *DataNode) (newShards, deletedShards []*erasure_coding.EcVolumeInfo) {
+	// convert into in memory struct storage.VolumeInfo
+	var shards []*erasure_coding.EcVolumeInfo
+	for _, shardInfo := range shardInfos {
+		// Create EcVolumeInfo directly with optimized format
+		ecVolumeInfo := &erasure_coding.EcVolumeInfo{
+			VolumeId:    needle.VolumeId(shardInfo.Id),
+			Collection:  shardInfo.Collection,
+			ShardsInfo:  erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(shardInfo),
+			DiskType:    shardInfo.DiskType,
+			DiskId:      shardInfo.DiskId,
+			ExpireAtSec: shardInfo.ExpireAtSec,
+			FileCount:   shardInfo.FileCount,
+			DeleteCount: shardInfo.DeleteCount,
+			EncodeTsNs:  shardInfo.EncodeTsNs,
+		}
+
+		shards = append(shards, ecVolumeInfo)
+	}
+	// find out the delta volumes
+	newShards, deletedShards = dn.UpdateEcShards(shards)
+	for _, v := range newShards {
+		t.RegisterEcShards(v, dn)
+	}
+	for _, v := range deletedShards {
+		t.UnRegisterEcShards(v, dn)
+	}
+	return
+}
+
+func (t *Topology) IncrementalSyncDataNodeEcShards(newEcShards, deletedEcShards []*master_pb.VolumeEcShardInformationMessage, dn *DataNode) {
+	// convert into in memory struct storage.VolumeInfo
+	var newShards, deletedShards []*erasure_coding.EcVolumeInfo
+	for _, shardInfo := range newEcShards {
+		// Create EcVolumeInfo directly with optimized format
+		ecVolumeInfo := &erasure_coding.EcVolumeInfo{
+			VolumeId:    needle.VolumeId(shardInfo.Id),
+			Collection:  shardInfo.Collection,
+			ShardsInfo:  erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(shardInfo),
+			DiskType:    shardInfo.DiskType,
+			DiskId:      shardInfo.DiskId,
+			ExpireAtSec: shardInfo.ExpireAtSec,
+			FileCount:   shardInfo.FileCount,
+			DeleteCount: shardInfo.DeleteCount,
+			EncodeTsNs:  shardInfo.EncodeTsNs,
+		}
+
+		newShards = append(newShards, ecVolumeInfo)
+	}
+	for _, shardInfo := range deletedEcShards {
+		// Create EcVolumeInfo directly with optimized format
+		ecVolumeInfo := &erasure_coding.EcVolumeInfo{
+			VolumeId:    needle.VolumeId(shardInfo.Id),
+			Collection:  shardInfo.Collection,
+			ShardsInfo:  erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(shardInfo),
+			DiskType:    shardInfo.DiskType,
+			DiskId:      shardInfo.DiskId,
+			ExpireAtSec: shardInfo.ExpireAtSec,
+			FileCount:   shardInfo.FileCount,
+			DeleteCount: shardInfo.DeleteCount,
+			EncodeTsNs:  shardInfo.EncodeTsNs,
+		}
+
+		deletedShards = append(deletedShards, ecVolumeInfo)
+	}
+
+	dn.DeltaUpdateEcShards(newShards, deletedShards)
+
+	for _, v := range newShards {
+		t.RegisterEcShards(v, dn)
+	}
+	for _, v := range deletedShards {
+		t.UnRegisterEcShards(v, dn)
+	}
+}
+
+func NewEcShardLocations(collection string) *EcShardLocations {
+	return &EcShardLocations{
+		Collection: collection,
+	}
+}
+
+func (loc *EcShardLocations) AddShard(shardId erasure_coding.ShardId, dn *DataNode) (added bool) {
+	// Defensive bounds check to prevent panic with out-of-range shard IDs
+	if int(shardId) >= erasure_coding.MaxShardCount {
+		return false
+	}
+	dataNodes := loc.Locations[shardId]
+	for _, n := range dataNodes {
+		if n.Id() == dn.Id() {
+			return false
+		}
+	}
+	loc.Locations[shardId] = append(dataNodes, dn)
+	return true
+}
+
+func (loc *EcShardLocations) DeleteShard(shardId erasure_coding.ShardId, dn *DataNode) (deleted bool) {
+	// Defensive bounds check to prevent panic with out-of-range shard IDs
+	if int(shardId) >= erasure_coding.MaxShardCount {
+		return false
+	}
+	dataNodes := loc.Locations[shardId]
+	foundIndex := -1
+	for index, n := range dataNodes {
+		if n.Id() == dn.Id() {
+			foundIndex = index
+		}
+	}
+	if foundIndex < 0 {
+		return false
+	}
+	loc.Locations[shardId] = append(dataNodes[:foundIndex], dataNodes[foundIndex+1:]...)
+	return true
+}
+
+func (t *Topology) RegisterEcShards(ecvi *erasure_coding.EcVolumeInfo, dn *DataNode) {
+
+	// EC-only volumes (source volume deleted after encoding) must bump
+	// maxVolumeId too, or a heartbeat-rebuilt master could re-issue their id.
+	t.UpAdjustMaxVolumeId(ecvi.VolumeId)
+
+	t.ecShardMapLock.Lock()
+	defer t.ecShardMapLock.Unlock()
+
+	locations, found := t.ecShardMap[ecvi.VolumeId]
+	if !found {
+		locations = NewEcShardLocations(ecvi.Collection)
+		t.ecShardMap[ecvi.VolumeId] = locations
+	}
+	for _, shardId := range ecvi.ShardsInfo.Ids() {
+		locations.AddShard(shardId, dn)
+	}
+}
+
+func (t *Topology) UnRegisterEcShards(ecvi *erasure_coding.EcVolumeInfo, dn *DataNode) {
+	glog.Infof("removing ec shard info:%+v", ecvi)
+	t.ecShardMapLock.Lock()
+	defer t.ecShardMapLock.Unlock()
+
+	locations, found := t.ecShardMap[ecvi.VolumeId]
+	if !found {
+		return
+	}
+	for _, shardId := range ecvi.ShardsInfo.Ids() {
+		locations.DeleteShard(shardId, dn)
+	}
+}
+
+func (t *Topology) LookupEcShards(vid needle.VolumeId) (locations *EcShardLocations, found bool) {
+	t.ecShardMapLock.RLock()
+	defer t.ecShardMapLock.RUnlock()
+
+	locations, found = t.ecShardMap[vid]
+
+	return
+}
+
+// ecVolumeCounts accumulates one EC volume's data size and needle counts while
+// they are collected from every node reporting its shards.
+type ecVolumeCounts struct {
+	countedShards erasure_coding.ShardBits
+	dataSize      uint64
+	fileCount     uint64
+	deleteCount   uint64
+}
+
+// CollectionEcVolumeStats sums the disk footprint and live needle count of the
+// EC volumes in one collection, or in every collection when collectionName is
+// empty. Every shard copy counts, parity included, the way a regular volume's
+// used size counts every replica; needle counts are per volume, again as a
+// regular volume reports them. The logical size instead counts each volume's
+// data shards once, leaving out parity and over-replicated copies.
+func (t *Topology) CollectionEcVolumeStats(collectionName string) *VolumeLayoutStats {
+	ret := &VolumeLayoutStats{}
+	perVolume := make(map[needle.VolumeId]*ecVolumeCounts)
+
+	for _, c := range t.Children() {
+		for _, r := range c.(*DataCenter).Children() {
+			for _, n := range r.(*Rack).Children() {
+				for _, ecInfo := range n.(*DataNode).GetEcShards() {
+					if collectionName != "" && ecInfo.Collection != collectionName {
+						continue
+					}
+					ret.UsedSize += uint64(ecInfo.ShardsInfo.TotalSize())
+					counts, found := perVolume[ecInfo.VolumeId]
+					if !found {
+						counts = &ecVolumeCounts{}
+						perVolume[ecInfo.VolumeId] = counts
+					}
+					dataShards := ecInfo.DataShardsOrDefault()
+					for id := range erasure_coding.ShardBits(ecInfo.ShardsInfo.Bitmap()).All() {
+						if int(id) >= dataShards || counts.countedShards.Has(id) {
+							continue
+						}
+						counts.countedShards = counts.countedShards.Set(id)
+						counts.dataSize += uint64(ecInfo.ShardsInfo.Size(id))
+					}
+					// .ecx and .ecj are both volume-wide files that travel with
+					// the shards, so take the largest count any holder reports
+					// rather than summing: a node still loading .ecx reports 0
+					// and must not pin the total down, and a shard move copies
+					// the journal, so several holders can report the same
+					// tombstones. Deletes recorded only on another holder since
+					// then are missed, which errs toward reporting files that
+					// are gone rather than losing a whole volume's count.
+					if ecInfo.FileCount > counts.fileCount {
+						counts.fileCount = ecInfo.FileCount
+					}
+					if ecInfo.DeleteCount > counts.deleteCount {
+						counts.deleteCount = ecInfo.DeleteCount
+					}
+				}
+			}
+		}
+	}
+
+	// an EC volume is sealed, so it offers no room beyond what it holds
+	ret.TotalSize = ret.UsedSize
+	for _, counts := range perVolume {
+		ret.LogicalUsedSize += counts.dataSize
+		if counts.fileCount > counts.deleteCount {
+			ret.FileCount += counts.fileCount - counts.deleteCount
+		}
+	}
+	return ret
+}
+
+func (t *Topology) ListEcServersByCollection(collection string) (dataNodes []pb.ServerAddress) {
+	t.ecShardMapLock.RLock()
+	defer t.ecShardMapLock.RUnlock()
+
+	dateNodeMap := make(map[pb.ServerAddress]bool)
+	for _, ecVolumeLocation := range t.ecShardMap {
+		if ecVolumeLocation.Collection == collection {
+			for _, locations := range ecVolumeLocation.Locations {
+				for _, loc := range locations {
+					dateNodeMap[loc.ServerAddress()] = true
+				}
+			}
+		}
+	}
+
+	for k, _ := range dateNodeMap {
+		dataNodes = append(dataNodes, k)
+	}
+
+	return
+}
+
+func (t *Topology) DeleteEcCollection(collection string) {
+	t.ecShardMapLock.Lock()
+	defer t.ecShardMapLock.Unlock()
+
+	var vids []needle.VolumeId
+	for vid, ecVolumeLocation := range t.ecShardMap {
+		if ecVolumeLocation.Collection == collection {
+			vids = append(vids, vid)
+		}
+	}
+
+	for _, vid := range vids {
+		delete(t.ecShardMap, vid)
+	}
+}

weed/topology/topology_ec_max_volume_id_test.go

@@ -0,0 +1,50 @@
+package topology
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// TestRegisterEcShardsAdvancesMaxVolumeId guards against volume-id reuse: an
+// EC-only volume must advance maxVolumeId so a heartbeat-rebuilt master cannot
+// re-issue its id.
+func TestRegisterEcShardsAdvancesMaxVolumeId(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 100})
+
+	const ecVid = uint32(39848)
+	if got := topo.GetMaxVolumeId(); uint32(got) >= ecVid {
+		t.Fatalf("precondition: maxVolumeId already %d, want < %d", got, ecVid)
+	}
+
+	// Full-sync heartbeat reporting an EC-only volume.
+	msg := buildEcShardMessage(ecVid, "prefeitura", "", 0, []erasure_coding.ShardId{0, 1, 2, 3})
+	topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{msg}, dn)
+
+	if got := topo.GetMaxVolumeId(); got != needle.VolumeId(ecVid) {
+		t.Fatalf("maxVolumeId after EC registration = %d, want %d", got, ecVid)
+	}
+}
+
+// TestIncrementalSyncDataNodeEcShardsAdvancesMaxVolumeId locks the same
+// invariant on the incremental heartbeat path.
+func TestIncrementalSyncDataNodeEcShardsAdvancesMaxVolumeId(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 100})
+
+	const ecVid = uint32(39849)
+	msg := buildEcShardMessage(ecVid, "prefeitura", "", 0, []erasure_coding.ShardId{0, 1, 2, 3})
+	topo.IncrementalSyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{msg}, nil, dn)
+
+	if got := topo.GetMaxVolumeId(); got != needle.VolumeId(ecVid) {
+		t.Fatalf("maxVolumeId after incremental EC registration = %d, want %d", got, ecVid)
+	}
+}

weed/topology/topology_event_handling.go

@@ -0,0 +1,135 @@
+package topology
+
+import (
+	"math/rand/v2"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+)
+
+func (t *Topology) StartRefreshWritableVolumes(grpcDialOption grpc.DialOption, garbageThreshold float64, concurrentVacuumLimitPerVolumeServer int, growThreshold float64, preallocate int64) {
+	go func() {
+		for {
+			if t.IsLeader() {
+				freshThreshHold := time.Now().Unix() - 3*t.pulse //3 times of sleep interval
+				t.CollectDeadNodeAndFullVolumes(freshThreshHold, t.volumeSizeLimit, growThreshold)
+				t.DecayQuietVolumeSizes()
+			}
+			time.Sleep(time.Duration(float32(t.pulse*1e3)*(1+rand.Float32())) * time.Millisecond)
+		}
+	}()
+	go func(garbageThreshold float64) {
+		for {
+			if t.IsLeader() {
+				// Safety net: if vacuum was disabled by the plugin monitor but the
+				// admin server is no longer connected, automatically re-enable.
+				// This handles the case where the admin server crashes without
+				// cleanup. Does NOT override an operator's intentional disable.
+				if t.IsVacuumDisabledByPlugin() && t.adminServerConnectedFunc != nil && !t.adminServerConnectedFunc() {
+					glog.V(0).Infof("Admin server disconnected while vacuum was disabled by plugin, clearing plugin disable")
+					t.EnableVacuumByPlugin()
+				}
+				if !t.IsVacuumDisabled() {
+					t.Vacuum(grpcDialOption, garbageThreshold, concurrentVacuumLimitPerVolumeServer, 0, "", preallocate, true)
+				}
+			} else {
+				stats.MasterReplicaPlacementMismatch.Reset()
+			}
+			time.Sleep(14*time.Minute + time.Duration(120*rand.Float32())*time.Second)
+		}
+	}(garbageThreshold)
+	go func() {
+		for {
+			select {
+			case fv := <-t.chanFullVolumes:
+				t.SetVolumeCapacityFull(fv)
+			case cv := <-t.chanCrowdedVolumes:
+				t.SetVolumeCrowded(cv)
+			}
+		}
+	}()
+}
+func (t *Topology) SetVolumeCapacityFull(volumeInfo storage.VolumeInfo) bool {
+	diskType := types.ToDiskType(volumeInfo.DiskType)
+	vl := t.GetVolumeLayout(volumeInfo.Collection, volumeInfo.ReplicaPlacement, volumeInfo.Ttl, diskType)
+	if !vl.SetVolumeCapacityFull(volumeInfo.Id) {
+		return false
+	}
+
+	vl.accessLock.RLock()
+	defer vl.accessLock.RUnlock()
+
+	vidLocations, found := vl.vid2location[volumeInfo.Id]
+	if !found {
+		return false
+	}
+
+	for _, dn := range vidLocations.list {
+		if !volumeInfo.ReadOnly {
+
+			disk := dn.getOrCreateDisk(volumeInfo.DiskType)
+			disk.UpAdjustDiskUsageDelta(types.ToDiskType(volumeInfo.DiskType), &DiskUsageCounts{
+				activeVolumeCount: -1,
+			})
+
+		}
+	}
+	return true
+}
+
+func (t *Topology) SetVolumeCrowded(volumeInfo storage.VolumeInfo) {
+	diskType := types.ToDiskType(volumeInfo.DiskType)
+	vl := t.GetVolumeLayout(volumeInfo.Collection, volumeInfo.ReplicaPlacement, volumeInfo.Ttl, diskType)
+	vl.SetVolumeCrowded(volumeInfo.Id)
+}
+
+// SetDataNodeMaintenanceMode records the maintenance flag a volume server
+// reported and returns whether it changed. On a change every volume the node
+// holds is re-evaluated: while the flag is on, a volume with a replica there
+// leaves the writable list, so no writes are assigned to it and the server can
+// be drained; once the flag is off the volumes are writable again. Reads are
+// untouched, and volume growth skips the node through AvailableSpaceFor.
+func (t *Topology) SetDataNodeMaintenanceMode(dn *DataNode, on bool) (changed bool) {
+	if !dn.SetMaintenanceMode(on) {
+		return false
+	}
+	glog.V(0).Infof("volume server %s maintenance mode: %v", dn.Id(), on)
+	for _, v := range dn.GetVolumes() {
+		diskType := types.ToDiskType(v.DiskType)
+		vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
+		vl.EnsureCorrectWritables(&v)
+	}
+	return true
+}
+
+func (t *Topology) UnRegisterDataNode(dn *DataNode) {
+	dn.IsTerminating = true
+	for _, v := range dn.GetVolumes() {
+		glog.V(0).Infoln("Removing Volume", v.Id, "from the dead volume server", dn.Id())
+		diskType := types.ToDiskType(v.DiskType)
+		vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType)
+		vl.SetVolumeUnavailable(dn, v.Id)
+	}
+
+	// unregister ec shards when volume server disconnected
+	for _, s := range dn.GetEcShards() {
+		t.UnRegisterEcShards(s, dn)
+	}
+
+	negativeUsages := dn.GetDiskUsages().negative()
+	for dt, du := range negativeUsages.usages {
+		dn.UpAdjustDiskUsageDelta(dt, du)
+	}
+	dn.DeltaUpdateVolumes([]storage.VolumeInfo{}, dn.GetVolumes())
+	dn.DeltaUpdateEcShards([]*erasure_coding.EcVolumeInfo{}, dn.GetEcShards())
+	t.unregisterDataNodeAddress(dn.ServerAddress(), dn)
+	if dn.Parent() != nil {
+		dn.Parent().UnlinkChildNode(dn.Id())
+	}
+}

weed/topology/topology_heartbeat_bench_test.go

@@ -0,0 +1,52 @@
+package topology
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func benchHeartbeatMessages(count int) []*master_pb.VolumeInformationMessage {
+	messages := make([]*master_pb.VolumeInformationMessage, 0, count)
+	for i := 0; i < count; i++ {
+		messages = append(messages, &master_pb.VolumeInformationMessage{
+			Id:               uint32(i),
+			Size:             1024 * 1024,
+			Collection:       "benchcollection",
+			FileCount:        100,
+			DeleteCount:      1,
+			DeletedByteCount: 1024,
+			ReplicaPlacement: 0,
+			Version:          3,
+			CompactRevision:  1,
+			ModifiedAtSecond: 1700000000,
+		})
+	}
+	return messages
+}
+
+// A volume server re-sends its whole volume list on every heartbeat, so this is
+// the master's steady-state cost per volume server every VolumePulsePeriod.
+func benchSyncDataNodeRegistration(b *testing.B, count int) {
+	topo := NewTopology("bench", nil, 32*1024*1024*1024, 5, false)
+	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": uint32(count) * 2})
+	topo.SyncDataNodeRegistration(benchHeartbeatMessages(count), dn)
+
+	messages := benchHeartbeatMessages(count)
+
+	b.ResetTimer()
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		topo.SyncDataNodeRegistration(messages, dn)
+	}
+}
+
+func BenchmarkSyncDataNodeRegistration(b *testing.B) {
+	for _, count := range []int{1000, 100000} {
+		b.Run(fmt.Sprintf("%dVolumes", count), func(b *testing.B) {
+			benchSyncDataNodeRegistration(b, count)
+		})
+	}
+}

weed/topology/topology_info.go

@@ -0,0 +1,141 @@
+package topology
+
+import (
+	"slices"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+type TopologyInfo struct {
+	Max         int64              `json:"Max"`
+	Free        int64              `json:"Free"`
+	DataCenters []DataCenterInfo   `json:"DataCenters"`
+	Layouts     []VolumeLayoutInfo `json:"Layouts"`
+}
+
+type VolumeLayoutCollection struct {
+	Collection   string
+	VolumeLayout *VolumeLayout
+}
+
+func (t *Topology) ToInfo() (info TopologyInfo) {
+	info.Max = t.diskUsages.GetMaxVolumeCount()
+	info.Free = t.diskUsages.FreeSpace()
+	var dcs []DataCenterInfo
+	for _, c := range t.Children() {
+		dc := c.(*DataCenter)
+		dcs = append(dcs, dc.ToInfo())
+	}
+
+	slices.SortFunc(dcs, func(a, b DataCenterInfo) int {
+		return strings.Compare(string(a.Id), string(b.Id))
+	})
+
+	info.DataCenters = dcs
+	var layouts []VolumeLayoutInfo
+	for _, col := range t.collectionMap.Items() {
+		c := col.(*Collection)
+		for _, layout := range c.storageType2VolumeLayout.Items() {
+			if layout != nil {
+				tmp := layout.(*VolumeLayout).ToInfo()
+				tmp.Collection = c.Name
+				layouts = append(layouts, tmp)
+			}
+		}
+	}
+	info.Layouts = layouts
+	return
+}
+
+func (t *Topology) ListVolumeLayoutCollections() (volumeLayouts []*VolumeLayoutCollection) {
+	for _, col := range t.collectionMap.Items() {
+		for _, volumeLayout := range col.(*Collection).storageType2VolumeLayout.Items() {
+			volumeLayouts = append(volumeLayouts,
+				&VolumeLayoutCollection{col.(*Collection).Name, volumeLayout.(*VolumeLayout)},
+			)
+		}
+	}
+	return volumeLayouts
+}
+
+func (t *Topology) ToVolumeMap() interface{} {
+	m := make(map[string]interface{})
+	m["Max"] = t.diskUsages.GetMaxVolumeCount()
+	m["Free"] = t.diskUsages.FreeSpace()
+	dcs := make(map[NodeId]interface{})
+	for _, c := range t.Children() {
+		dc := c.(*DataCenter)
+		racks := make(map[NodeId]interface{})
+		for _, r := range dc.Children() {
+			rack := r.(*Rack)
+			dataNodes := make(map[NodeId]interface{})
+			for _, d := range rack.Children() {
+				dn := d.(*DataNode)
+				dnVolumes := dn.GetVolumes()
+				volumes := make([]interface{}, 0, len(dnVolumes))
+				for _, v := range dnVolumes {
+					volumes = append(volumes, v)
+				}
+				dataNodes[d.Id()] = volumes
+			}
+			racks[r.Id()] = dataNodes
+		}
+		dcs[dc.Id()] = racks
+	}
+	m["DataCenters"] = dcs
+	return m
+}
+
+// ToVolumeLocations snapshots every data node's volume set into a list of
+// per-node VolumeLocation messages. NewVids carries every volume; RemoteVids
+// repeats the remote-tier subset so clients can route reads to a local
+// replica first when one exists on any node. EC shards are flattened into
+// NewEcVids so each vid is reported once even when its shards live on
+// multiple disks of the same node.
+func (t *Topology) ToVolumeLocations() (volumeLocations []*master_pb.VolumeLocation) {
+	for _, c := range t.Children() {
+		dc := c.(*DataCenter)
+		for _, r := range dc.Children() {
+			rack := r.(*Rack)
+			for _, d := range rack.Children() {
+				dn := d.(*DataNode)
+				volumeLocation := &master_pb.VolumeLocation{
+					Url:        dn.Url(),
+					PublicUrl:  dn.PublicUrl,
+					DataCenter: dn.GetDataCenterId(),
+					GrpcPort:   uint32(dn.GrpcPort),
+				}
+
+				volumeLocation.NewVids, volumeLocation.RemoteVids, volumeLocation.ReadOnlyVids, volumeLocation.ReadOnlyCanDeleteVids = dn.AppendVolumeIds(nil, nil, nil, nil)
+
+				// A single EC volume's shards can live on multiple disks of
+				// one DataNode, so GetEcShards returns per-(vid,disk) entries.
+				// Dedupe so the snapshot carries each vid once.
+				seenEcVids := make(map[uint32]struct{})
+				for _, s := range dn.GetEcShards() {
+					vid := uint32(s.VolumeId)
+					if _, ok := seenEcVids[vid]; ok {
+						continue
+					}
+					seenEcVids[vid] = struct{}{}
+					volumeLocation.NewEcVids = append(volumeLocation.NewEcVids, vid)
+				}
+				volumeLocations = append(volumeLocations, volumeLocation)
+			}
+		}
+	}
+	return
+}
+
+func (t *Topology) ToTopologyInfo(filter VolumeFilter) *master_pb.TopologyInfo {
+	m := &master_pb.TopologyInfo{
+		Id:        string(t.Id()),
+		DiskInfos: t.diskUsages.ToDiskInfo(),
+	}
+	for _, c := range t.Children() {
+		dc := c.(*DataCenter)
+		m.DataCenterInfos = append(m.DataCenterInfos, dc.ToDataCenterInfo(filter))
+	}
+	return m
+}

weed/topology/topology_stats_test.go

@@ -0,0 +1,135 @@
+package topology
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+func TestCollectionVolumeStats(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	maxVolumeCounts := map[string]uint32{"": 25, "ssd": 12}
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
+
+	volumeMessages := []*master_pb.VolumeInformationMessage{
+		{Id: 1, Size: 1000, Collection: "", FileCount: 10, ReplicaPlacement: 0, Version: uint32(needle.GetCurrentVersion())},
+		{Id: 2, Size: 2000, Collection: "c1", FileCount: 20, ReplicaPlacement: 0, Version: uint32(needle.GetCurrentVersion())},
+		{Id: 3, Size: 3000, Collection: "c1", FileCount: 30, ReplicaPlacement: 0, Version: uint32(needle.GetCurrentVersion()), DiskType: "ssd"},
+		{Id: 4, Size: 4000, Collection: "c2", FileCount: 40, ReplicaPlacement: 1, Version: uint32(needle.GetCurrentVersion())},
+	}
+	topo.SyncDataNodeRegistration(volumeMessages, dn)
+
+	// volume 4 is replicated, so a second node holds a copy of it
+	replicaDn := rack.GetOrCreateDataNode("127.0.0.2", 34534, 0, "127.0.0.2", "", maxVolumeCounts)
+	topo.SyncDataNodeRegistration(volumeMessages[3:], replicaDn)
+
+	// VolumeLocationList.Stats only counts nodes connected for over a minute
+	dn.LastSeen = time.Now().Unix() - 61
+	replicaDn.LastSeen = dn.LastSeen
+
+	allStats := topo.CollectionVolumeStats("")
+	assert(t, "all collections used size", int(allStats.UsedSize), 14000)
+	assert(t, "all collections logical used size", int(allStats.LogicalUsedSize), 10000)
+	assert(t, "all collections file count", int(allStats.FileCount), 100)
+
+	c1Stats := topo.CollectionVolumeStats("c1")
+	assert(t, "c1 used size across disk types", int(c1Stats.UsedSize), 5000)
+	assert(t, "c1 file count", int(c1Stats.FileCount), 50)
+
+	c2Stats := topo.CollectionVolumeStats("c2")
+	// both copies of volume 4 count against the space, only one against the data
+	assert(t, "c2 used size", int(c2Stats.UsedSize), 8000)
+	assert(t, "c2 logical used size", int(c2Stats.LogicalUsedSize), 4000)
+
+	missingStats := topo.CollectionVolumeStats("no-such-collection")
+	assert(t, "missing collection used size", int(missingStats.UsedSize), 0)
+	if _, found := topo.FindCollection("no-such-collection"); found {
+		t.Errorf("stats query should not create a phantom collection")
+	}
+}
+
+// ecShardMessage builds a heartbeat message for the given shard ids, sized
+// (id+1)*sizeUnit bytes each.
+func ecShardMessage(vid uint32, collection string, sizeUnit int, fileCount, deleteCount uint64, shardIds ...erasure_coding.ShardId) *master_pb.VolumeEcShardInformationMessage {
+	shards := erasure_coding.NewShardsInfo()
+	for _, id := range shardIds {
+		shards.Set(erasure_coding.NewShardInfo(id, erasure_coding.ShardSize((int(id)+1)*sizeUnit)))
+	}
+	return &master_pb.VolumeEcShardInformationMessage{
+		Id:          vid,
+		Collection:  collection,
+		EcIndexBits: shards.Bitmap(),
+		ShardSizes:  shards.SizesInt64(),
+		FileCount:   fileCount,
+		DeleteCount: deleteCount,
+	}
+}
+
+func TestCollectionVolumeStatsWithEcVolumes(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
+	maxVolumeCounts := map[string]uint32{"": 25}
+	dn1 := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
+	dn2 := rack.GetOrCreateDataNode("127.0.0.2", 34534, 0, "127.0.0.2", "", maxVolumeCounts)
+
+	// volume 10 spans both nodes, and shard 0 has a second copy on dn2. dn2 has
+	// not finished loading its .ecx yet and reports a zero file count, and both
+	// nodes report the 5 tombstones of the journal they share.
+	topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{
+		ecShardMessage(10, "c1", 100, 50, 5, 0, 1, 2, 3, 4, 5, 6),
+		ecShardMessage(20, "", 10, 7, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13),
+	}, dn1)
+	topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{
+		ecShardMessage(10, "c1", 100, 0, 5, 0, 7, 8, 9, 10, 11, 12, 13),
+	}, dn2)
+
+	// every shard copy of volume 10, parity included, each sized (id+1)*100:
+	// 100..700 on dn1, plus a second copy of shard 0 and 800..1400 on dn2
+	c1Stats := topo.CollectionVolumeStats("c1")
+	assert(t, "c1 ec used size", int(c1Stats.UsedSize), 2800+7800)
+	// data shards 0..9 once each, so the second copy of shard 0 and the
+	// parity shards 10..13 are left out
+	assert(t, "c1 ec logical used size", int(c1Stats.LogicalUsedSize), 5500)
+	// the shared journal counts once, not once per holder: 50 - 5
+	assert(t, "c1 ec file count", int(c1Stats.FileCount), 45)
+
+	// volume 20 adds all 14 shards, each sized (id+1)*10
+	allStats := topo.CollectionVolumeStats("")
+	assert(t, "all collections ec used size", int(allStats.UsedSize), 10600+1050)
+	assert(t, "all collections ec logical used size", int(allStats.LogicalUsedSize), 5500+550)
+	assert(t, "all collections ec file count", int(allStats.FileCount), 45+7)
+
+	if _, found := topo.FindCollection("c1"); found {
+		t.Errorf("ec stats query should not create a phantom collection")
+	}
+}
+
+func TestCollectionVolumeStatsClampsDeletedOverTotals(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
+
+	// Volume 1 reports more deleted than it holds, on both counters.
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		{Id: 1, Size: 1000, FileCount: 10, DeletedByteCount: 4000, DeleteCount: 40, Version: uint32(needle.GetCurrentVersion())},
+		{Id: 2, Size: 3000, FileCount: 30, DeletedByteCount: 1000, DeleteCount: 10, Version: uint32(needle.GetCurrentVersion())},
+	}, dn)
+
+	// VolumeLocationList.Stats only counts nodes connected for over a minute
+	dn.LastSeen = time.Now().Unix() - 61
+
+	stats := topo.CollectionVolumeStats("")
+	// Volume 1 contributes nothing rather than wrapping.
+	assert(t, "used size", int(stats.UsedSize), 2000)
+	assert(t, "logical used size", int(stats.LogicalUsedSize), 2000)
+	assert(t, "file count", int(stats.FileCount), 20)
+}

weed/topology/topology_test.go

@@ -0,0 +1,866 @@
+package topology
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func TestRemoveDataCenter(t *testing.T) {
+	topo := setup(topologyLayout)
+	topo.UnlinkChildNode(NodeId("dc2"))
+	if topo.diskUsages.usages[types.HardDriveType].activeVolumeCount != 15 {
+		t.Fail()
+	}
+	topo.UnlinkChildNode(NodeId("dc3"))
+	if topo.diskUsages.usages[types.HardDriveType].activeVolumeCount != 12 {
+		t.Fail()
+	}
+}
+
+func TestHandlingVolumeServerHeartbeat(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	maxVolumeCounts := make(map[string]uint32)
+	maxVolumeCounts[""] = 25
+	maxVolumeCounts["ssd"] = 12
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
+
+	{
+		volumeCount := 7
+		var volumeMessages []*master_pb.VolumeInformationMessage
+		for k := 1; k <= volumeCount; k++ {
+			volumeMessage := &master_pb.VolumeInformationMessage{
+				Id:               uint32(k),
+				Size:             uint64(25432),
+				Collection:       "",
+				FileCount:        uint64(2343),
+				DeleteCount:      uint64(345),
+				DeletedByteCount: 34524,
+				ReadOnly:         false,
+				ReplicaPlacement: uint32(0),
+				Version:          uint32(needle.GetCurrentVersion()),
+				Ttl:              0,
+			}
+			volumeMessages = append(volumeMessages, volumeMessage)
+		}
+
+		for k := 1; k <= volumeCount; k++ {
+			volumeMessage := &master_pb.VolumeInformationMessage{
+				Id:               uint32(volumeCount + k),
+				Size:             uint64(25432),
+				Collection:       "",
+				FileCount:        uint64(2343),
+				DeleteCount:      uint64(345),
+				DeletedByteCount: 34524,
+				ReadOnly:         false,
+				ReplicaPlacement: uint32(0),
+				Version:          uint32(needle.GetCurrentVersion()),
+				Ttl:              0,
+				DiskType:         "ssd",
+			}
+			volumeMessages = append(volumeMessages, volumeMessage)
+		}
+
+		topo.SyncDataNodeRegistration(volumeMessages, dn)
+
+		usageCounts := topo.diskUsages.usages[types.HardDriveType]
+
+		assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount)
+		assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount)
+		assert(t, "ssdVolumeCount", int(topo.diskUsages.usages[types.SsdType].volumeCount), volumeCount)
+	}
+
+	{
+		volumeCount := 7 - 1
+		var volumeMessages []*master_pb.VolumeInformationMessage
+		for k := 1; k <= volumeCount; k++ {
+			volumeMessage := &master_pb.VolumeInformationMessage{
+				Id:               uint32(k),
+				Size:             uint64(30000),
+				Collection:       "",
+				FileCount:        uint64(2343),
+				DeleteCount:      uint64(345),
+				DeletedByteCount: 345240,
+				ReadOnly:         false,
+				ReplicaPlacement: uint32(0),
+				Version:          uint32(needle.GetCurrentVersion()),
+				Ttl:              0,
+			}
+			volumeMessages = append(volumeMessages, volumeMessage)
+		}
+		topo.SyncDataNodeRegistration(volumeMessages, dn)
+
+		//rp, _ := storage.NewReplicaPlacementFromString("000")
+		//layout := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL)
+		//assert(t, "writables", len(layout.writables), volumeCount)
+
+		usageCounts := topo.diskUsages.usages[types.HardDriveType]
+
+		assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount)
+		assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount)
+	}
+
+	{
+		volumeCount := 6
+		newVolumeShortMessage := &master_pb.VolumeShortInformationMessage{
+			Id:               uint32(3),
+			Collection:       "",
+			ReplicaPlacement: uint32(0),
+			Version:          uint32(needle.GetCurrentVersion()),
+			Ttl:              0,
+		}
+		topo.IncrementalSyncDataNodeRegistration(
+			[]*master_pb.VolumeShortInformationMessage{newVolumeShortMessage},
+			nil,
+			dn)
+		rp, _ := super_block.NewReplicaPlacementFromString("000")
+		layout := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
+		assert(t, "writables after repeated add", len(layout.writables), volumeCount)
+
+		usageCounts := topo.diskUsages.usages[types.HardDriveType]
+
+		assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount)
+		assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount)
+
+		topo.IncrementalSyncDataNodeRegistration(
+			nil,
+			[]*master_pb.VolumeShortInformationMessage{newVolumeShortMessage},
+			dn)
+		assert(t, "writables after deletion", len(layout.writables), volumeCount-1)
+		assert(t, "activeVolumeCount1", int(usageCounts.activeVolumeCount), volumeCount-1)
+		assert(t, "volumeCount", int(usageCounts.volumeCount), volumeCount-1)
+
+		topo.IncrementalSyncDataNodeRegistration(
+			[]*master_pb.VolumeShortInformationMessage{newVolumeShortMessage},
+			nil,
+			dn)
+
+		for vid := range layout.vid2location {
+			println("after add volume id", vid)
+		}
+		for _, vid := range layout.writables {
+			println("after add writable volume id", vid)
+		}
+
+		assert(t, "writables after add back", len(layout.writables), volumeCount)
+
+	}
+
+	topo.UnRegisterDataNode(dn)
+
+	usageCounts := topo.diskUsages.usages[types.HardDriveType]
+
+	assert(t, "activeVolumeCount2", int(usageCounts.activeVolumeCount), 0)
+
+}
+
+func TestIncrementalSyncReplacesVolumeReadOnlyState(t *testing.T) {
+	for _, tc := range []struct {
+		name         string
+		fromReadOnly bool
+		toReadOnly   bool
+	}{
+		{name: "writable to read-only", toReadOnly: true},
+		{name: "read-only to writable", fromReadOnly: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+			dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+				GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
+			volume := func(readOnly bool) *master_pb.VolumeShortInformationMessage {
+				return &master_pb.VolumeShortInformationMessage{
+					Id: 1, Collection: "c", Version: uint32(needle.GetCurrentVersion()), ReadOnly: readOnly,
+				}
+			}
+
+			oldVolume := volume(tc.fromReadOnly)
+			topo.IncrementalSyncDataNodeRegistration([]*master_pb.VolumeShortInformationMessage{oldVolume}, nil, dn)
+			topo.IncrementalSyncDataNodeRegistration(
+				[]*master_pb.VolumeShortInformationMessage{volume(tc.toReadOnly)},
+				[]*master_pb.VolumeShortInformationMessage{oldVolume}, dn)
+
+			locations := topo.Lookup("c", needle.VolumeId(1))
+			if len(locations) != 1 || locations[0] != dn {
+				t.Fatalf("lookup locations = %v, want only %v", locations, dn)
+			}
+			stored, err := dn.GetVolumesById(needle.VolumeId(1))
+			if err != nil || stored.ReadOnly != tc.toReadOnly {
+				t.Fatalf("stored volume = %+v, err = %v, want read-only %t", stored, err, tc.toReadOnly)
+			}
+			rp, _ := super_block.NewReplicaPlacementFromString("000")
+			active, _ := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType).GetWritableVolumeCount()
+			want := 0
+			if !tc.toReadOnly {
+				want = 1
+			}
+			if active != want {
+				t.Fatalf("writable count = %d, want %d", active, want)
+			}
+			if !dn.HasConsistentVolumeIndex() {
+				t.Fatal("replacement left the held and servable volume indexes inconsistent")
+			}
+		})
+	}
+}
+
+func TestIncrementalSyncRegistersMovedVolumeBeforeRemoval(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25, "ssd": 25})
+	oldVolume := &master_pb.VolumeShortInformationMessage{
+		Id: 1, Collection: "c", Version: uint32(needle.GetCurrentVersion()),
+	}
+	newVolume := &master_pb.VolumeShortInformationMessage{
+		Id: 1, Collection: "c", Version: uint32(needle.GetCurrentVersion()), DiskType: "ssd",
+	}
+	topo.IncrementalSyncDataNodeRegistration([]*master_pb.VolumeShortInformationMessage{oldVolume}, nil, dn)
+
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	oldLayout := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
+	newLayout := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.SsdType)
+	oldLayout.accessLock.Lock()
+	done := make(chan struct{})
+	go func() {
+		topo.IncrementalSyncDataNodeRegistration(
+			[]*master_pb.VolumeShortInformationMessage{newVolume},
+			[]*master_pb.VolumeShortInformationMessage{oldVolume}, dn)
+		close(done)
+	}()
+
+	deadline := time.NewTimer(time.Second)
+	defer deadline.Stop()
+	ticker := time.NewTicker(time.Millisecond)
+	defer ticker.Stop()
+	movedBeforeRemoval := false
+	for !movedBeforeRemoval {
+		select {
+		case <-deadline.C:
+			oldLayout.accessLock.Unlock()
+			<-done
+			t.Fatal("destination layout was not registered before source removal")
+		case <-ticker.C:
+			movedBeforeRemoval = len(newLayout.Lookup(needle.VolumeId(1))) == 1
+		}
+	}
+	oldLayout.accessLock.Unlock()
+	<-done
+
+	locations := topo.Lookup("c", needle.VolumeId(1))
+	if len(locations) != 1 || locations[0] != dn {
+		t.Fatalf("lookup locations = %v, want only %v", locations, dn)
+	}
+}
+
+func TestDataNodeToDataNodeInfo_IncludeEmptyDiskFromUsage(t *testing.T) {
+	dn := NewDataNode("node-1")
+	dn.Ip = "127.0.0.1"
+	dn.Port = 18080
+	dn.GrpcPort = 28080
+
+	// Simulate a node that has slot counters but no mounted volumes yet.
+	usage := dn.diskUsages.getOrCreateDisk(types.HardDriveType)
+	usage.maxVolumeCount = 8
+
+	info := dn.ToDataNodeInfo(VolumeFilter{})
+	diskInfo, found := info.DiskInfos[""]
+	if !found {
+		t.Fatalf("expected default disk entry for empty node")
+	}
+	if diskInfo.MaxVolumeCount != 8 {
+		t.Fatalf("unexpected max volume count: got=%d want=8", diskInfo.MaxVolumeCount)
+	}
+	if len(diskInfo.VolumeInfos) != 0 {
+		t.Fatalf("expected no volumes for empty disk, got=%d", len(diskInfo.VolumeInfos))
+	}
+}
+
+func assert(t *testing.T, message string, actual, expected int) {
+	if actual != expected {
+		t.Fatalf("unexpected %s: %d, expected: %d", message, actual, expected)
+	}
+}
+
+func TestAddRemoveVolume(t *testing.T) {
+
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	maxVolumeCounts := make(map[string]uint32)
+	maxVolumeCounts[""] = 25
+	maxVolumeCounts["ssd"] = 12
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
+
+	v := storage.VolumeInfo{
+		Id:               needle.VolumeId(1),
+		Size:             100,
+		Collection:       "xcollection",
+		DiskType:         "ssd",
+		FileCount:        123,
+		DeleteCount:      23,
+		DeletedByteCount: 45,
+		ReadOnly:         false,
+		Version:          needle.GetCurrentVersion(),
+		ReplicaPlacement: &super_block.ReplicaPlacement{},
+		Ttl:              needle.EMPTY_TTL,
+	}
+
+	dn.UpdateVolumes([]storage.VolumeInfo{v})
+	topo.RegisterVolumeLayout(v, dn)
+	topo.RegisterVolumeLayout(v, dn)
+
+	if _, hasCollection := topo.FindCollection(v.Collection); !hasCollection {
+		t.Errorf("collection %v should exist", v.Collection)
+	}
+
+	topo.UnRegisterVolumeLayout(v, dn)
+
+	if _, hasCollection := topo.FindCollection(v.Collection); hasCollection {
+		t.Errorf("collection %v should not exist", v.Collection)
+	}
+}
+
+func TestUnRegisterVolumeLayoutClearsReplicaPlacementMismatchMetric(t *testing.T) {
+	stats.MasterReplicaPlacementMismatch.Reset()
+	t.Cleanup(stats.MasterReplicaPlacementMismatch.Reset)
+
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	maxVolumeCounts := map[string]uint32{"": 25}
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
+
+	rp, err := super_block.NewReplicaPlacementFromString("001")
+	if err != nil {
+		t.Fatalf("NewReplicaPlacementFromString: %v", err)
+	}
+	v := storage.VolumeInfo{
+		Id:               needle.VolumeId(42),
+		Size:             100,
+		Collection:       "metrics-test",
+		ReplicaPlacement: rp,
+		Ttl:              needle.EMPTY_TTL,
+	}
+
+	dn.UpdateVolumes([]storage.VolumeInfo{v})
+	topo.RegisterVolumeLayout(v, dn)
+
+	stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1)
+	if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
+		t.Fatalf("expected 1 replica_placement_mismatch series, got %d", n)
+	}
+
+	topo.UnRegisterVolumeLayout(v, dn)
+
+	if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 0 {
+		t.Errorf("%d replica_placement_mismatch series left after volume left topology", n)
+	}
+}
+
+func TestUnRegisterVolumeLayoutKeepsReplicaPlacementMismatchMetricWhilePlacementsRemain(t *testing.T) {
+	stats.MasterReplicaPlacementMismatch.Reset()
+	t.Cleanup(stats.MasterReplicaPlacementMismatch.Reset)
+
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	maxVolumeCounts := map[string]uint32{"": 25}
+	dn1 := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
+	dn2 := rack.GetOrCreateDataNode("127.0.0.1", 34535, 0, "127.0.0.1", "", maxVolumeCounts)
+
+	rp, err := super_block.NewReplicaPlacementFromString("001")
+	if err != nil {
+		t.Fatalf("NewReplicaPlacementFromString: %v", err)
+	}
+	v := storage.VolumeInfo{
+		Id:               needle.VolumeId(42),
+		Size:             100,
+		Collection:       "metrics-test",
+		ReplicaPlacement: rp,
+		Ttl:              needle.EMPTY_TTL,
+	}
+
+	dn1.UpdateVolumes([]storage.VolumeInfo{v})
+	dn2.UpdateVolumes([]storage.VolumeInfo{v})
+	topo.RegisterVolumeLayout(v, dn1)
+	topo.RegisterVolumeLayout(v, dn2)
+
+	stats.MasterReplicaPlacementMismatch.WithLabelValues(v.Collection, v.Id.String()).Set(1)
+	if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
+		t.Fatalf("expected 1 replica_placement_mismatch series, got %d", n)
+	}
+
+	topo.UnRegisterVolumeLayout(v, dn1)
+
+	if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 1 {
+		t.Errorf("expected series to remain while %s still holds the volume, got %d", dn2.Id(), n)
+	}
+	if got := len(topo.Lookup(v.Collection, v.Id)); got != 1 {
+		t.Fatalf("expected 1 remaining placement, got %d", got)
+	}
+
+	topo.UnRegisterVolumeLayout(v, dn2)
+
+	if n := testutil.CollectAndCount(stats.MasterReplicaPlacementMismatch); n != 0 {
+		t.Errorf("%d replica_placement_mismatch series left after last placement left", n)
+	}
+}
+
+func TestVolumeReadOnlyStatusChange(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	maxVolumeCounts := make(map[string]uint32)
+	maxVolumeCounts[""] = 25
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
+
+	// Create a writable volume
+	v := storage.VolumeInfo{
+		Id:               needle.VolumeId(1),
+		Size:             100,
+		Collection:       "",
+		DiskType:         "",
+		FileCount:        10,
+		DeleteCount:      0,
+		DeletedByteCount: 0,
+		ReadOnly:         false, // Initially writable
+		Version:          needle.GetCurrentVersion(),
+		ReplicaPlacement: &super_block.ReplicaPlacement{},
+		Ttl:              needle.EMPTY_TTL,
+	}
+
+	dn.UpdateVolumes([]storage.VolumeInfo{v})
+	topo.RegisterVolumeLayout(v, dn)
+
+	// Check initial active count (should be 1 since volume is writable)
+	usageCounts := topo.diskUsages.usages[types.HardDriveType]
+	assert(t, "initial activeVolumeCount", int(usageCounts.activeVolumeCount), 1)
+	assert(t, "initial remoteVolumeCount", int(usageCounts.remoteVolumeCount), 0)
+
+	// Change volume to read-only
+	v.ReadOnly = true
+	dn.UpdateVolumes([]storage.VolumeInfo{v})
+
+	// Check active count after marking read-only (should be 0)
+	usageCounts = topo.diskUsages.usages[types.HardDriveType]
+	assert(t, "activeVolumeCount after read-only", int(usageCounts.activeVolumeCount), 0)
+
+	// Change volume back to writable
+	v.ReadOnly = false
+	dn.UpdateVolumes([]storage.VolumeInfo{v})
+
+	// Check active count after marking writable again (should be 1)
+	usageCounts = topo.diskUsages.usages[types.HardDriveType]
+	assert(t, "activeVolumeCount after writable again", int(usageCounts.activeVolumeCount), 1)
+}
+
+func TestVolumeReadOnlyAndRemoteStatusChange(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	maxVolumeCounts := make(map[string]uint32)
+	maxVolumeCounts[""] = 25
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
+
+	// Create a writable, local volume
+	v := storage.VolumeInfo{
+		Id:                needle.VolumeId(1),
+		Size:              100,
+		Collection:        "",
+		DiskType:          "",
+		FileCount:         10,
+		DeleteCount:       0,
+		DeletedByteCount:  0,
+		ReadOnly:          false, // Initially writable
+		RemoteStorageName: "",    // Initially local
+		Version:           needle.GetCurrentVersion(),
+		ReplicaPlacement:  &super_block.ReplicaPlacement{},
+		Ttl:               needle.EMPTY_TTL,
+	}
+
+	dn.UpdateVolumes([]storage.VolumeInfo{v})
+	topo.RegisterVolumeLayout(v, dn)
+
+	// Check initial counts
+	usageCounts := topo.diskUsages.usages[types.HardDriveType]
+	assert(t, "initial activeVolumeCount", int(usageCounts.activeVolumeCount), 1)
+	assert(t, "initial remoteVolumeCount", int(usageCounts.remoteVolumeCount), 0)
+
+	// Simultaneously change to read-only AND remote
+	v.ReadOnly = true
+	v.RemoteStorageName = "s3"
+	v.RemoteStorageName = "s3.default"
+	dn.UpdateVolumes([]storage.VolumeInfo{v})
+
+	// Check counts after both changes
+	usageCounts = topo.diskUsages.usages[types.HardDriveType]
+	assert(t, "activeVolumeCount after read-only+remote", int(usageCounts.activeVolumeCount), 0)
+	assert(t, "remoteVolumeCount after read-only+remote", int(usageCounts.remoteVolumeCount), 1)
+
+	// Change back to writable but keep remote
+	v.ReadOnly = false
+	dn.UpdateVolumes([]storage.VolumeInfo{v})
+
+	// Check counts - should be writable (active=1) and still remote
+	usageCounts = topo.diskUsages.usages[types.HardDriveType]
+	assert(t, "activeVolumeCount after writable+remote", int(usageCounts.activeVolumeCount), 1)
+	assert(t, "remoteVolumeCount after writable+remote", int(usageCounts.remoteVolumeCount), 1)
+
+	// Change back to local AND read-only simultaneously
+	v.ReadOnly = true
+	v.RemoteStorageName = ""
+	v.RemoteStorageName = ""
+	dn.UpdateVolumes([]storage.VolumeInfo{v})
+
+	// Check final counts
+	usageCounts = topo.diskUsages.usages[types.HardDriveType]
+	assert(t, "final activeVolumeCount", int(usageCounts.activeVolumeCount), 0)
+	assert(t, "final remoteVolumeCount", int(usageCounts.remoteVolumeCount), 0)
+}
+
+func TestListCollections(t *testing.T) {
+	rp, _ := super_block.NewReplicaPlacementFromString("002")
+
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", nil)
+
+	topo.RegisterVolumeLayout(storage.VolumeInfo{
+		Id:               needle.VolumeId(1111),
+		ReplicaPlacement: rp,
+	}, dn)
+	topo.RegisterVolumeLayout(storage.VolumeInfo{
+		Id:               needle.VolumeId(2222),
+		ReplicaPlacement: rp,
+		Collection:       "vol_collection_a",
+	}, dn)
+	topo.RegisterVolumeLayout(storage.VolumeInfo{
+		Id:               needle.VolumeId(3333),
+		ReplicaPlacement: rp,
+		Collection:       "vol_collection_b",
+	}, dn)
+
+	topo.RegisterEcShards(&erasure_coding.EcVolumeInfo{
+		VolumeId:   needle.VolumeId(4444),
+		Collection: "ec_collection_a",
+		ShardsInfo: erasure_coding.NewShardsInfo(),
+	}, dn)
+	topo.RegisterEcShards(&erasure_coding.EcVolumeInfo{
+		VolumeId:   needle.VolumeId(5555),
+		Collection: "ec_collection_b",
+		ShardsInfo: erasure_coding.NewShardsInfo(),
+	}, dn)
+
+	testCases := []struct {
+		name                 string
+		includeNormalVolumes bool
+		includeEcVolumes     bool
+		want                 []string
+	}{
+		{
+			name:                 "no volume types selected",
+			includeNormalVolumes: false,
+			includeEcVolumes:     false,
+			want:                 nil,
+		}, {
+			name:                 "normal volumes",
+			includeNormalVolumes: true,
+			includeEcVolumes:     false,
+			want:                 []string{"", "vol_collection_a", "vol_collection_b"},
+		}, {
+			name:                 "EC volumes",
+			includeNormalVolumes: false,
+			includeEcVolumes:     true,
+			want:                 []string{"ec_collection_a", "ec_collection_b"},
+		}, {
+			name:                 "normal + EC volumes",
+			includeNormalVolumes: true,
+			includeEcVolumes:     true,
+			want:                 []string{"", "ec_collection_a", "ec_collection_b", "vol_collection_a", "vol_collection_b"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := topo.ListCollections(tc.includeNormalVolumes, tc.includeEcVolumes)
+
+			if !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("got %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestDataNodeIdBasedIdentification(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+
+	maxVolumeCounts := make(map[string]uint32)
+	maxVolumeCounts[""] = 10
+
+	// Test 1: Create a DataNode with explicit id
+	dn1 := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "10.0.0.1:8080", "node-1", maxVolumeCounts)
+	if string(dn1.Id()) != "node-1" {
+		t.Errorf("expected node id 'node-1', got '%s'", dn1.Id())
+	}
+	if dn1.Ip != "10.0.0.1" {
+		t.Errorf("expected ip '10.0.0.1', got '%s'", dn1.Ip)
+	}
+
+	// Test 2: Same id with different IP should return the same DataNode (K8s pod reschedule scenario)
+	dn2 := rack.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "10.0.0.2:8080", "node-1", maxVolumeCounts)
+	if dn1 != dn2 {
+		t.Errorf("expected same DataNode for same id, got different nodes")
+	}
+	// IP should be updated to the new value
+	if dn2.Ip != "10.0.0.2" {
+		t.Errorf("expected ip to be updated to '10.0.0.2', got '%s'", dn2.Ip)
+	}
+	if dn2.PublicUrl != "10.0.0.2:8080" {
+		t.Errorf("expected publicUrl to be updated to '10.0.0.2:8080', got '%s'", dn2.PublicUrl)
+	}
+
+	// Test 3: Different id should create a new DataNode
+	dn3 := rack.GetOrCreateDataNode("10.0.0.3", 8080, 18080, "10.0.0.3:8080", "node-2", maxVolumeCounts)
+	if string(dn3.Id()) != "node-2" {
+		t.Errorf("expected node id 'node-2', got '%s'", dn3.Id())
+	}
+	if dn1 == dn3 {
+		t.Errorf("expected different DataNode for different id")
+	}
+
+	// Test 4: Empty id should fall back to ip:port (backward compatibility)
+	dn4 := rack.GetOrCreateDataNode("10.0.0.4", 8080, 18080, "10.0.0.4:8080", "", maxVolumeCounts)
+	if string(dn4.Id()) != "10.0.0.4:8080" {
+		t.Errorf("expected node id '10.0.0.4:8080' for empty id, got '%s'", dn4.Id())
+	}
+
+	// Test 5: Same ip:port with empty id should return the same DataNode
+	dn5 := rack.GetOrCreateDataNode("10.0.0.4", 8080, 18080, "10.0.0.4:8080", "", maxVolumeCounts)
+	if dn4 != dn5 {
+		t.Errorf("expected same DataNode for same ip:port with empty id")
+	}
+
+	// Verify we have 3 unique DataNodes total:
+	// - node-1 (dn1/dn2 share the same id)
+	// - node-2 (dn3)
+	// - 10.0.0.4:8080 (dn4/dn5 share the same ip:port)
+	children := rack.Children()
+	if len(children) != 3 {
+		t.Errorf("expected 3 DataNodes, got %d", len(children))
+	}
+
+	// Test 6: Transition from ip:port to explicit id
+	// First, the node exists with ip:port as id (dn4/dn5)
+	// Now the same volume server starts sending an explicit id
+	dn6 := rack.GetOrCreateDataNode("10.0.0.4", 8080, 18080, "10.0.0.4:8080", "node-4-explicit", maxVolumeCounts)
+	// Should return the same DataNode instance
+	if dn6 != dn4 {
+		t.Errorf("expected same DataNode instance during transition")
+	}
+	// But the id should now be updated to the explicit id
+	if string(dn6.Id()) != "node-4-explicit" {
+		t.Errorf("expected node id to transition to 'node-4-explicit', got '%s'", dn6.Id())
+	}
+	// The node should be re-keyed in the children map
+	if rack.FindDataNodeById("node-4-explicit") != dn6 {
+		t.Errorf("expected to find DataNode by new explicit id")
+	}
+	// Old ip:port key should no longer work
+	if rack.FindDataNodeById("10.0.0.4:8080") != nil {
+		t.Errorf("expected old ip:port id to be removed from children map")
+	}
+
+	// Still 3 unique DataNodes (node-1, node-2, node-4-explicit)
+	children = rack.Children()
+	if len(children) != 3 {
+		t.Errorf("expected 3 DataNodes after transition, got %d", len(children))
+	}
+
+	// Test 7: Prevent incorrect transition when a new node reuses ip:port of a node with explicit id
+	// Scenario: node-1 runs at 10.0.0.1:8080, dies, new node-99 starts at same ip:port
+	// The transition should NOT happen because node-1 already has an explicit id
+	dn7 := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "10.0.0.1:8080", "node-99", maxVolumeCounts)
+	// Should create a NEW DataNode, not reuse node-1
+	if dn7 == dn1 {
+		t.Errorf("expected new DataNode for node-99, got reused node-1")
+	}
+	if string(dn7.Id()) != "node-99" {
+		t.Errorf("expected node id 'node-99', got '%s'", dn7.Id())
+	}
+	// node-1 should still exist with its original id
+	if rack.FindDataNodeById("node-1") == nil {
+		t.Errorf("node-1 should still exist")
+	}
+	// Now we have 4 DataNodes
+	children = rack.Children()
+	if len(children) != 4 {
+		t.Errorf("expected 4 DataNodes, got %d", len(children))
+	}
+}
+
+func TestLookupDataNodeByAddress(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+
+	maxVolumeCounts := map[string]uint32{"": 10}
+
+	// Brand-new registration must be discoverable by both the http and
+	// grpc forms of the address.
+	dn := rack.GetOrCreateDataNode("10.1.2.3", 8080, 18080, "10.1.2.3:8080", "n1", maxVolumeCounts)
+	if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.1.2.3:8080")); got != dn {
+		t.Fatalf("lookup by http address: got %v, want %v", got, dn)
+	}
+	if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.1.2.3:8080.18080")); got != dn {
+		t.Fatalf("lookup by grpc-suffix address: got %v, want %v", got, dn)
+	}
+
+	// Unknown addresses must miss.
+	if got := topo.LookupDataNodeByAddress(pb.ServerAddress("127.0.0.1:1")); got != nil {
+		t.Fatalf("unknown address must not be found, got %v", got)
+	}
+
+	// Heartbeat from a moved pod (same id, new ip) updates the index in
+	// place: the old address is dropped and the new one resolves.
+	dnMoved := rack.GetOrCreateDataNode("10.9.9.9", 8080, 18080, "10.9.9.9:8080", "n1", maxVolumeCounts)
+	if dnMoved != dn {
+		t.Fatalf("expected same node instance after move, got different")
+	}
+	if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.1.2.3:8080")); got != nil {
+		t.Fatalf("old address must be unregistered after move, got %v", got)
+	}
+	if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.9.9.9:8080")); got != dn {
+		t.Fatalf("new address lookup: got %v, want %v", got, dn)
+	}
+
+	// UnRegisterDataNode evicts the index entry.
+	topo.UnRegisterDataNode(dn)
+	if got := topo.LookupDataNodeByAddress(pb.ServerAddress("10.9.9.9:8080")); got != nil {
+		t.Fatalf("address must be unregistered after UnRegisterDataNode, got %v", got)
+	}
+}
+
+// TestSyncDataNodeRegistrationReRegistersMissingVolume reproduces the divergence
+// where a volume present on a data node (shown by volume.list / admin UI) is
+// missing from the lookup index, which surfaces as "volume id not found" on
+// LookupVolume. SetVolumeUnavailable (used by
+// UnRegisterDataNode on a disconnect) drops the volume from the index, and the
+// reconnecting full heartbeat used to skip it because it was no longer "new" to
+// the disk map. The full heartbeat now self-heals.
+func TestSyncDataNodeRegistrationReRegistersMissingVolume(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
+
+	vid := needle.VolumeId(18994)
+	volumeMessage := &master_pb.VolumeInformationMessage{
+		Id:               uint32(vid),
+		Size:             100,
+		Collection:       "drr",
+		ReplicaPlacement: uint32(0),
+		Version:          uint32(needle.GetCurrentVersion()),
+		Ttl:              0,
+	}
+
+	// Initial full heartbeat registers the volume in the lookup index.
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{volumeMessage}, dn)
+	if got := topo.Lookup("", vid); len(got) != 1 {
+		t.Fatalf("after registration: lookup %d got %v, want 1 location", vid, got)
+	}
+
+	// Drop the volume from the index the way UnRegisterDataNode does, but leave
+	// it in the data node's disk map (the reconnecting heartbeat did not report
+	// it as new).
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := topo.GetVolumeLayout("drr", rp, needle.EMPTY_TTL, types.HardDriveType)
+	vl.SetVolumeUnavailable(dn, vid)
+
+	// The empty entry must be removed, otherwise Lookup returns a non-nil empty
+	// list that still reads as "not found".
+	if got := topo.Lookup("", vid); got != nil {
+		t.Fatalf("after SetVolumeUnavailable: expected lookup miss, got %v", got)
+	}
+	if _, err := dn.GetVolumesById(vid); err != nil {
+		t.Fatalf("volume %d should still be in the data node disk map: %v", vid, err)
+	}
+
+	// The next full heartbeat re-registers the volume even though it is not new.
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{volumeMessage}, dn)
+	if got := topo.Lookup("", vid); len(got) != 1 {
+		t.Fatalf("after self-heal: lookup %d got %v, want 1 location", vid, got)
+	}
+}
+
+// TestSetVolumeAvailableRepairsMissingVolume covers the vacuum-commit variant of
+// the same divergence. A disconnect during a long vacuum can drop a
+// single-replica volume from the lookup index while it stays on the node; the
+// commit then calls SetVolumeAvailable on it. That used to dereference a nil
+// location and panic; it now re-creates the entry and repairs the split.
+func TestSetVolumeAvailableRepairsMissingVolume(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
+
+	vid := needle.VolumeId(2640)
+	volumeMessage := &master_pb.VolumeInformationMessage{
+		Id:               uint32(vid),
+		Size:             100,
+		Collection:       "drr",
+		ReplicaPlacement: uint32(0),
+		Version:          uint32(needle.GetCurrentVersion()),
+		Ttl:              0,
+	}
+
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{volumeMessage}, dn)
+
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := topo.GetVolumeLayout("drr", rp, needle.EMPTY_TTL, types.HardDriveType)
+
+	// Disconnect drops the volume from the index but leaves it on the node.
+	vl.SetVolumeUnavailable(dn, vid)
+	if got := topo.Lookup("", vid); got != nil {
+		t.Fatalf("after SetVolumeUnavailable: expected lookup miss, got %v", got)
+	}
+	if _, err := dn.GetVolumesById(vid); err != nil {
+		t.Fatalf("volume %d should still be in the data node disk map: %v", vid, err)
+	}
+
+	// The vacuum commit re-marks the volume available; it must re-register it.
+	vl.SetVolumeAvailable(dn, vid, false, false)
+	if got := topo.Lookup("", vid); len(got) != 1 {
+		t.Fatalf("after SetVolumeAvailable: lookup %d got %v, want 1 location", vid, got)
+	}
+	// Size tracking must be seeded too, or assigns go uncounted until the next
+	// heartbeat and the volume can overfill.
+	vl.accessLock.RLock()
+	_, tracked := vl.sizeTracking[vid]
+	vl.accessLock.RUnlock()
+	if !tracked {
+		t.Fatalf("after SetVolumeAvailable: size tracking for %d not seeded", vid)
+	}
+}

weed/topology/topology_vacuum.go

@@ -0,0 +1,371 @@
+package topology
+
+import (
+	"context"
+	"io"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func (t *Topology) batchVacuumVolumeCheck(grpcDialOption grpc.DialOption, vid needle.VolumeId,
+	locationlist *VolumeLocationList, garbageThreshold float64) (*VolumeLocationList, bool) {
+	ch := make(chan int, locationlist.Length())
+	errCount := int32(0)
+	for index, dn := range locationlist.list {
+		go func(index int, url pb.ServerAddress, vid needle.VolumeId) {
+			err := operation.WithVolumeServerClient(false, url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+				resp, err := volumeServerClient.VacuumVolumeCheck(context.Background(), &volume_server_pb.VacuumVolumeCheckRequest{
+					VolumeId: uint32(vid),
+				})
+				if err != nil {
+					atomic.AddInt32(&errCount, 1)
+					ch <- -1
+					return err
+				}
+				if resp.GarbageRatio >= garbageThreshold {
+					ch <- index
+				} else {
+					ch <- -1
+				}
+				return nil
+			})
+			if err != nil {
+				glog.V(0).Infof("Checking vacuuming %d on %s: %v", vid, url, err)
+			}
+		}(index, dn.ServerAddress(), vid)
+	}
+	vacuumLocationList := NewVolumeLocationList()
+
+	waitTimeout := time.NewTimer(time.Minute * time.Duration(t.volumeSizeLimit/1024/1024/1000+1))
+	defer waitTimeout.Stop()
+
+	for range locationlist.list {
+		select {
+		case index := <-ch:
+			if index != -1 {
+				vacuumLocationList.list = append(vacuumLocationList.list, locationlist.list[index])
+			}
+		case <-waitTimeout.C:
+			return vacuumLocationList, false
+		}
+	}
+	return vacuumLocationList, errCount == 0 && len(vacuumLocationList.list) > 0
+}
+
+func (t *Topology) batchVacuumVolumeCompact(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId,
+	locationlist *VolumeLocationList, preallocate int64) bool {
+	vl.DrainAndRemoveFromWritable(vid)
+
+	ch := make(chan bool, locationlist.Length())
+	for index, dn := range locationlist.list {
+		go func(index int, url pb.ServerAddress, vid needle.VolumeId) {
+			glog.V(0).Infoln(index, "Start vacuuming", vid, "on", url)
+			err := operation.WithVolumeServerClient(true, url, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+				stream, err := volumeServerClient.VacuumVolumeCompact(context.Background(), &volume_server_pb.VacuumVolumeCompactRequest{
+					VolumeId:    uint32(vid),
+					Preallocate: preallocate,
+				})
+				if err != nil {
+					return err
+				}
+
+				for {
+					resp, recvErr := stream.Recv()
+					if recvErr != nil {
+						if recvErr == io.EOF {
+							break
+						} else {
+							return recvErr
+						}
+					}
+					glog.V(0).Infof("%d vacuum %d on %s processed %d bytes, loadAvg %.02f%%",
+						index, vid, url, resp.ProcessedBytes, resp.LoadAvg_1M*100)
+				}
+				return nil
+			})
+			if err != nil {
+				glog.Errorf("Error when vacuuming %d on %s: %v", vid, url, err)
+				ch <- false
+			} else {
+				glog.V(0).Infof("Complete vacuuming %d on %s", vid, url)
+				ch <- true
+			}
+		}(index, dn.ServerAddress(), vid)
+	}
+	isVacuumSuccess := true
+
+	waitTimeout := time.NewTimer(3 * time.Minute * time.Duration(t.volumeSizeLimit/1024/1024/1000+1))
+	defer waitTimeout.Stop()
+
+	for range locationlist.list {
+		select {
+		case canCommit := <-ch:
+			isVacuumSuccess = isVacuumSuccess && canCommit
+		case <-waitTimeout.C:
+			return false
+		}
+	}
+	return isVacuumSuccess
+}
+
+func (t *Topology) batchVacuumVolumeCommit(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, vacuumLocationList, locationList *VolumeLocationList) bool {
+	isCommitSuccess := true
+	isReadOnly := false
+	isFullCapacity := false
+	for _, dn := range vacuumLocationList.list {
+		glog.V(0).Infoln("Start Committing vacuum", vid, "on", dn.Url())
+		err := operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+			resp, err := volumeServerClient.VacuumVolumeCommit(context.Background(), &volume_server_pb.VacuumVolumeCommitRequest{
+				VolumeId: uint32(vid),
+			})
+			if resp != nil {
+				if resp.IsReadOnly {
+					isReadOnly = true
+				}
+				if resp.VolumeSize > t.volumeSizeLimit {
+					isFullCapacity = true
+				}
+			}
+			return err
+		})
+		if err != nil {
+			glog.Errorf("Error when committing vacuum %d on %s: %v", vid, dn.Url(), err)
+			isCommitSuccess = false
+		} else {
+			glog.V(0).Infof("Complete Committing vacuum %d on %s", vid, dn.Url())
+		}
+	}
+
+	//we should check the status of all replicas
+	if len(locationList.list) > len(vacuumLocationList.list) {
+		for _, dn := range locationList.list {
+			isFound := false
+			for _, dnVaccum := range vacuumLocationList.list {
+				if dn.id == dnVaccum.id {
+					isFound = true
+					break
+				}
+			}
+			if !isFound {
+				err := operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+					resp, err := volumeServerClient.VolumeStatus(context.Background(), &volume_server_pb.VolumeStatusRequest{
+						VolumeId: uint32(vid),
+					})
+					if resp != nil {
+						if resp.IsReadOnly {
+							isReadOnly = true
+						}
+						if resp.VolumeSize > t.volumeSizeLimit {
+							isFullCapacity = true
+						}
+					}
+					return err
+				})
+				if err != nil {
+					glog.Errorf("Error when checking volume %d status on %s: %v", vid, dn.Url(), err)
+					//we mark volume read-only, since the volume state is unknown
+					isReadOnly = true
+				}
+			}
+		}
+	}
+
+	if isCommitSuccess {
+
+		//record vacuum time of volume
+		vl.accessLock.Lock()
+		vl.vacuumedVolumes[vid] = time.Now()
+		vl.accessLock.Unlock()
+
+		for _, dn := range vacuumLocationList.list {
+			vl.SetVolumeAvailable(dn, vid, isReadOnly, isFullCapacity)
+		}
+	}
+	return isCommitSuccess
+}
+
+func (t *Topology) batchVacuumVolumeCleanup(grpcDialOption grpc.DialOption, vl *VolumeLayout, vid needle.VolumeId, locationlist *VolumeLocationList) {
+	for _, dn := range locationlist.list {
+		glog.V(0).Infoln("Start cleaning up", vid, "on", dn.Url())
+		err := operation.WithVolumeServerClient(false, dn.ServerAddress(), grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+			_, err := volumeServerClient.VacuumVolumeCleanup(context.Background(), &volume_server_pb.VacuumVolumeCleanupRequest{
+				VolumeId: uint32(vid),
+			})
+			return err
+		})
+		if err != nil {
+			glog.Errorf("Error when cleaning up vacuum %d on %s: %v", vid, dn.Url(), err)
+		} else {
+			glog.V(0).Infof("Complete cleaning up vacuum %d on %s", vid, dn.Url())
+		}
+	}
+}
+
+func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, maxParallelVacuumPerServer int, volumeId uint32, collection string, preallocate int64, automatic bool) {
+
+	// if there is vacuum going on, return immediately
+	swapped := atomic.CompareAndSwapInt64(&t.vacuumLockCounter, 0, 1)
+	if !swapped {
+		glog.V(0).Infof("Vacuum is already running")
+		return
+	}
+	defer atomic.StoreInt64(&t.vacuumLockCounter, 0)
+
+	// now only one vacuum process going on
+
+	glog.V(1).Infof("Start vacuum on demand with threshold: %f collection: %s volumeId: %d",
+		garbageThreshold, collection, volumeId)
+	for _, col := range t.collectionMap.Items() {
+		c := col.(*Collection)
+		if collection != "" && collection != c.Name {
+			continue
+		}
+		for _, vl := range c.storageType2VolumeLayout.Items() {
+			if vl != nil {
+				volumeLayout := vl.(*VolumeLayout)
+				if volumeId > 0 {
+					vid := needle.VolumeId(volumeId)
+					volumeLayout.accessLock.RLock()
+					locationList, ok := volumeLayout.vid2location[vid]
+					if ok {
+						locationList = locationList.Copy()
+					}
+					volumeLayout.accessLock.RUnlock()
+					if ok {
+						t.vacuumOneVolumeId(grpcDialOption, volumeLayout, c, garbageThreshold, locationList, vid, preallocate, false)
+					}
+				} else {
+					t.vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, maxParallelVacuumPerServer, preallocate, automatic)
+				}
+			}
+			if automatic && t.IsVacuumDisabled() {
+				break
+			}
+		}
+		if automatic && t.IsVacuumDisabled() {
+			glog.V(0).Infof("Vacuum is disabled")
+			break
+		}
+	}
+}
+
+func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, maxParallelVacuumPerServer int, preallocate int64, automatic bool) {
+
+	volumeLayout.accessLock.RLock()
+	todoVolumeMap := make(map[needle.VolumeId]*VolumeLocationList)
+	for vid, locationList := range volumeLayout.vid2location {
+		todoVolumeMap[vid] = locationList.Copy()
+	}
+	volumeLayout.accessLock.RUnlock()
+
+	// limiter for each volume server
+	limiter := make(map[NodeId]int)
+	var limiterLock sync.Mutex
+	for _, locationList := range todoVolumeMap {
+		for _, dn := range locationList.list {
+			if _, ok := limiter[dn.Id()]; !ok {
+				limiter[dn.Id()] = maxParallelVacuumPerServer
+			}
+		}
+	}
+
+	executor := util.NewLimitedConcurrentExecutor(100)
+
+	var wg sync.WaitGroup
+
+	for len(todoVolumeMap) > 0 {
+		pendingVolumeMap := make(map[needle.VolumeId]*VolumeLocationList)
+		for vid, locationList := range todoVolumeMap {
+			hasEnoughQuota := true
+			for _, dn := range locationList.list {
+				limiterLock.Lock()
+				quota := limiter[dn.Id()]
+				limiterLock.Unlock()
+				if quota <= 0 {
+					hasEnoughQuota = false
+					break
+				}
+			}
+			if !hasEnoughQuota {
+				pendingVolumeMap[vid] = locationList
+				continue
+			}
+
+			// debit the quota
+			for _, dn := range locationList.list {
+				limiterLock.Lock()
+				limiter[dn.Id()]--
+				limiterLock.Unlock()
+			}
+
+			wg.Add(1)
+			executor.Execute(func() {
+				defer wg.Done()
+				t.vacuumOneVolumeId(grpcDialOption, volumeLayout, c, garbageThreshold, locationList, vid, preallocate, true)
+				// credit the quota
+				for _, dn := range locationList.list {
+					limiterLock.Lock()
+					limiter[dn.Id()]++
+					limiterLock.Unlock()
+				}
+			})
+			if automatic && t.IsVacuumDisabled() {
+				break
+			}
+		}
+		if automatic && t.IsVacuumDisabled() {
+			break
+		}
+		if len(todoVolumeMap) == len(pendingVolumeMap) {
+			time.Sleep(10 * time.Second)
+		}
+		todoVolumeMap = pendingVolumeMap
+	}
+
+	wg.Wait()
+
+}
+
+// skipReadOnly is set by the background scan and all-volumes sweep, where a
+// read-only flag usually means an unhealthy disk. An explicit volumeId clears
+// it so a benignly read-only (full/oversized) volume can be reclaimed.
+func (t *Topology) vacuumOneVolumeId(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, locationList *VolumeLocationList, vid needle.VolumeId, preallocate int64, skipReadOnly bool) {
+	volumeLayout.accessLock.RLock()
+	isReadOnly := volumeLayout.vid2location[vid].AnyReadOnly()
+	isEnoughCopies := volumeLayout.enoughCopies(vid)
+	volumeLayout.accessLock.RUnlock()
+
+	if isReadOnly {
+		if skipReadOnly {
+			return
+		}
+		glog.V(0).Infof("vacuuming read-only volume %d on explicit request", vid)
+	}
+	if !isEnoughCopies {
+		glog.Warningf("skip vacuuming: not enough copies for volume:%d", vid)
+		return
+	}
+
+	glog.V(1).Infof("check vacuum on collection:%s volume:%d", c.Name, vid)
+	if vacuumLocationList, needVacuum := t.batchVacuumVolumeCheck(
+		grpcDialOption, vid, locationList, garbageThreshold); needVacuum {
+		if t.batchVacuumVolumeCompact(grpcDialOption, volumeLayout, vid, vacuumLocationList, preallocate) {
+			t.batchVacuumVolumeCommit(grpcDialOption, volumeLayout, vid, vacuumLocationList, locationList)
+		} else {
+			t.batchVacuumVolumeCleanup(grpcDialOption, volumeLayout, vid, vacuumLocationList)
+		}
+	}
+}

weed/topology/volume_digest_test.go

@@ -0,0 +1,485 @@
+package topology
+
+import (
+	"testing"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func digestTestNode(t *testing.T) (*Topology, *DataNode) {
+	t.Helper()
+	topo := NewTopology("digest", nil, 32*1024*1024*1024, 5, false)
+	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 1000})
+	return topo, dn
+}
+
+func digestTestVolume(id uint32) *master_pb.VolumeInformationMessage {
+	return &master_pb.VolumeInformationMessage{
+		Id:               id,
+		Size:             1024 * 1024,
+		Collection:       "c",
+		FileCount:        10,
+		DeleteCount:      1,
+		DeletedByteCount: 128,
+		ReplicaPlacement: 0,
+		Version:          3,
+		CompactRevision:  1,
+		ModifiedAtSecond: 1700000000,
+	}
+}
+
+func TestVolumeDigestIsStableAcrossRepeatedHeartbeats(t *testing.T) {
+	topo, dn := digestTestNode(t)
+	volumes := []*master_pb.VolumeInformationMessage{digestTestVolume(1), digestTestVolume(2), digestTestVolume(3)}
+
+	topo.SyncDataNodeRegistration(volumes, dn)
+	first := dn.VolumeDigest()
+	if first == 0 {
+		t.Fatal("expected a non-zero digest for a node holding volumes")
+	}
+	for i := 0; i < 3; i++ {
+		topo.SyncDataNodeRegistration(volumes, dn)
+		if got := dn.VolumeDigest(); got != first {
+			t.Fatalf("heartbeat %d changed the digest with no change to report: %d != %d", i, got, first)
+		}
+	}
+}
+
+func TestVolumeDigestIsIndependentOfReportOrder(t *testing.T) {
+	topoA, dnA := digestTestNode(t)
+	topoA.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		digestTestVolume(1), digestTestVolume(2), digestTestVolume(3),
+	}, dnA)
+
+	topoB, dnB := digestTestNode(t)
+	topoB.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		digestTestVolume(3), digestTestVolume(1), digestTestVolume(2),
+	}, dnB)
+
+	if dnA.VolumeDigest() != dnB.VolumeDigest() {
+		t.Errorf("digest depends on report order: %d != %d", dnA.VolumeDigest(), dnB.VolumeDigest())
+	}
+}
+
+// Every field of VolumeInformationMessage has to reach the digest: one the hash
+// skips is a change the master would never be told about. Enumerated from the
+// message rather than listed here, so a field added later cannot quietly fall
+// outside the digest while this still passes.
+// Fields a server reports that the master deliberately does not keep, so a
+// change to one alters nothing it holds and the digest is right not to move.
+var notRetainedByTheMaster = map[string]string{
+	"remote_storage_key": "unique per volume, and the master decides nothing from it",
+}
+
+func TestVolumeDigestTracksEveryReportedField(t *testing.T) {
+	base := digestTestVolume(1)
+	baseInfo, err := storage.NewVolumeInfo(base)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fields := base.ProtoReflect().Descriptor().Fields()
+	for i := 0; i < fields.Len(); i++ {
+		fd := fields.Get(i)
+		t.Run(string(fd.Name()), func(t *testing.T) {
+			why, skipped := notRetainedByTheMaster[string(fd.Name())]
+			candidates := distinctValuesFor(t, fd, base.ProtoReflect().Get(fd))
+			for _, candidate := range candidates {
+				changed := proto.Clone(base).(*master_pb.VolumeInformationMessage)
+				changed.ProtoReflect().Set(fd, candidate)
+				changedInfo, err := storage.NewVolumeInfo(changed)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if baseInfo.ReportHash() != changedInfo.ReportHash() {
+					if skipped {
+						t.Errorf("%s moves the digest, but is listed as not retained by the master (%s)", fd.Name(), why)
+					}
+					return
+				}
+			}
+			if skipped {
+				return
+			}
+			t.Errorf("no change to %s moves the digest, so the master would never be told about one", fd.Name())
+		})
+	}
+}
+
+// distinctValuesFor offers values that differ from current. Several, because
+// some fields are narrowed or normalised on the way into VolumeInfo and the
+// smallest change to the wire value can land back on the stored one.
+func distinctValuesFor(t *testing.T, fd protoreflect.FieldDescriptor, current protoreflect.Value) []protoreflect.Value {
+	t.Helper()
+	switch fd.Kind() {
+	case protoreflect.BoolKind:
+		return []protoreflect.Value{protoreflect.ValueOfBool(!current.Bool())}
+	case protoreflect.Uint32Kind:
+		return []protoreflect.Value{
+			protoreflect.ValueOfUint32(uint32(current.Uint()) + 1),
+			protoreflect.ValueOfUint32(uint32(current.Uint()) + 1<<8),
+			protoreflect.ValueOfUint32(uint32(current.Uint()) + 1<<16),
+		}
+	case protoreflect.Uint64Kind:
+		return []protoreflect.Value{protoreflect.ValueOfUint64(current.Uint() + 1)}
+	case protoreflect.Int64Kind:
+		return []protoreflect.Value{protoreflect.ValueOfInt64(current.Int() + 1)}
+	case protoreflect.StringKind:
+		return []protoreflect.Value{protoreflect.ValueOfString(current.String() + "x")}
+	}
+	t.Fatalf("field %s has kind %s, which this test does not know how to vary", fd.Name(), fd.Kind())
+	return nil
+}
+
+func TestVolumeDigestFollowsVolumeChanges(t *testing.T) {
+	topo, dn := digestTestNode(t)
+	volumes := []*master_pb.VolumeInformationMessage{digestTestVolume(1), digestTestVolume(2)}
+	topo.SyncDataNodeRegistration(volumes, dn)
+	original := dn.VolumeDigest()
+
+	grown := []*master_pb.VolumeInformationMessage{digestTestVolume(1), digestTestVolume(2)}
+	grown[1].Size += 4096
+	topo.SyncDataNodeRegistration(grown, dn)
+	if dn.VolumeDigest() == original {
+		t.Error("a volume that grew left the digest unchanged")
+	}
+
+	topo.SyncDataNodeRegistration(volumes, dn)
+	if dn.VolumeDigest() != original {
+		t.Error("reverting a volume did not restore the digest")
+	}
+
+	topo.SyncDataNodeRegistration(volumes[:1], dn)
+	if dn.VolumeDigest() == original {
+		t.Error("dropping a volume left the digest unchanged")
+	}
+
+	topo.SyncDataNodeRegistration(volumes, dn)
+	if dn.VolumeDigest() != original {
+		t.Error("restoring a dropped volume did not restore the digest")
+	}
+}
+
+func TestVolumeDigestEmptiesWithTheNode(t *testing.T) {
+	topo, dn := digestTestNode(t)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		digestTestVolume(1), digestTestVolume(2),
+	}, dn)
+
+	topo.SyncDataNodeRegistration(nil, dn)
+	if got := dn.VolumeDigest(); got != 0 {
+		t.Errorf("expected an empty node to digest to 0, got %d", got)
+	}
+}
+
+func TestVolumeDigestFollowsDeltaRegistration(t *testing.T) {
+	topo, dn := digestTestNode(t)
+	full := []*master_pb.VolumeInformationMessage{digestTestVolume(1), digestTestVolume(2)}
+	topo.SyncDataNodeRegistration(full, dn)
+	both := dn.VolumeDigest()
+
+	topo.IncrementalSyncDataNodeRegistration(nil, []*master_pb.VolumeShortInformationMessage{{Id: 2}}, dn)
+	if dn.VolumeDigest() == both {
+		t.Error("unmounting a volume left the digest unchanged")
+	}
+
+	topo.SyncDataNodeRegistration(full, dn)
+	if dn.VolumeDigest() != both {
+		t.Error("a full heartbeat did not restore the digest after an unmount")
+	}
+}
+
+// The point of the digest is not to detect that volumes changed -- in any live
+// cluster some always have. It is to confirm that after applying the changes a
+// heartbeat did carry, the master holds what the volume server holds. So a
+// heartbeat reporting only the volumes that moved must still reconcile.
+func TestVolumeDigestMatchesAfterApplyingOnlyChangedVolumes(t *testing.T) {
+	const total = 50
+	full := make([]*master_pb.VolumeInformationMessage, 0, total)
+	for i := 1; i <= total; i++ {
+		v := digestTestVolume(uint32(i))
+		v.ReadOnly = i > 5 // only the first few are writable, as in a tiered cluster
+		full = append(full, v)
+	}
+
+	topo, dn := digestTestNode(t)
+	topo.SyncDataNodeRegistration(full, dn)
+
+	// Three writable volumes take writes between two heartbeats.
+	changed := make([]storage.VolumeInfo, 0, 3)
+	for _, v := range full[:3] {
+		v.Size += 4096
+		v.FileCount++
+		v.ModifiedAtSecond += 5
+		vi, err := storage.NewVolumeInfo(v)
+		if err != nil {
+			t.Fatal(err)
+		}
+		changed = append(changed, vi)
+	}
+
+	// What the volume server would now report for its whole set.
+	reference, referenceNode := digestTestNode(t)
+	reference.SyncDataNodeRegistration(full, referenceNode)
+	want := referenceNode.VolumeDigest()
+
+	if dn.VolumeDigest() == want {
+		t.Fatal("expected the master to be behind before the changes are applied")
+	}
+
+	// The heartbeat carries three volumes, not fifty.
+	dn.DeltaUpdateVolumes(changed, nil)
+
+	if got := dn.VolumeDigest(); got != want {
+		t.Errorf("digest still disagrees after applying the reported changes: %d != %d", got, want)
+	}
+}
+
+// A volume that disappears without a delta is exactly what the full list exists
+// to catch, and is the case the digest has to keep catching.
+func TestVolumeDigestCatchesASilentlyLostVolume(t *testing.T) {
+	full := []*master_pb.VolumeInformationMessage{
+		digestTestVolume(1), digestTestVolume(2), digestTestVolume(3),
+	}
+
+	topo, dn := digestTestNode(t)
+	topo.SyncDataNodeRegistration(full, dn)
+
+	// The volume server no longer has volume 2 and never got to say so.
+	reference, referenceNode := digestTestNode(t)
+	reference.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{full[0], full[2]}, referenceNode)
+
+	if dn.VolumeDigest() == referenceNode.VolumeDigest() {
+		t.Error("a volume lost without a delta went undetected, which is what the full list is for")
+	}
+}
+
+// The disk map and the lookup index are maintained separately, and a disconnect
+// racing a reconnect has been seen to drop a volume from the lookup index while
+// leaving it on the node. The volume server's report is identical either way, so
+// the heartbeat digest cannot see it and the master has to notice on its own.
+func TestVolumeIndexDigestSeesLookupDivergence(t *testing.T) {
+	topo, dn := digestTestNode(t)
+	v := digestTestVolume(1)
+	v.Collection = "drr"
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{v}, dn)
+
+	if !dn.HasConsistentVolumeIndex() {
+		t.Fatal("a freshly registered node should have a consistent index")
+	}
+	reported := dn.VolumeDigest()
+
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := topo.GetVolumeLayout("drr", rp, needle.EMPTY_TTL, types.HardDriveType)
+	vl.SetVolumeUnavailable(dn, needle.VolumeId(1))
+
+	if got := topo.Lookup("drr", needle.VolumeId(1)); got != nil {
+		t.Fatalf("expected the volume to have become unservable, got %v", got)
+	}
+	if _, err := dn.GetVolumesById(needle.VolumeId(1)); err != nil {
+		t.Fatalf("the volume should still be on the node: %v", err)
+	}
+	if dn.VolumeDigest() != reported {
+		t.Error("the reported digest should not move: the volume server sees no change")
+	}
+	if dn.HasConsistentVolumeIndex() {
+		t.Error("a volume held but not servable left the index digests agreeing, so nothing would repair it")
+	}
+
+	// The full heartbeat self-heal puts it back.
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{v}, dn)
+	if !dn.HasConsistentVolumeIndex() {
+		t.Error("the self-heal did not restore index consistency")
+	}
+}
+
+func TestVolumeIndexDigestFollowsNodeLifecycle(t *testing.T) {
+	topo, dn := digestTestNode(t)
+	full := []*master_pb.VolumeInformationMessage{digestTestVolume(1), digestTestVolume(2), digestTestVolume(3)}
+	topo.SyncDataNodeRegistration(full, dn)
+	if !dn.HasConsistentVolumeIndex() {
+		t.Fatal("registration left the indexes disagreeing")
+	}
+
+	topo.SyncDataNodeRegistration(full[:2], dn)
+	if !dn.HasConsistentVolumeIndex() {
+		t.Error("dropping a volume left the indexes disagreeing")
+	}
+
+	topo.IncrementalSyncDataNodeRegistration(
+		[]*master_pb.VolumeShortInformationMessage{{Id: 9}}, nil, dn)
+	if !dn.HasConsistentVolumeIndex() {
+		t.Error("a mount delta left the indexes disagreeing")
+	}
+
+	topo.IncrementalSyncDataNodeRegistration(
+		nil, []*master_pb.VolumeShortInformationMessage{{Id: 9}}, dn)
+	if !dn.HasConsistentVolumeIndex() {
+		t.Error("an unmount delta left the indexes disagreeing")
+	}
+
+	topo.UnRegisterDataNode(dn)
+	held, servable := dn.VolumeIndexDigests()
+	if held != 0 || servable != 0 {
+		t.Errorf("an unregistered node should hold nothing: held=%d servable=%d", held, servable)
+	}
+}
+
+// A volume id mounted on two disks of one server is reported twice with
+// different disk ids, but the master keys volumes by id alone, so it keeps only
+// one copy and its digest can never equal the server's. Resending the full list
+// cannot fix that, so the node has to be excluded from digest comparison
+// entirely rather than resend forever.
+func TestVolumeDigestRefusesDuplicateVolumeIds(t *testing.T) {
+	topo, dn := digestTestNode(t)
+
+	first := digestTestVolume(1)
+	second := digestTestVolume(1)
+	second.DiskId = 1
+	second.Size = first.Size * 2
+
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{first, second}, dn)
+	if !dn.HasDuplicateVolumeIds() {
+		t.Fatal("a volume id reported twice went unnoticed, so the digest would be trusted and never reconcile")
+	}
+
+	firstInfo, err := storage.NewVolumeInfo(first)
+	if err != nil {
+		t.Fatal(err)
+	}
+	secondInfo, err := storage.NewVolumeInfo(second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if dn.VolumeDigest() == firstInfo.ReportHash()^secondInfo.ReportHash() {
+		t.Error("expected the master to be unable to represent both copies; if it now can, the guard is no longer needed")
+	}
+
+	// Once the stale twin is gone the node is comparable again.
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{first}, dn)
+	if dn.HasDuplicateVolumeIds() {
+		t.Error("the node stayed marked as duplicated after reporting a clean list")
+	}
+	if dn.VolumeDigest() != firstInfo.ReportHash() {
+		t.Error("digest did not settle on the surviving copy")
+	}
+}
+
+// Two volume servers can hold one address. GetOrCreateDataNode keys on the id a
+// server reports and deliberately refuses to merge a new id onto an address an
+// older node still claims, while the lookup list keys on address alone -- so
+// registering the second server displaces the first from the lookup entry
+// without either node being told. The digest has to follow the entry, not the
+// node that was passed in.
+func addressSharingNodes(t *testing.T) (*Topology, *DataNode, *DataNode) {
+	t.Helper()
+	topo := NewTopology("digest", nil, 32*1024*1024*1024, 5, false)
+	rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
+	counts := map[string]uint32{"": 1000}
+	old := rack.GetOrCreateDataNode("10.1.2.3", 8080, 18080, "", "n1", counts)
+	fresh := rack.GetOrCreateDataNode("10.1.2.3", 8080, 18080, "", "n2", counts)
+	if old == fresh {
+		t.Skip("address reuse no longer produces two nodes")
+	}
+	return topo, old, fresh
+}
+
+func TestVolumeIndexDigestFollowsDisplacedLookupEntry(t *testing.T) {
+	topo, old, fresh := addressSharingNodes(t)
+	v := digestTestVolume(1)
+
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{v}, old)
+	if !old.HasConsistentVolumeIndex() {
+		t.Fatal("the first node should be consistent before it is displaced")
+	}
+
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{v}, fresh)
+
+	servable := map[*DataNode]bool{}
+	for _, dn := range topo.Lookup("", needle.VolumeId(1)) {
+		servable[dn] = true
+	}
+	if servable[old] || !servable[fresh] {
+		t.Fatalf("expected the lookup entry to move to the new node, got old=%v fresh=%v",
+			servable[old], servable[fresh])
+	}
+
+	if old.HasConsistentVolumeIndex() {
+		t.Error("the displaced node still holds the volume and can no longer serve it, so its index must read as inconsistent")
+	}
+	if !fresh.HasConsistentVolumeIndex() {
+		t.Error("the node the lookup entry now names reads as inconsistent")
+	}
+}
+
+func TestVolumeIndexDigestFollowsRemovedLookupEntry(t *testing.T) {
+	topo, old, fresh := addressSharingNodes(t)
+	v := digestTestVolume(1)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{v}, old)
+
+	// fresh shares old's address, so unregistering through it removes old's
+	// lookup entry. The digest must come off the node that was actually removed.
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
+	vl.SetVolumeUnavailable(fresh, needle.VolumeId(1))
+
+	if got := topo.Lookup("c", needle.VolumeId(1)); got != nil {
+		t.Fatalf("expected the lookup entry to be gone, got %v", got)
+	}
+	if old.HasConsistentVolumeIndex() {
+		t.Error("the removed node still holds the volume and cannot serve it, so its index must read as inconsistent")
+	}
+	if _, servable := fresh.VolumeIndexDigests(); servable != 0 {
+		t.Error("the node that was merely passed in should never have gained the entry")
+	}
+}
+
+// The two ends must agree on real heartbeat data, not just on hand-built
+// messages: the volume server hashes what it is about to send, the master
+// hashes what it stored from it.
+func TestMasterDigestMatchesWhatAVolumeServerReports(t *testing.T) {
+	dir := t.TempDir()
+	loc := storage.NewDiskLocation(dir, 100, util.MinFreeSpace{}, "", types.HardDriveType, nil,
+		stats.DefaultDiskIOProbeConfig())
+	for _, vid := range []needle.VolumeId{1, 2, 3} {
+		v, err := storage.NewVolume(loc.Directory, loc.IdxDirectory, "", vid, storage.NeedleMapInMemory,
+			&super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+		loc.SetVolume(vid, v)
+	}
+
+	reported := make([]*master_pb.VolumeInformationMessage, 0, 3)
+	var serverDigest uint64
+	for _, vid := range []needle.VolumeId{1, 2, 3} {
+		v, _ := loc.FindVolume(vid)
+		_, m := v.ToVolumeInformationMessage(nil)
+		if m == nil {
+			t.Fatalf("volume %d reported nothing", vid)
+		}
+		vi, err := storage.NewVolumeInfo(m)
+		if err != nil {
+			t.Fatal(err)
+		}
+		serverDigest ^= vi.ReportHash()
+		reported = append(reported, m)
+	}
+
+	topo, dn := digestTestNode(t)
+	topo.SyncDataNodeRegistration(reported, dn)
+
+	if got := dn.VolumeDigest(); got != serverDigest {
+		t.Errorf("master digest %d does not match the reporting server's %d", got, serverDigest)
+	}
+}

weed/topology/volume_disk_move_test.go

@@ -0,0 +1,112 @@
+package topology
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+func diskMoveNode(t *testing.T) (*Topology, *DataNode) {
+	t.Helper()
+	topo := NewTopology("move", nil, 32*1024*1024*1024, 5, false)
+	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100, "ssd": 100})
+	return topo, dn
+}
+
+func diskMoveVolume(id uint32, diskType string, diskId uint32) *master_pb.VolumeInformationMessage {
+	return &master_pb.VolumeInformationMessage{
+		Id: id, Size: 1024, Collection: "c", Version: 3, DiskType: diskType, DiskId: diskId,
+	}
+}
+
+func heldCopies(dn *DataNode) int {
+	total := 0
+	for _, c := range dn.Children() {
+		total += c.(*Disk).VolumeCount()
+	}
+	return total
+}
+
+func TestVolumeMovedBetweenDisks(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		from, to string
+	}{
+		{"SameDiskType", "", ""},
+		{"DifferentDiskType", "", "ssd"},
+		{"BackAgain", "ssd", ""},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			topo, dn := diskMoveNode(t)
+			topo.SyncDataNodeRegistration(
+				[]*master_pb.VolumeInformationMessage{diskMoveVolume(1, tc.from, 0)}, dn)
+			topo.SyncDataNodeRegistration(
+				[]*master_pb.VolumeInformationMessage{diskMoveVolume(1, tc.to, 1)}, dn)
+
+			if got := heldCopies(dn); got != 1 {
+				t.Errorf("master holds %d copies of a volume that moved, want 1", got)
+			}
+			stored, err := dn.GetVolumesById(needle.VolumeId(1))
+			if err != nil {
+				t.Fatalf("moved volume is no longer on the node: %v", err)
+			}
+			if stored.DiskType != tc.to || stored.DiskId != 1 {
+				t.Errorf("master has the volume on disk %q/%d, server reports %q/1",
+					stored.DiskType, stored.DiskId, tc.to)
+			}
+			if !dn.HasConsistentVolumeIndex() {
+				t.Error("the move left the lookup index disagreeing with the disks")
+			}
+		})
+	}
+}
+
+// A stale twin is two reports, not a move: dropping one would tell the master a
+// replica vanished.
+func TestVolumeReportedOnTwoDiskTypesIsKept(t *testing.T) {
+	topo, dn := diskMoveNode(t)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		diskMoveVolume(1, "", 0), diskMoveVolume(1, "ssd", 1),
+	}, dn)
+
+	if got := heldCopies(dn); got != 2 {
+		t.Errorf("master holds %d copies of a volume reported on two disks, want 2", got)
+	}
+	if dn.HasDuplicateVolumeIds() {
+		t.Error("a volume on two disk types is representable, so it should not disable digest comparison")
+	}
+
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{diskMoveVolume(1, "ssd", 1)}, dn)
+	if got := heldCopies(dn); got != 1 {
+		t.Errorf("master holds %d copies after the twin was unmounted, want 1", got)
+	}
+}
+
+func TestVolumeReportedTwiceOnOneDiskTypeIsFlagged(t *testing.T) {
+	topo, dn := diskMoveNode(t)
+	first := diskMoveVolume(1, "ssd", 0)
+	second := diskMoveVolume(1, "ssd", 1)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{first, second}, dn)
+
+	if !dn.HasDuplicateVolumeIds() {
+		t.Error("a volume reported twice on one disk type went unflagged")
+	}
+}
+
+func TestVolumeDigestSurvivesADiskMove(t *testing.T) {
+	topo, dn := diskMoveNode(t)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{diskMoveVolume(1, "", 0)}, dn)
+
+	moved := diskMoveVolume(1, "ssd", 1)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{moved}, dn)
+
+	reference, referenceNode := diskMoveNode(t)
+	reference.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{moved}, referenceNode)
+
+	if dn.VolumeDigest() != referenceNode.VolumeDigest() {
+		t.Errorf("after a disk move the digest is %d, a server holding only the moved volume reports %d",
+			dn.VolumeDigest(), referenceNode.VolumeDigest())
+	}
+}

weed/topology/volume_filter.go

@@ -0,0 +1,106 @@
+package topology
+
+import (
+	"errors"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
+)
+
+// VolumeFilter narrows a topology listing to the volumes a caller asked about,
+// selecting only what is listed under a disk. A nil field filters nothing, and
+// only nil does: the empty collection is a real one.
+type VolumeFilter struct {
+	Collection        *string
+	remoteStorageName *string
+	// VolumeIds selects the volumes it holds, and an empty one selects them all.
+	VolumeIds map[needle.VolumeId]struct{}
+	// nothing selects the topology alone, for a listing whose volumes travel
+	// in messages of their own.
+	nothing bool
+}
+
+// NoVolumes selects the topology and no volume in it.
+func NoVolumes() VolumeFilter {
+	return VolumeFilter{nothing: true}
+}
+
+// NewVolumeFilter reads what a VolumeList request asked for, where empty and
+// zero mean everything so a caller that forgets to narrow gets too much rather
+// than the wrong thing. The exception is topology_only, which asks for no
+// volumes at all: pairing it with a selector is refused rather than resolved,
+// since either reading would silently drop half the request.
+func NewVolumeFilter(req *master_pb.VolumeListRequest) (VolumeFilter, error) {
+	var filter VolumeFilter
+
+	if req.TopologyOnly {
+		if req.Collection != "" || len(req.VolumeIds) > 0 || req.DefaultCollectionOnly ||
+			req.RemoteStorageName != "" || req.LocalVolumeOnly {
+			return filter, errors.New("topology_only asks for no volumes, so it cannot be combined with collection, volume_ids, default_collection_only, remote_storage_name or local_volume_only")
+		}
+		filter.nothing = true
+		return filter, nil
+	}
+
+	switch {
+	case req.Collection != "":
+		collection := req.Collection
+		filter.Collection = &collection
+	case req.DefaultCollectionOnly:
+		defaultCollection := ""
+		filter.Collection = &defaultCollection
+	}
+
+	switch {
+	case req.RemoteStorageName != "":
+		filter.remoteStorageName = new(req.RemoteStorageName)
+	case req.LocalVolumeOnly:
+		filter.remoteStorageName = new("")
+	}
+
+	if len(req.VolumeIds) > 0 {
+		filter.VolumeIds = make(map[needle.VolumeId]struct{}, len(req.VolumeIds))
+		for _, volumeId := range req.VolumeIds {
+			filter.VolumeIds[needle.VolumeId(volumeId)] = struct{}{}
+		}
+	}
+	return filter, nil
+}
+
+// SelectsEverything lets a caller size its result for the whole disk up front.
+func (f VolumeFilter) SelectsEverything() bool {
+	return !f.nothing && f.Collection == nil && len(f.VolumeIds) == 0 && f.remoteStorageName == nil
+}
+
+type volumeLike interface {
+	GetCollection() string
+	GetVolumeId() needle.VolumeId
+	GetRemoteStorageName() string
+}
+
+func (f VolumeFilter) matches(vi volumeLike) bool {
+	if f.nothing {
+		return false
+	}
+	if f.Collection != nil && *f.Collection != vi.GetCollection() {
+		return false
+	}
+	if f.remoteStorageName != nil {
+		pattern, name := *f.remoteStorageName, vi.GetRemoteStorageName()
+		// the empty name is the local volumes, which only asking for the empty
+		// name selects; even * leaves them out.
+		if name == "" && pattern != "" {
+			return false
+		}
+		if !wildcard.MatchesWildcard(pattern, name) {
+			return false
+		}
+	}
+	if len(f.VolumeIds) > 0 {
+		if _, ok := f.VolumeIds[vi.GetVolumeId()]; !ok {
+			return false
+		}
+	}
+	return true
+}

weed/topology/volume_filter_test.go

@@ -0,0 +1,406 @@
+package topology
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+func filterTestTopology(t *testing.T) *Topology {
+	t.Helper()
+	topo := NewTopology("filter", nil, 32*1024*1024*1024, 5, false)
+	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		{Id: 1, Collection: "", Size: 1000, Version: 3, DiskId: 2},
+		{Id: 2, Collection: "c", Size: 2000, Version: 3, DiskId: 2},
+		{Id: 3, Collection: "other", Size: 3000, Version: 3, DiskId: 2},
+	}, dn)
+	topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{
+		{Id: 8, Collection: "c", EcIndexBits: 0x3fff, DiskId: 2},
+		{Id: 9, Collection: "other", EcIndexBits: 0x3fff, DiskId: 2},
+	}, dn)
+	return topo
+}
+
+// listed returns the volume and ec ids a filtered listing carries.
+func listed(info *master_pb.TopologyInfo) (volumes []uint32, ecVolumes []uint32) {
+	for _, dc := range info.DataCenterInfos {
+		for _, rack := range dc.RackInfos {
+			for _, node := range rack.DataNodeInfos {
+				for _, disk := range node.DiskInfos {
+					for _, v := range disk.VolumeInfos {
+						volumes = append(volumes, v.Id)
+					}
+					for _, ec := range disk.EcShardInfos {
+						ecVolumes = append(ecVolumes, ec.Id)
+					}
+				}
+			}
+		}
+	}
+	return
+}
+
+func equalIds(got, want []uint32) bool {
+	if len(got) != len(want) {
+		return false
+	}
+	seen := map[uint32]int{}
+	for _, id := range got {
+		seen[id]++
+	}
+	for _, id := range want {
+		seen[id]--
+	}
+	for _, n := range seen {
+		if n != 0 {
+			return false
+		}
+	}
+	return true
+}
+
+func TestVolumeFilterSelects(t *testing.T) {
+	topo := filterTestTopology(t)
+	collection := func(name string) *string { return &name }
+	volumes := func(ids ...needle.VolumeId) map[needle.VolumeId]struct{} {
+		set := make(map[needle.VolumeId]struct{}, len(ids))
+		for _, id := range ids {
+			set[id] = struct{}{}
+		}
+		return set
+	}
+
+	for _, tc := range []struct {
+		name          string
+		filter        VolumeFilter
+		wantVolumes   []uint32
+		wantEcVolumes []uint32
+	}{
+		{"no filter takes everything", VolumeFilter{}, []uint32{1, 2, 3}, []uint32{8, 9}},
+		{"one collection", VolumeFilter{Collection: collection("c")}, []uint32{2}, []uint32{8}},
+		// Asking for it must not read as asking for everything.
+		{"the default collection", VolumeFilter{Collection: collection("")}, []uint32{1}, nil},
+		{"a collection nothing is in", VolumeFilter{Collection: collection("none")}, nil, nil},
+		{"one volume", VolumeFilter{VolumeIds: volumes(3)}, []uint32{3}, nil},
+		{"one ec volume", VolumeFilter{VolumeIds: volumes(9)}, nil, []uint32{9}},
+		{"both, agreeing", VolumeFilter{Collection: collection("other"), VolumeIds: volumes(3)}, []uint32{3}, nil},
+		{"both, disagreeing", VolumeFilter{Collection: collection("c"), VolumeIds: volumes(3)}, nil, nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			volumes, ecVolumes := listed(topo.ToTopologyInfo(tc.filter))
+			if !equalIds(volumes, tc.wantVolumes) {
+				t.Errorf("listed volumes %v, want %v", volumes, tc.wantVolumes)
+			}
+			if !equalIds(ecVolumes, tc.wantEcVolumes) {
+				t.Errorf("listed ec volumes %v, want %v", ecVolumes, tc.wantEcVolumes)
+			}
+		})
+	}
+}
+
+// an id nothing answers to narrows the listing rather than widening it.
+func TestVolumeFilterVolumeIds(t *testing.T) {
+	topo := filterTestTopology(t)
+
+	for _, tc := range []struct {
+		name          string
+		ids           []needle.VolumeId
+		wantVolumes   []uint32
+		wantEcVolumes []uint32
+	}{
+		// No id asked of the filter, so every volume is listed.
+		{"no id asked", nil, []uint32{1, 2, 3}, []uint32{8, 9}},
+		{"one regular volume asked", []needle.VolumeId{2}, []uint32{2}, nil},
+		{"one ec volume asked", []needle.VolumeId{8}, nil, []uint32{8}},
+		// Both kinds answer to the same set of ids.
+		{"both kinds asked", []needle.VolumeId{1, 8}, []uint32{1}, []uint32{8}},
+		{"every volume asked", []needle.VolumeId{1, 2, 3, 8, 9}, []uint32{1, 2, 3}, []uint32{8, 9}},
+		// An id nothing answers to must not read as asking for everything.
+		{"a missing id", []needle.VolumeId{7}, nil, nil},
+		{"a found and a missing id", []needle.VolumeId{2, 7}, []uint32{2}, nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			ids := make(map[needle.VolumeId]struct{}, len(tc.ids))
+			for _, id := range tc.ids {
+				ids[id] = struct{}{}
+			}
+			volumes, ecVolumes := listed(topo.ToTopologyInfo(VolumeFilter{VolumeIds: ids}))
+			if !equalIds(volumes, tc.wantVolumes) {
+				t.Errorf("listed volumes %v, want %v", volumes, tc.wantVolumes)
+			}
+			if !equalIds(ecVolumes, tc.wantEcVolumes) {
+				t.Errorf("listed ec volumes %v, want %v", ecVolumes, tc.wantEcVolumes)
+			}
+		})
+	}
+}
+
+func TestVolumeFilterRemoteStorageNameWildcards(t *testing.T) {
+	topo := NewTopology("filter", nil, 32*1024*1024*1024, 5, false)
+	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("10.0.0.2", 8080, 18080, "", "", map[string]uint32{"": 100})
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		{Id: 11, Collection: "c", Size: 100, Version: 3, DiskId: 3, RemoteStorageName: "s3.backup"},
+		{Id: 12, Collection: "c", Size: 100, Version: 3, DiskId: 3, RemoteStorageName: "s3.archive"},
+		{Id: 13, Collection: "c", Size: 100, Version: 3, DiskId: 3, RemoteStorageName: "gcs.cold"},
+		{Id: 14, Collection: "c", Size: 100, Version: 3, DiskId: 3},
+	}, dn)
+
+	for _, tc := range []struct {
+		name string
+		f    VolumeFilter
+		want []uint32
+	}{
+		// A nil name asks for every volume, tiered or local alike.
+		{"no remote storage asked", VolumeFilter{}, []uint32{11, 12, 13, 14}},
+		{"an exact name", VolumeFilter{remoteStorageName: new("s3.backup")}, []uint32{11}},
+		{"a wildcarded storage", VolumeFilter{remoteStorageName: new("s3.*")}, []uint32{11, 12}},
+		{"a wildcarded tier", VolumeFilter{remoteStorageName: new("*.cold")}, []uint32{13}},
+		{"every tiered volume", VolumeFilter{remoteStorageName: new("*")}, []uint32{11, 12, 13}},
+		// Not a wildcard, so it matches nothing rather than everything.
+		{"a name no volume is on", VolumeFilter{remoteStorageName: new("s3.other")}, nil},
+		// The local volumes are the ones with no remote storage name.
+		{"the local ones only", VolumeFilter{remoteStorageName: new("")}, []uint32{14}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			volumes, ecVolumes := listed(topo.ToTopologyInfo(tc.f))
+			if !equalIds(volumes, tc.want) {
+				t.Errorf("listed volumes %v, want %v", volumes, tc.want)
+			}
+			if len(ecVolumes) != 0 {
+				t.Errorf("listed ec volumes %v, want none", ecVolumes)
+			}
+		})
+	}
+}
+
+// A filter never changes which disks are reported or what they say about
+// themselves.
+func TestVolumeFilterKeepsTheTopology(t *testing.T) {
+	topo := filterTestTopology(t)
+	full := topo.ToTopologyInfo(VolumeFilter{})
+	name := "c"
+	narrowed := topo.ToTopologyInfo(VolumeFilter{Collection: &name})
+
+	for _, info := range []*master_pb.TopologyInfo{full, narrowed} {
+		if len(info.DataCenterInfos) != 1 || len(info.DataCenterInfos[0].RackInfos) != 1 {
+			t.Fatalf("listing lost its topology: %v", info)
+		}
+	}
+
+	fullDisk := full.DataCenterInfos[0].RackInfos[0].DataNodeInfos[0].DiskInfos[""]
+	narrowedDisk := narrowed.DataCenterInfos[0].RackInfos[0].DataNodeInfos[0].DiskInfos[""]
+	if narrowedDisk == nil {
+		t.Fatal("the filtered listing dropped the disk")
+	}
+	if narrowedDisk.VolumeCount != fullDisk.VolumeCount {
+		t.Errorf("volume count %d, want the disk's own %d", narrowedDisk.VolumeCount, fullDisk.VolumeCount)
+	}
+	if narrowedDisk.MaxVolumeCount != fullDisk.MaxVolumeCount {
+		t.Errorf("max volume count %d, want %d", narrowedDisk.MaxVolumeCount, fullDisk.MaxVolumeCount)
+	}
+	if narrowedDisk.DiskId != fullDisk.DiskId {
+		t.Errorf("disk id %d, want %d", narrowedDisk.DiskId, fullDisk.DiskId)
+	}
+}
+
+// So a caller that finds nothing can tell an empty answer from a missing disk.
+func TestVolumeFilterKeepsTheDiskIdWhenNothingMatches(t *testing.T) {
+	topo := filterTestTopology(t)
+	name := "none"
+	info := topo.ToTopologyInfo(VolumeFilter{Collection: &name})
+	disk := info.DataCenterInfos[0].RackInfos[0].DataNodeInfos[0].DiskInfos[""]
+	if len(disk.VolumeInfos) != 0 {
+		t.Fatalf("expected nothing listed, got %d volumes", len(disk.VolumeInfos))
+	}
+	if disk.DiskId != 2 {
+		t.Errorf("reported disk id %d, want the one its volumes are on", disk.DiskId)
+	}
+}
+
+func TestNewVolumeFilterReadsTheRequest(t *testing.T) {
+	collectionOf := func(f VolumeFilter) string {
+		if f.Collection == nil {
+			return "<every>"
+		}
+		return *f.Collection
+	}
+
+	for _, tc := range []struct {
+		name    string
+		request *master_pb.VolumeListRequest
+		want    string
+	}{
+		// A caller passing through its own "" is answered too much, not wrongly.
+		{"an empty request", &master_pb.VolumeListRequest{}, "<every>"},
+		{"an empty collection", &master_pb.VolumeListRequest{Collection: ""}, "<every>"},
+		{"a named collection", &master_pb.VolumeListRequest{Collection: "c"}, "c"},
+		{"the default collection", &master_pb.VolumeListRequest{DefaultCollectionOnly: true}, ""},
+		// Contradictory, so the more specific of the two wins.
+		{"both", &master_pb.VolumeListRequest{Collection: "c", DefaultCollectionOnly: true}, "c"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			filter, err := NewVolumeFilter(tc.request)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got := collectionOf(filter); got != tc.want {
+				t.Errorf("selected collection %q, want %q", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestNewVolumeFilterReadsTheRemoteStorageRequest(t *testing.T) {
+	remoteStorageOf := func(f VolumeFilter) string {
+		if f.remoteStorageName == nil {
+			return "<every>"
+		}
+		return *f.remoteStorageName
+	}
+
+	for _, tc := range []struct {
+		name    string
+		request *master_pb.VolumeListRequest
+		want    string
+	}{
+		// A caller passing through its own "" is answered too much, not wrongly.
+		{"an empty request", &master_pb.VolumeListRequest{}, "<every>"},
+		{"an empty remote storage name", &master_pb.VolumeListRequest{RemoteStorageName: ""}, "<every>"},
+		{"a named remote storage", &master_pb.VolumeListRequest{RemoteStorageName: "s3.backup"}, "s3.backup"},
+		// A wildcard is the caller's pattern, carried across untouched.
+		{"a wildcarded storage", &master_pb.VolumeListRequest{RemoteStorageName: "*"}, "*"},
+		// The one storage (local) the empty string cannot name.
+		{"the local volumes only", &master_pb.VolumeListRequest{LocalVolumeOnly: true}, ""},
+		// Contradictory, so the more specific of the two wins.
+		{"both", &master_pb.VolumeListRequest{RemoteStorageName: "s3.backup", LocalVolumeOnly: true}, "s3.backup"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			filter, err := NewVolumeFilter(tc.request)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got := remoteStorageOf(filter); got != tc.want {
+				t.Errorf("selected remote storage %q, want %q", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestNewVolumeFilterReadsTheVolumeIds(t *testing.T) {
+	idsOf := func(f VolumeFilter) []uint32 {
+		ids := make([]uint32, 0, len(f.VolumeIds))
+		for id := range f.VolumeIds {
+			ids = append(ids, uint32(id))
+		}
+		return ids
+	}
+
+	for _, tc := range []struct {
+		name    string
+		request *master_pb.VolumeListRequest
+		want    []uint32
+	}{
+		// No id asked of the request, so every volume is listed.
+		{"an empty request", &master_pb.VolumeListRequest{}, nil},
+		{"one id", &master_pb.VolumeListRequest{VolumeIds: []uint32{7}}, []uint32{7}},
+		{"a list of ids", &master_pb.VolumeListRequest{VolumeIds: []uint32{2, 8}}, []uint32{2, 8}},
+		// Asking twice for a volume selects it once.
+		{"duplicates", &master_pb.VolumeListRequest{VolumeIds: []uint32{2, 2}}, []uint32{2}},
+		// The list is taken literally: no volume answers to zero, so a zero in
+		// it narrows the listing rather than widening it.
+		{"a zero among the ids", &master_pb.VolumeListRequest{VolumeIds: []uint32{0}}, []uint32{0}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			filter, err := NewVolumeFilter(tc.request)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got := idsOf(filter); !equalIds(got, tc.want) {
+				t.Errorf("selected volume ids %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// topology_only leaves out the ec shards too, not just the volumes; the disks
+// and their counters stay.
+func TestNewVolumeFilterReadsTopologyOnly(t *testing.T) {
+	topo := filterTestTopology(t)
+	filter, err := NewVolumeFilter(&master_pb.VolumeListRequest{TopologyOnly: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	info := topo.ToTopologyInfo(filter)
+	volumes, ecVolumes := listed(info)
+	if len(volumes) != 0 || len(ecVolumes) != 0 {
+		t.Errorf("listed volumes %v and ec volumes %v, want neither", volumes, ecVolumes)
+	}
+
+	full := topo.ToTopologyInfo(VolumeFilter{})
+	fullDisk := full.DataCenterInfos[0].RackInfos[0].DataNodeInfos[0].DiskInfos[""]
+	disk := info.DataCenterInfos[0].RackInfos[0].DataNodeInfos[0].DiskInfos[""]
+	if disk == nil {
+		t.Fatal("the topology_only listing dropped the disk")
+	}
+	if disk.VolumeCount != fullDisk.VolumeCount || disk.MaxVolumeCount != fullDisk.MaxVolumeCount {
+		t.Errorf("disk counters %d/%d, want the full listing's %d/%d",
+			disk.VolumeCount, disk.MaxVolumeCount, fullDisk.VolumeCount, fullDisk.MaxVolumeCount)
+	}
+}
+
+// Selecting volumes in a request that asks for none is a caller bug, refused
+// rather than resolved either way in silence.
+func TestNewVolumeFilterRefusesTopologyOnlyWithSelectors(t *testing.T) {
+	for _, tc := range []struct {
+		name    string
+		request *master_pb.VolumeListRequest
+	}{
+		{"a collection", &master_pb.VolumeListRequest{TopologyOnly: true, Collection: "c"}},
+		{"volume ids", &master_pb.VolumeListRequest{TopologyOnly: true, VolumeIds: []uint32{1}}},
+		{"the default collection", &master_pb.VolumeListRequest{TopologyOnly: true, DefaultCollectionOnly: true}},
+		{"a remote storage", &master_pb.VolumeListRequest{TopologyOnly: true, RemoteStorageName: "s3.backup"}},
+		{"the local volumes", &master_pb.VolumeListRequest{TopologyOnly: true, LocalVolumeOnly: true}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			if _, err := NewVolumeFilter(tc.request); err == nil {
+				t.Error("expected the contradiction refused, got a filter")
+			}
+		})
+	}
+}
+
+// Everything a listing reads off a disk must be read under its lock, including
+// how much room to reserve, or a heartbeat writing the map races it.
+func TestVolumeFilterListsWhileVolumesChange(t *testing.T) {
+	topo := filterTestTopology(t)
+	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
+
+	stop := make(chan struct{})
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := 0; ; i++ {
+			select {
+			case <-stop:
+				return
+			default:
+			}
+			topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+				{Id: uint32(100 + i%50), Collection: "c", Size: 1, Version: 3, DiskId: 2},
+			}, dn)
+		}
+	}()
+
+	name := "c"
+	for i := 0; i < 200; i++ {
+		topo.ToTopologyInfo(VolumeFilter{})
+		topo.ToTopologyInfo(VolumeFilter{Collection: &name})
+	}
+	close(stop)
+	<-done
+}

weed/topology/volume_growth.go

@@ -0,0 +1,415 @@
+package topology
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/rand/v2"
+	"reflect"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+/*
+This package is created to resolve these replica placement issues:
+1. growth factor for each replica level, e.g., add 10 volumes for 1 copy, 20 volumes for 2 copies, 30 volumes for 3 copies
+2. in time of tight storage, how to reduce replica level
+3. optimizing for hot data on faster disk, cold data on cheaper storage,
+4. volume allocation for each bucket
+*/
+
+type VolumeGrowRequest struct {
+	Option *VolumeGrowOption
+	Count  uint32
+	Force  bool
+	Reason string
+}
+
+func (vg *VolumeGrowRequest) Equals(req *VolumeGrowRequest) bool {
+	return reflect.DeepEqual(vg.Option, req.Option) && vg.Count == req.Count && vg.Force == req.Force
+}
+
+type volumeGrowthStrategy struct {
+	Copy1Count     uint32
+	Copy2Count     uint32
+	Copy3Count     uint32
+	CopyOtherCount uint32
+	Threshold      float64
+}
+
+var (
+	VolumeGrowStrategy = volumeGrowthStrategy{
+		Copy1Count:     7,
+		Copy2Count:     6,
+		Copy3Count:     3,
+		CopyOtherCount: 1,
+		Threshold:      0.9,
+	}
+)
+
+type VolumeGrowOption struct {
+	Collection         string                        `json:"collection,omitempty"`
+	ReplicaPlacement   *super_block.ReplicaPlacement `json:"replication,omitempty"`
+	Ttl                *needle.TTL                   `json:"ttl,omitempty"`
+	DiskType           types.DiskType                `json:"disk,omitempty"`
+	Preallocate        int64                         `json:"preallocate,omitempty"`
+	DataCenter         string                        `json:"dataCenter,omitempty"`
+	Rack               string                        `json:"rack,omitempty"`
+	DataNode           string                        `json:"dataNode,omitempty"`
+	MemoryMapMaxSizeMb uint32                        `json:"memoryMapMaxSizeMb,omitempty"`
+	Version            uint32                        `json:"version,omitempty"`
+}
+
+type VolumeGrowth struct {
+	accessLock sync.Mutex
+}
+
+// VolumeGrowReservation tracks capacity reservations for a volume creation operation
+type VolumeGrowReservation struct {
+	servers        []*DataNode
+	reservationIds []string
+	diskType       types.DiskType
+}
+
+// releaseAllReservations releases all reservations in this volume grow operation
+func (vgr *VolumeGrowReservation) releaseAllReservations() {
+	for i, server := range vgr.servers {
+		if i < len(vgr.reservationIds) && vgr.reservationIds[i] != "" {
+			server.ReleaseReservedCapacity(vgr.reservationIds[i])
+		}
+	}
+}
+
+func (o *VolumeGrowOption) String() string {
+	blob, _ := json.Marshal(o)
+	return string(blob)
+}
+
+func NewDefaultVolumeGrowth() *VolumeGrowth {
+	return &VolumeGrowth{}
+}
+
+// VolumeGrowthCountForCopies returns the configured number of logical volumes
+// to create at once for a given replica copy count (master.volume_growth.copy_N).
+func VolumeGrowthCountForCopies(copyCount int) uint32 {
+	switch copyCount {
+	case 1:
+		return VolumeGrowStrategy.Copy1Count
+	case 2:
+		return VolumeGrowStrategy.Copy2Count
+	case 3:
+		return VolumeGrowStrategy.Copy3Count
+	default:
+		return VolumeGrowStrategy.CopyOtherCount
+	}
+}
+
+// one replication type may need rp.GetCopyCount() actual volumes
+// given copyCount, how many logical volumes to create
+func (vg *VolumeGrowth) findVolumeCount(copyCount int) (count uint32) {
+	return VolumeGrowthCountForCopies(copyCount)
+}
+
+func (vg *VolumeGrowth) AutomaticGrowByType(option *VolumeGrowOption, grpcDialOption grpc.DialOption, topo *Topology, targetCount uint32) (result []*master_pb.VolumeLocation, err error) {
+	if targetCount == 0 {
+		targetCount = vg.findVolumeCount(option.ReplicaPlacement.GetCopyCount())
+	}
+	result, err = vg.GrowByCountAndType(grpcDialOption, targetCount, option, topo)
+	if len(result) > 0 && len(result)%option.ReplicaPlacement.GetCopyCount() == 0 {
+		return result, nil
+	}
+	return result, err
+}
+func (vg *VolumeGrowth) GrowByCountAndType(grpcDialOption grpc.DialOption, targetCount uint32, option *VolumeGrowOption, topo *Topology) (result []*master_pb.VolumeLocation, err error) {
+	vg.accessLock.Lock()
+	defer vg.accessLock.Unlock()
+
+	for i := uint32(0); i < targetCount; i++ {
+		if res, e := vg.findAndGrow(grpcDialOption, topo, option); e == nil {
+			result = append(result, res...)
+			stats.MasterVolumeCreationCounter.WithLabelValues("success").Inc()
+		} else {
+			glog.V(0).Infof("create %d volume, created %d: %v", targetCount, len(result), e)
+			stats.MasterVolumeCreationCounter.WithLabelValues("failure").Inc()
+			return result, e
+		}
+	}
+	return
+}
+
+func (vg *VolumeGrowth) findAndGrow(grpcDialOption grpc.DialOption, topo *Topology, option *VolumeGrowOption) (result []*master_pb.VolumeLocation, err error) {
+	// Wait for warmup to complete before searching for slots so the
+	// topology has all volume servers registered.
+	for topo.IsWarmingUp() {
+		glog.V(0).Infof("wait for volume servers to join back")
+		time.Sleep(topo.WarmupDuration() / WarmupPulseMultiplier / 2)
+	}
+
+	servers, reservation, e := vg.findEmptySlotsForOneVolume(topo, option, true) // use reservations
+	if e != nil {
+		return nil, e
+	}
+	// Ensure reservations are released if anything goes wrong
+	defer func() {
+		if err != nil && reservation != nil {
+			reservation.releaseAllReservations()
+		}
+	}()
+	vid, raftErr := topo.NextVolumeId()
+	if raftErr != nil {
+		return nil, raftErr
+	}
+	if err = vg.grow(grpcDialOption, topo, vid, option, reservation, servers...); err == nil {
+		for _, server := range servers {
+			result = append(result, &master_pb.VolumeLocation{
+				Url:        server.Url(),
+				PublicUrl:  server.PublicUrl,
+				DataCenter: server.GetDataCenterId(),
+				GrpcPort:   uint32(server.GrpcPort),
+				NewVids:    []uint32{uint32(vid)},
+			})
+		}
+	}
+	return
+}
+
+// 1. find the main data node
+// 1.1 collect all data nodes that have 1 slots
+// 2.2 collect all racks that have rp.SameRackCount+1
+// 2.2 collect all data centers that have DiffRackCount+rp.SameRackCount+1
+// 2. find rest data nodes
+// If useReservations is true, reserves capacity on each server and returns reservation info
+func (vg *VolumeGrowth) findEmptySlotsForOneVolume(topo *Topology, option *VolumeGrowOption, useReservations bool) (servers []*DataNode, reservation *VolumeGrowReservation, err error) {
+	//find main datacenter and other data centers
+	rp := option.ReplicaPlacement
+
+	// Track tentative reservations to make the process atomic
+	var tentativeReservation *VolumeGrowReservation
+
+	// Select appropriate functions based on useReservations flag
+	var availableSpaceFunc func(Node, *VolumeGrowOption) int64
+	var reserveOneVolumeFunc func(Node, int64, *VolumeGrowOption) (*DataNode, error)
+
+	if useReservations {
+		// Initialize tentative reservation tracking
+		tentativeReservation = &VolumeGrowReservation{
+			servers:        make([]*DataNode, 0),
+			reservationIds: make([]string, 0),
+			diskType:       option.DiskType,
+		}
+
+		// For reservations, we make actual reservations during node selection
+		availableSpaceFunc = func(node Node, option *VolumeGrowOption) int64 {
+			return node.AvailableSpaceForReservation(option)
+		}
+		reserveOneVolumeFunc = func(node Node, r int64, option *VolumeGrowOption) (*DataNode, error) {
+			return node.ReserveOneVolumeForReservation(r, option)
+		}
+	} else {
+		availableSpaceFunc = func(node Node, option *VolumeGrowOption) int64 {
+			return node.AvailableSpaceFor(option)
+		}
+		reserveOneVolumeFunc = func(node Node, r int64, option *VolumeGrowOption) (*DataNode, error) {
+			return node.ReserveOneVolume(r, option)
+		}
+	}
+
+	// Ensure cleanup of partial reservations on error
+	defer func() {
+		if err != nil && tentativeReservation != nil {
+			tentativeReservation.releaseAllReservations()
+		}
+	}()
+	mainDataCenter, otherDataCenters, dc_err := topo.PickNodesByWeight(rp.DiffDataCenterCount+1, option, func(node Node) error {
+		if option.DataCenter != "" && node.IsDataCenter() && node.Id() != NodeId(option.DataCenter) {
+			return fmt.Errorf("Not matching preferred data center:%s", option.DataCenter)
+		}
+		if len(node.Children()) < rp.DiffRackCount+1 {
+			return fmt.Errorf("Only has %d racks, not enough for %d.", len(node.Children()), rp.DiffRackCount+1)
+		}
+		if availableSpaceFunc(node, option) < int64(rp.DiffRackCount+rp.SameRackCount+1) {
+			return fmt.Errorf("Free:%d < Expected:%d", availableSpaceFunc(node, option), rp.DiffRackCount+rp.SameRackCount+1)
+		}
+		possibleRacksCount := 0
+		for _, rack := range node.Children() {
+			possibleDataNodesCount := 0
+			for _, n := range rack.Children() {
+				if availableSpaceFunc(n, option) >= 1 {
+					possibleDataNodesCount++
+				}
+			}
+			if possibleDataNodesCount >= rp.SameRackCount+1 {
+				possibleRacksCount++
+			}
+		}
+		if possibleRacksCount < rp.DiffRackCount+1 {
+			return fmt.Errorf("Only has %d racks with more than %d free data nodes, not enough for %d.", possibleRacksCount, rp.SameRackCount+1, rp.DiffRackCount+1)
+		}
+		return nil
+	})
+	if dc_err != nil {
+		return nil, nil, dc_err
+	}
+
+	//find main rack and other racks
+	mainRack, otherRacks, rackErr := mainDataCenter.(*DataCenter).PickNodesByWeight(rp.DiffRackCount+1, option, func(node Node) error {
+		if option.Rack != "" && node.IsRack() && node.Id() != NodeId(option.Rack) {
+			return fmt.Errorf("Not matching preferred rack:%s", option.Rack)
+		}
+		if availableSpaceFunc(node, option) < int64(rp.SameRackCount+1) {
+			return fmt.Errorf("Free:%d < Expected:%d", availableSpaceFunc(node, option), rp.SameRackCount+1)
+		}
+		if len(node.Children()) < rp.SameRackCount+1 {
+			// a bit faster way to test free racks
+			return fmt.Errorf("Only has %d data nodes, not enough for %d.", len(node.Children()), rp.SameRackCount+1)
+		}
+		possibleDataNodesCount := 0
+		for _, n := range node.Children() {
+			if availableSpaceFunc(n, option) >= 1 {
+				possibleDataNodesCount++
+			}
+		}
+		if possibleDataNodesCount < rp.SameRackCount+1 {
+			return fmt.Errorf("Only has %d data nodes with a slot, not enough for %d.", possibleDataNodesCount, rp.SameRackCount+1)
+		}
+		return nil
+	})
+	if rackErr != nil {
+		return nil, nil, rackErr
+	}
+
+	//find main server and other servers
+	mainServer, otherServers, serverErr := mainRack.(*Rack).PickNodesByWeight(rp.SameRackCount+1, option, func(node Node) error {
+		if option.DataNode != "" && node.IsDataNode() && node.Id() != NodeId(option.DataNode) {
+			return fmt.Errorf("Not matching preferred data node:%s", option.DataNode)
+		}
+
+		if useReservations {
+			// For reservations, atomically check and reserve capacity
+			if node.IsDataNode() {
+				reservationId, success := node.TryReserveCapacity(option.DiskType, 1)
+				if !success {
+					return fmt.Errorf("Cannot reserve capacity on node %s", node.Id())
+				}
+				// Track the reservation for later cleanup if needed
+				tentativeReservation.servers = append(tentativeReservation.servers, node.(*DataNode))
+				tentativeReservation.reservationIds = append(tentativeReservation.reservationIds, reservationId)
+			} else if availableSpaceFunc(node, option) < 1 {
+				return fmt.Errorf("Free:%d < Expected:%d", availableSpaceFunc(node, option), 1)
+			}
+		} else if availableSpaceFunc(node, option) < 1 {
+			return fmt.Errorf("Free:%d < Expected:%d", availableSpaceFunc(node, option), 1)
+		}
+		return nil
+	})
+	if serverErr != nil {
+		return nil, nil, serverErr
+	}
+
+	servers = append(servers, mainServer.(*DataNode))
+	for _, server := range otherServers {
+		servers = append(servers, server.(*DataNode))
+	}
+	for _, rack := range otherRacks {
+		r := rand.Int64N(availableSpaceFunc(rack, option))
+		if server, e := reserveOneVolumeFunc(rack, r, option); e == nil {
+			servers = append(servers, server)
+
+			// If using reservations, also make a reservation on the selected server
+			if useReservations {
+				reservationId, success := server.TryReserveCapacity(option.DiskType, 1)
+				if !success {
+					return servers, nil, fmt.Errorf("failed to reserve capacity on server %s from other rack", server.Id())
+				}
+				tentativeReservation.servers = append(tentativeReservation.servers, server)
+				tentativeReservation.reservationIds = append(tentativeReservation.reservationIds, reservationId)
+			}
+		} else {
+			return servers, nil, e
+		}
+	}
+	for _, datacenter := range otherDataCenters {
+		r := rand.Int64N(availableSpaceFunc(datacenter, option))
+		if server, e := reserveOneVolumeFunc(datacenter, r, option); e == nil {
+			servers = append(servers, server)
+
+			// If using reservations, also make a reservation on the selected server
+			if useReservations {
+				reservationId, success := server.TryReserveCapacity(option.DiskType, 1)
+				if !success {
+					return servers, nil, fmt.Errorf("failed to reserve capacity on server %s from other datacenter", server.Id())
+				}
+				tentativeReservation.servers = append(tentativeReservation.servers, server)
+				tentativeReservation.reservationIds = append(tentativeReservation.reservationIds, reservationId)
+			}
+		} else {
+			return servers, nil, e
+		}
+	}
+
+	// If reservations were made, return the tentative reservation
+	if useReservations && tentativeReservation != nil {
+		reservation = tentativeReservation
+		glog.V(1).Infof("Successfully reserved capacity on %d servers for volume creation", len(servers))
+	}
+
+	return servers, reservation, nil
+}
+
+// grow creates volumes on the provided servers, optionally managing capacity reservations
+func (vg *VolumeGrowth) grow(grpcDialOption grpc.DialOption, topo *Topology, vid needle.VolumeId, option *VolumeGrowOption, reservation *VolumeGrowReservation, servers ...*DataNode) (growErr error) {
+	var createdVolumes []storage.VolumeInfo
+	for _, server := range servers {
+		if err := AllocateVolume(server, grpcDialOption, vid, option); err == nil {
+			createdVolumes = append(createdVolumes, storage.VolumeInfo{
+				Id:               vid,
+				Size:             0,
+				Collection:       option.Collection,
+				ReplicaPlacement: option.ReplicaPlacement,
+				Ttl:              option.Ttl,
+				Version:          needle.Version(option.Version),
+				DiskType:         option.DiskType.String(),
+				ModifiedAtSecond: time.Now().Unix(),
+			})
+			glog.V(0).Infof("Created Volume %d on %s", vid, server.NodeImpl.String())
+		} else {
+			glog.Warningf("Failed to assign volume %d on %s: %v", vid, server.NodeImpl.String(), err)
+			growErr = fmt.Errorf("failed to assign volume %d on %s: %v", vid, server.NodeImpl.String(), err)
+			break
+		}
+	}
+
+	if growErr == nil {
+		for i, vi := range createdVolumes {
+			server := servers[i]
+			server.AddProvisionalVolume(vi)
+			topo.RegisterVolumeLayout(vi, server)
+			glog.V(0).Infof("Registered Volume %d on %s", vid, server.NodeImpl.String())
+		}
+		// Release reservations on success since volumes are now registered
+		if reservation != nil {
+			reservation.releaseAllReservations()
+		}
+	} else {
+		// cleaning up created volume replicas
+		for i, vi := range createdVolumes {
+			server := servers[i]
+			if err := DeleteVolume(server, grpcDialOption, vi.Id); err != nil {
+				glog.Warningf("Failed to clean up volume %d on %s", vid, server.NodeImpl.String())
+			}
+		}
+		// Reservations will be released by the caller in case of failure
+	}
+
+	return growErr
+}

weed/topology/volume_growth_reservation_test.go

@@ -0,0 +1,295 @@
+package topology
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// MockGrpcDialOption simulates grpc connection for testing
+type MockGrpcDialOption struct{}
+
+func TestVolumeGrowth_ReservationBasedAllocation(t *testing.T) {
+	// Create test topology with single server for predictable behavior
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	// Create data center and rack
+	dc := NewDataCenter("dc1")
+	topo.LinkChildNode(dc)
+	rack := NewRack("rack1")
+	dc.LinkChildNode(rack)
+
+	// Create single data node with limited capacity
+	dn := NewDataNode("server1")
+	rack.LinkChildNode(dn)
+
+	// Set up disk with limited capacity (only 5 volumes)
+	disk := NewDisk(types.HardDriveType.String())
+	disk.diskUsages.getOrCreateDisk(types.HardDriveType).maxVolumeCount = 5
+	dn.LinkChildNode(disk)
+
+	// Test volume growth with reservation
+	vg := NewDefaultVolumeGrowth()
+	rp, _ := super_block.NewReplicaPlacementFromString("000") // Single copy (no replicas)
+
+	option := &VolumeGrowOption{
+		Collection:       "test",
+		ReplicaPlacement: rp,
+		DiskType:         types.HardDriveType,
+	}
+
+	// Try to create volumes and verify reservations work
+	for i := 0; i < 5; i++ {
+		servers, reservation, err := vg.findEmptySlotsForOneVolume(topo, option, true)
+		if err != nil {
+			t.Errorf("Failed to find slots with reservation on iteration %d: %v", i, err)
+			continue
+		}
+
+		if len(servers) != 1 {
+			t.Errorf("Expected 1 server for replica placement 000, got %d", len(servers))
+		}
+
+		if len(reservation.reservationIds) != 1 {
+			t.Errorf("Expected 1 reservation ID, got %d", len(reservation.reservationIds))
+		}
+
+		// Verify the reservation is on our expected server
+		server := servers[0]
+		if server != dn {
+			t.Errorf("Expected volume to be allocated on server1, got %s", server.Id())
+		}
+
+		// Check available space before and after reservation
+		availableBeforeCreation := server.AvailableSpaceFor(option)
+		expectedBefore := int64(5 - i)
+		if availableBeforeCreation != expectedBefore {
+			t.Errorf("Iteration %d: Expected %d base available space, got %d", i, expectedBefore, availableBeforeCreation)
+		}
+
+		// Simulate successful volume creation
+		// Acquire lock briefly to access children map, then release before updating
+		dn.RLock()
+		disk := dn.children[NodeId(types.HardDriveType.String())].(*Disk)
+		dn.RUnlock()
+
+		deltaDiskUsage := &DiskUsageCounts{
+			volumeCount: 1,
+		}
+		disk.UpAdjustDiskUsageDelta(types.HardDriveType, deltaDiskUsage)
+
+		// Release reservation after successful creation
+		reservation.releaseAllReservations()
+
+		// Verify available space after creation
+		availableAfterCreation := server.AvailableSpaceFor(option)
+		expectedAfter := int64(5 - i - 1)
+		if availableAfterCreation != expectedAfter {
+			t.Errorf("Iteration %d: Expected %d available space after creation, got %d", i, expectedAfter, availableAfterCreation)
+		}
+	}
+
+	// After 5 volumes, should have no more capacity
+	_, _, err := vg.findEmptySlotsForOneVolume(topo, option, true)
+	if err == nil {
+		t.Error("Expected volume allocation to fail when server is at capacity")
+	}
+}
+
+func TestVolumeGrowth_ConcurrentAllocationPreventsRaceCondition(t *testing.T) {
+	// Create test topology with very limited capacity
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	dc := NewDataCenter("dc1")
+	topo.LinkChildNode(dc)
+	rack := NewRack("rack1")
+	dc.LinkChildNode(rack)
+
+	// Single data node with capacity for only 5 volumes
+	dn := NewDataNode("server1")
+	rack.LinkChildNode(dn)
+
+	disk := NewDisk(types.HardDriveType.String())
+	disk.diskUsages.getOrCreateDisk(types.HardDriveType).maxVolumeCount = 5
+	dn.LinkChildNode(disk)
+
+	vg := NewDefaultVolumeGrowth()
+	rp, _ := super_block.NewReplicaPlacementFromString("000") // Single copy (no replicas)
+
+	option := &VolumeGrowOption{
+		Collection:       "test",
+		ReplicaPlacement: rp,
+		DiskType:         types.HardDriveType,
+	}
+
+	// Simulate concurrent volume creation attempts
+	const concurrentRequests = 10
+	var wg sync.WaitGroup
+	var successCount, failureCount atomic.Int32
+	var commitMutex sync.Mutex // Ensures atomic commit of volume creation + reservation release
+
+	for i := 0; i < concurrentRequests; i++ {
+		wg.Add(1)
+		go func(requestId int) {
+			defer wg.Done()
+
+			_, reservation, err := vg.findEmptySlotsForOneVolume(topo, option, true)
+
+			if err != nil {
+				failureCount.Add(1)
+				t.Logf("Request %d failed as expected: %v", requestId, err)
+			} else {
+				successCount.Add(1)
+				t.Logf("Request %d succeeded, got reservation", requestId)
+
+				// Simulate completion: increment volume count BEFORE releasing reservation
+				if reservation != nil {
+					commitMutex.Lock()
+
+					// First, increment the volume count to reflect the created volume
+					// Acquire lock briefly to access children map, then release before updating
+					dn.RLock()
+					disk := dn.children[NodeId(types.HardDriveType.String())].(*Disk)
+					dn.RUnlock()
+
+					deltaDiskUsage := &DiskUsageCounts{
+						volumeCount: 1,
+					}
+					disk.UpAdjustDiskUsageDelta(types.HardDriveType, deltaDiskUsage)
+
+					// Then release the reservation
+					reservation.releaseAllReservations()
+
+					commitMutex.Unlock()
+				}
+			}
+		}(i)
+	}
+
+	wg.Wait()
+
+	// Collect results
+	successes := successCount.Load()
+	failures := failureCount.Load()
+	total := successes + failures
+
+	if total != concurrentRequests {
+		t.Fatalf("Expected %d total attempts recorded, got %d", concurrentRequests, total)
+	}
+
+	// At most the available capacity should succeed
+	const capacity = 5
+	if successes > capacity {
+		t.Errorf("Expected no more than %d successful reservations, got %d", capacity, successes)
+	}
+
+	// We should see at least the remaining attempts fail
+	minExpectedFailures := concurrentRequests - capacity
+	if failures < int32(minExpectedFailures) {
+		t.Errorf("Expected at least %d failed reservations, got %d", minExpectedFailures, failures)
+	}
+
+	// Verify final state matches the number of successful allocations
+	finalAvailable := dn.AvailableSpaceFor(option)
+	expectedAvailable := int64(capacity - successes)
+	if finalAvailable != expectedAvailable {
+		t.Errorf("Expected %d available space after allocations, got %d", expectedAvailable, finalAvailable)
+	}
+
+	t.Logf("Concurrent test completed: %d successes, %d failures", successes, failures)
+}
+
+func TestVolumeGrowth_ReservationFailureRollback(t *testing.T) {
+	// Create topology with multiple servers, but limited total capacity
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	dc := NewDataCenter("dc1")
+	topo.LinkChildNode(dc)
+	rack := NewRack("rack1")
+	dc.LinkChildNode(rack)
+
+	// Create two servers with different available capacity
+	dn1 := NewDataNode("server1")
+	dn2 := NewDataNode("server2")
+	rack.LinkChildNode(dn1)
+	rack.LinkChildNode(dn2)
+
+	// Server 1: 5 available slots
+	disk1 := NewDisk(types.HardDriveType.String())
+	disk1.diskUsages.getOrCreateDisk(types.HardDriveType).maxVolumeCount = 5
+	dn1.LinkChildNode(disk1)
+
+	// Server 2: 0 available slots (full)
+	disk2 := NewDisk(types.HardDriveType.String())
+	diskUsage2 := disk2.diskUsages.getOrCreateDisk(types.HardDriveType)
+	diskUsage2.maxVolumeCount = 5
+	diskUsage2.volumeCount = 5
+	dn2.LinkChildNode(disk2)
+
+	vg := NewDefaultVolumeGrowth()
+	rp, _ := super_block.NewReplicaPlacementFromString("010") // requires 2 replicas
+
+	option := &VolumeGrowOption{
+		Collection:       "test",
+		ReplicaPlacement: rp,
+		DiskType:         types.HardDriveType,
+	}
+
+	// This should fail because we can't satisfy replica requirements
+	// (need 2 servers but only 1 has space)
+	_, _, err := vg.findEmptySlotsForOneVolume(topo, option, true)
+	if err == nil {
+		t.Error("Expected reservation to fail due to insufficient replica capacity")
+	}
+
+	// Verify no reservations are left hanging
+	available1 := dn1.AvailableSpaceForReservation(option)
+	if available1 != 5 {
+		t.Errorf("Expected server1 to have all capacity available after failed reservation, got %d", available1)
+	}
+
+	available2 := dn2.AvailableSpaceForReservation(option)
+	if available2 != 0 {
+		t.Errorf("Expected server2 to have no capacity available, got %d", available2)
+	}
+}
+
+func TestVolumeGrowth_ReservationTimeout(t *testing.T) {
+	dn := NewDataNode("server1")
+	diskType := types.HardDriveType
+
+	// Set up capacity
+	diskUsage := dn.diskUsages.getOrCreateDisk(diskType)
+	diskUsage.maxVolumeCount = 5
+
+	// Create a reservation
+	reservationId, success := dn.TryReserveCapacity(diskType, 2)
+	if !success {
+		t.Fatal("Expected successful reservation")
+	}
+
+	// Manually set the reservation time to simulate old reservation
+	dn.capacityReservations.Lock()
+	if reservation, exists := dn.capacityReservations.reservations[reservationId]; exists {
+		reservation.createdAt = time.Now().Add(-10 * time.Minute)
+	}
+	dn.capacityReservations.Unlock()
+
+	// Try another reservation - this should trigger cleanup and succeed
+	_, success = dn.TryReserveCapacity(diskType, 3)
+	if !success {
+		t.Error("Expected reservation to succeed after cleanup of expired reservation")
+	}
+
+	// Original reservation should be cleaned up
+	option := &VolumeGrowOption{DiskType: diskType}
+	available := dn.AvailableSpaceForReservation(option)
+	if available != 2 { // 5 - 3 = 2
+		t.Errorf("Expected 2 available slots after cleanup and new reservation, got %d", available)
+	}
+}

weed/topology/volume_growth_test.go

@@ -0,0 +1,514 @@
+package topology
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+)
+
+var topologyLayout = `
+{
+  "dc1":{
+    "rack1":{
+      "server111":{
+        "volumes":[
+          {"id":1, "size":12312},
+          {"id":2, "size":12312},
+          {"id":3, "size":12312}
+        ],
+        "limit":3
+      },
+      "server112":{
+        "volumes":[
+          {"id":4, "size":12312},
+          {"id":5, "size":12312},
+          {"id":6, "size":12312}
+        ],
+        "limit":10
+      }
+    },
+    "rack2":{
+      "server121":{
+        "volumes":[
+          {"id":4, "size":12312},
+          {"id":5, "size":12312},
+          {"id":6, "size":12312}
+        ],
+        "limit":4
+      },
+      "server122":{
+        "volumes":[],
+        "limit":4
+      },
+      "server123":{
+        "volumes":[
+          {"id":2, "size":12312},
+          {"id":3, "size":12312},
+          {"id":4, "size":12312}
+        ],
+        "limit":5
+      }
+    }
+  },
+  "dc2":{
+  },
+  "dc3":{
+    "rack2":{
+      "server321":{
+        "volumes":[
+          {"id":1, "size":12312},
+          {"id":3, "size":12312},
+          {"id":5, "size":12312}
+        ],
+        "limit":4
+      }
+    }
+  }
+}
+`
+
+func setup(topologyLayout string) *Topology {
+	var data interface{}
+	err := json.Unmarshal([]byte(topologyLayout), &data)
+	if err != nil {
+		fmt.Println("error:", err)
+	}
+	fmt.Println("data:", data)
+
+	//need to connect all nodes first before server adding volumes
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	mTopology := data.(map[string]interface{})
+	for dcKey, dcValue := range mTopology {
+		dc := NewDataCenter(dcKey)
+		dcMap := dcValue.(map[string]interface{})
+		topo.LinkChildNode(dc)
+		for rackKey, rackValue := range dcMap {
+			dcRack := NewRack(rackKey)
+			rackMap := rackValue.(map[string]interface{})
+			dc.LinkChildNode(dcRack)
+			for serverKey, serverValue := range rackMap {
+				server := NewDataNode(serverKey)
+				serverMap := serverValue.(map[string]interface{})
+				if ip, ok := serverMap["ip"]; ok {
+					server.Ip = ip.(string)
+				}
+				dcRack.LinkChildNode(server)
+				for _, v := range serverMap["volumes"].([]interface{}) {
+					m := v.(map[string]interface{})
+					vi := storage.VolumeInfo{
+						Id:      needle.VolumeId(int64(m["id"].(float64))),
+						Size:    uint64(m["size"].(float64)),
+						Version: needle.GetCurrentVersion(),
+					}
+					if mVal, ok := m["collection"]; ok {
+						vi.Collection = mVal.(string)
+					}
+					if mVal, ok := m["replication"]; ok {
+						rp, _ := super_block.NewReplicaPlacementFromString(mVal.(string))
+						vi.ReplicaPlacement = rp
+					}
+					if vi.ReplicaPlacement != nil {
+						vl := topo.GetVolumeLayout(vi.Collection, vi.ReplicaPlacement, needle.EMPTY_TTL, types.HardDriveType)
+						vl.RegisterVolume(&vi, server)
+						vl.setVolumeWritable(vi.Id)
+					}
+					server.AddOrUpdateVolume(vi)
+				}
+
+				disk := server.getOrCreateDisk("")
+				disk.UpAdjustDiskUsageDelta("", &DiskUsageCounts{
+					maxVolumeCount: int64(serverMap["limit"].(float64)),
+				})
+
+			}
+		}
+	}
+
+	return topo
+}
+
+func TestFindEmptySlotsForOneVolume(t *testing.T) {
+	topo := setup(topologyLayout)
+	vg := NewDefaultVolumeGrowth()
+	rp, _ := super_block.NewReplicaPlacementFromString("002")
+	volumeGrowOption := &VolumeGrowOption{
+		Collection:       "",
+		ReplicaPlacement: rp,
+		DataCenter:       "dc1",
+		Rack:             "",
+		DataNode:         "",
+	}
+	servers, _, err := vg.findEmptySlotsForOneVolume(topo, volumeGrowOption, false)
+	if err != nil {
+		fmt.Println("finding empty slots error :", err)
+		t.Fail()
+	}
+	for _, server := range servers {
+		fmt.Println("assigned node :", server.Id())
+	}
+}
+
+var topologyLayout2 = `
+{
+  "dc1":{
+    "rack1":{
+      "server111":{
+        "volumes":[
+          {"id":1, "size":12312},
+          {"id":2, "size":12312},
+          {"id":3, "size":12312}
+        ],
+        "limit":300
+      },
+      "server112":{
+        "volumes":[
+          {"id":4, "size":12312},
+          {"id":5, "size":12312},
+          {"id":6, "size":12312}
+        ],
+        "limit":300
+      },
+      "server113":{
+        "volumes":[],
+        "limit":300
+      },
+      "server114":{
+        "volumes":[],
+        "limit":300
+      },
+      "server115":{
+        "volumes":[],
+        "limit":300
+      },
+      "server116":{
+        "volumes":[],
+        "limit":300
+      }
+    },
+    "rack2":{
+      "server121":{
+        "volumes":[
+          {"id":4, "size":12312},
+          {"id":5, "size":12312},
+          {"id":6, "size":12312}
+        ],
+        "limit":300
+      },
+      "server122":{
+        "volumes":[],
+        "limit":300
+      },
+      "server123":{
+        "volumes":[
+          {"id":2, "size":12312},
+          {"id":3, "size":12312},
+          {"id":4, "size":12312}
+        ],
+        "limit":300
+      },
+      "server124":{
+        "volumes":[],
+        "limit":300
+      },
+      "server125":{
+        "volumes":[],
+        "limit":300
+      },
+      "server126":{
+        "volumes":[],
+        "limit":300
+      }
+    },
+    "rack3":{
+      "server131":{
+        "volumes":[],
+        "limit":300
+      },
+      "server132":{
+        "volumes":[],
+        "limit":300
+      },
+      "server133":{
+        "volumes":[],
+        "limit":300
+      },
+      "server134":{
+        "volumes":[],
+        "limit":300
+      },
+      "server135":{
+        "volumes":[],
+        "limit":300
+      },
+      "server136":{
+        "volumes":[],
+        "limit":300
+      }
+    }
+  }
+}
+`
+
+func TestReplication011(t *testing.T) {
+	topo := setup(topologyLayout2)
+	vg := NewDefaultVolumeGrowth()
+	rp, _ := super_block.NewReplicaPlacementFromString("011")
+	volumeGrowOption := &VolumeGrowOption{
+		Collection:       "MAIL",
+		ReplicaPlacement: rp,
+		DataCenter:       "dc1",
+		Rack:             "",
+		DataNode:         "",
+	}
+	servers, _, err := vg.findEmptySlotsForOneVolume(topo, volumeGrowOption, false)
+	if err != nil {
+		fmt.Println("finding empty slots error :", err)
+		t.Fail()
+	}
+	for _, server := range servers {
+		fmt.Println("assigned node :", server.Id())
+	}
+}
+
+var topologyLayout3 = `
+{
+  "dc1":{
+    "rack1":{
+      "server111":{
+        "volumes":[],
+        "limit":2000
+      }
+    }
+  },
+  "dc2":{
+    "rack2":{
+      "server222":{
+        "volumes":[],
+        "limit":2000
+      }
+    }
+  },
+  "dc3":{
+    "rack3":{
+      "server333":{
+        "volumes":[],
+        "limit":1000
+      }
+    }
+  },
+  "dc4":{
+    "rack4":{
+      "server444":{
+        "volumes":[],
+        "limit":1000
+      }
+    }
+  },
+  "dc5":{
+    "rack5":{
+      "server555":{
+        "volumes":[],
+        "limit":500
+      }
+    }
+  },
+  "dc6":{
+    "rack6":{
+      "server666":{
+        "volumes":[],
+        "limit":500
+      }
+    }
+  }
+}
+`
+
+func TestFindEmptySlotsForOneVolumeScheduleByWeight(t *testing.T) {
+	topo := setup(topologyLayout3)
+	vg := NewDefaultVolumeGrowth()
+	rp, _ := super_block.NewReplicaPlacementFromString("100")
+	volumeGrowOption := &VolumeGrowOption{
+		Collection:       "Weight",
+		ReplicaPlacement: rp,
+		DataCenter:       "",
+		Rack:             "",
+		DataNode:         "",
+	}
+
+	distribution := map[NodeId]int{}
+	// assign 1000 volumes
+	for i := 0; i < 1000; i++ {
+		servers, _, err := vg.findEmptySlotsForOneVolume(topo, volumeGrowOption, false)
+		if err != nil {
+			fmt.Println("finding empty slots error :", err)
+			t.Fail()
+		}
+		for _, server := range servers {
+			// fmt.Println("assigned node :", server.Id())
+			if _, ok := distribution[server.id]; !ok {
+				distribution[server.id] = 0
+			}
+			distribution[server.id] += 1
+		}
+	}
+
+	for k, v := range distribution {
+		fmt.Printf("%s : %d\n", k, v)
+	}
+}
+
+var topologyLayout4 = `
+{
+  "dc1":{
+    "rack1":{
+      "serverdc111":{
+		"ip": "127.0.0.1",
+        "volumes":[
+          {"id":1, "size":12312, "collection":"test", "replication":"001"},
+          {"id":2, "size":12312, "collection":"test", "replication":"100"},
+          {"id":4, "size":12312, "collection":"test", "replication":"100"},
+          {"id":6, "size":12312, "collection":"test", "replication":"010"}
+        ],
+        "limit":100
+      }
+    }
+  },
+  "dc2":{
+    "rack1":{
+      "serverdc211":{
+		"ip": "127.0.0.2",
+        "volumes":[
+          {"id":2, "size":12312, "collection":"test", "replication":"100"},
+          {"id":3, "size":12312, "collection":"test", "replication":"010"},
+          {"id":5, "size":12312, "collection":"test", "replication":"001"},
+          {"id":6, "size":12312, "collection":"test", "replication":"010"}
+		],
+        "limit":100
+      }
+    }
+  },
+  "dc3":{
+    "rack1":{
+      "serverdc311":{
+		"ip": "127.0.0.3",
+        "volumes":[
+          {"id":1, "size":12312, "collection":"test", "replication":"001"},
+          {"id":3, "size":12312, "collection":"test", "replication":"010"},
+          {"id":4, "size":12312, "collection":"test", "replication":"100"},
+          {"id":5, "size":12312, "collection":"test", "replication":"001"}
+		],
+        "limit":100
+      }
+    }
+  }
+}
+`
+
+func TestPickForWrite(t *testing.T) {
+	topo := setup(topologyLayout4)
+	volumeGrowOption := &VolumeGrowOption{
+		Collection: "test",
+		DataCenter: "",
+		Rack:       "",
+		DataNode:   "",
+	}
+	VolumeGrowStrategy.Threshold = 0.9
+	for _, rpStr := range []string{"001", "010", "100"} {
+		rp, _ := super_block.NewReplicaPlacementFromString(rpStr)
+		vl := topo.GetVolumeLayout("test", rp, needle.EMPTY_TTL, types.HardDriveType)
+		volumeGrowOption.ReplicaPlacement = rp
+		for _, dc := range []string{"", "dc1", "dc2", "dc3", "dc0"} {
+			volumeGrowOption.DataCenter = dc
+			for _, r := range []string{""} {
+				volumeGrowOption.Rack = r
+				for _, dn := range []string{""} {
+					if dc == "" && dn != "" {
+						continue
+					}
+					volumeGrowOption.DataNode = dn
+					// Small expectedDataSize hint: this test iterates many times
+					// and the layout uses a 32KB volume size limit. With hint=0
+					// (default 1MB pendingDelta), RecordAssign would exceed the
+					// limit on the first call and eagerly remove the volume
+					// from writable. Keep the hint tiny so effectiveSize stays
+					// below the limit across all iterations.
+					fileId, count, _, shouldGrow, err := topo.PickForWrite(1, volumeGrowOption, vl, 1024)
+					if dc == "dc0" {
+						if err == nil || count != 0 || !shouldGrow {
+							fmt.Println(dc, r, dn, "pick for write should be with error")
+							t.Fail()
+						}
+					} else if err != nil {
+						fmt.Println(dc, r, dn, "pick for write error :", err)
+						t.Fail()
+					} else if count == 0 {
+						fmt.Println(dc, r, dn, "pick for write count is zero")
+						t.Fail()
+					} else if len(fileId) == 0 {
+						fmt.Println(dc, r, dn, "pick for write file id is empty")
+						t.Fail()
+					} else if shouldGrow {
+						fmt.Println(dc, r, dn, "pick for write error : not should grow")
+						t.Fail()
+					}
+
+					// Also verify with a non-zero expectedDataSize hint
+					if dc != "dc0" {
+						fileId2, count2, _, shouldGrow2, err2 := topo.PickForWrite(1, volumeGrowOption, vl, 1024)
+						if err2 != nil {
+							fmt.Println(dc, r, dn, "pick for write with size hint error:", err2)
+							t.Fail()
+						} else if count2 == 0 || len(fileId2) == 0 || shouldGrow2 {
+							fmt.Println(dc, r, dn, "pick for write with size hint unexpected result")
+							t.Fail()
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+// TestPickNodesByWeightPrefersDistinctHosts: a rack runs two physical machines
+// with two volume servers each. Picking two nodes (e.g. for a same-rack replica
+// pair) must land them on distinct hosts so a single machine failure cannot take
+// out both replicas, even though the four data nodes are independent.
+func TestPickNodesByWeightPrefersDistinctHosts(t *testing.T) {
+	rack := NewRack("rack1")
+	mk := func(id, ip string, maxVol int64) *DataNode {
+		dn := NewDataNode(id)
+		dn.Ip = ip
+		rack.LinkChildNode(dn)
+		disk := dn.getOrCreateDisk("")
+		disk.UpAdjustDiskUsageDelta("", &DiskUsageCounts{maxVolumeCount: maxVol})
+		return dn
+	}
+	mk("s1a", "10.0.0.1", 10)
+	mk("s1b", "10.0.0.1", 10)
+	mk("s2a", "10.0.0.2", 10)
+	mk("s2b", "10.0.0.2", 10)
+
+	option := &VolumeGrowOption{DiskType: types.HardDriveType}
+
+	// Selection is weighted-random, so exercise it repeatedly: with two machines
+	// available the chosen pair must always span both.
+	for i := 0; i < 50; i++ {
+		first, rest, err := rack.PickNodesByWeight(2, option, func(node Node) error { return nil })
+		if err != nil {
+			t.Fatalf("PickNodesByWeight: %v", err)
+		}
+		if len(rest) != 1 {
+			t.Fatalf("got %d rest nodes, want 1", len(rest))
+		}
+		if nodeHost(first) == nodeHost(rest[0]) {
+			t.Errorf("both picks on host %s; expected distinct machines", nodeHost(first))
+		}
+	}
+}

weed/topology/volume_index_digest.go

@@ -0,0 +1,84 @@
+package topology
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// VolumeIdDigestHash spreads a volume id across the whole 64-bit range so that
+// xoring a set of them together does not collide for the near-consecutive ids a
+// cluster actually allocates.
+func VolumeIdDigestHash(vid needle.VolumeId) uint64 {
+	h := uint64(vid) + 0x9E3779B97F4A7C15
+	h = (h ^ (h >> 30)) * 0xBF58476D1CE4E5B9
+	h = (h ^ (h >> 27)) * 0x94D049BB133111EB
+	return h ^ (h >> 31)
+}
+
+// VolumeIndexDigests returns the digest of the volumes the node's disks hold and
+// the digest of the volumes the lookup index will serve from that node.
+//
+// The master keeps those two indexes separately, and a disconnect racing a
+// reconnect has been seen to drop a volume from the lookup index while leaving
+// it on the node: the volume stays visible in volume.list and the admin UI while
+// LookupVolume answers "volume id not found". A volume server cannot detect that
+// -- its own report is identical either way -- so the master has to notice it
+// on its own rather than rely on the heartbeat digest.
+func (dn *DataNode) VolumeIndexDigests() (held, servable uint64) {
+	dn.RLock()
+	for _, c := range dn.children {
+		held ^= c.(*Disk).VolumeIdDigest()
+	}
+	dn.RUnlock()
+	return held, dn.lookupDigest.Load()
+}
+
+// HasConsistentVolumeIndex reports whether every volume on the node is reachable
+// through the lookup index, and nothing else is.
+func (dn *DataNode) HasConsistentVolumeIndex() bool {
+	held, servable := dn.VolumeIndexDigests()
+	return held == servable
+}
+
+// releaseLookupOwnership clears the lookup digest bit of every location this
+// layout serves. Used when the layout is dropped wholesale with its collection
+// rather than volume by volume.
+func (vl *VolumeLayout) releaseLookupOwnership() {
+	vl.accessLock.Lock()
+	defer vl.accessLock.Unlock()
+	for vid, location := range vl.vid2location {
+		for _, dn := range location.list {
+			moveLookupOwnership(vid, dn, nil)
+		}
+	}
+	vl.vid2location = make(map[needle.VolumeId]*VolumeLocationList)
+	vl.dropped = true
+}
+
+// moveLookupOwnership transfers the digest bit for vid from the node a lookup
+// entry used to name to the node it now names. Either may be nil, for an entry
+// being created or dropped.
+func moveLookupOwnership(vid needle.VolumeId, from, to *DataNode) {
+	if from == to {
+		return
+	}
+	if from != nil {
+		from.trackLookupChange(vid)
+	}
+	if to != nil {
+		to.trackLookupChange(vid)
+	}
+}
+
+// trackLookupChange records that vid became reachable through this node, or
+// stopped being: xor is its own inverse, so both are the same update. Volume
+// layouts for different collections share the node, so it cannot assume the
+// caller's lock.
+func (dn *DataNode) trackLookupChange(vid needle.VolumeId) {
+	delta := VolumeIdDigestHash(vid)
+	for {
+		old := dn.lookupDigest.Load()
+		if dn.lookupDigest.CompareAndSwap(old, old^delta) {
+			return
+		}
+	}
+}

weed/topology/volume_layout_capacity_recovery_test.go

@@ -0,0 +1,332 @@
+package topology
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// A volume removed by the heartbeat capacity path (not RecordAssign) must
+// still recover once it shrinks — only works if SetVolumeCapacityFull stamps
+// fullSince.
+func TestSetVolumeCapacityFullStampsFullSinceAndRecovers(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":4000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	topo, vl := setupPickTest(t, layout, 10000)
+
+	initialActive := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount
+	initialWritables, _ := vl.GetWritableVolumeCount()
+	if initialWritables != 1 {
+		t.Fatalf("expected 1 writable volume initially, got %d", initialWritables)
+	}
+
+	if !vl.SetVolumeCapacityFull(1) {
+		t.Fatalf("SetVolumeCapacityFull should report the volume was writable")
+	}
+	vl.AdjustActiveVolumeCountForFull(1)
+	if w, _ := vl.GetWritableVolumeCount(); w != 0 {
+		t.Fatalf("expected 0 writable after capacity full, got %d", w)
+	}
+	if vl.sizeTracking[1].fullSince.IsZero() {
+		t.Fatalf("expected SetVolumeCapacityFull to stamp fullSince")
+	}
+
+	// No recovery before capacityRecoveryDelay.
+	advanceSizeTrackingClock(vl, 1, 3*time.Second)
+	if vl.UpdateVolumeSize(1, 4000, 0, true) {
+		t.Fatalf("recovery should not fire before capacityRecoveryDelay")
+	}
+
+	// After the delay, a smaller size restores it.
+	advanceSizeTrackingClock(vl, 1, capacityRecoveryDelay)
+	if !vl.UpdateVolumeSize(1, 4000, 0, true) {
+		t.Fatalf("expected volume to recover to writable after shrinking")
+	}
+	vl.AdjustActiveVolumeCountAfterRecovery(1)
+
+	if w, _ := vl.GetWritableVolumeCount(); w != initialWritables {
+		t.Fatalf("expected %d writable after recovery, got %d", initialWritables, w)
+	}
+	if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive {
+		t.Fatalf("expected activeVolumeCount restored to %d, got %d", initialActive, got)
+	}
+	if !vl.sizeTracking[1].fullSince.IsZero() {
+		t.Fatalf("expected fullSince cleared after recovery")
+	}
+}
+
+func TestSetVolumeAvailableRestoresActiveCountForCapacityFullVolume(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":4000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	topo, vl := setupPickTest(t, layout, 10000)
+
+	initialActive := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount
+	if !vl.SetVolumeCapacityFull(1) {
+		t.Fatalf("SetVolumeCapacityFull should report the volume was writable")
+	}
+	vl.AdjustActiveVolumeCountForFull(1)
+	if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive-1 {
+		t.Fatalf("expected activeVolumeCount decremented to %d, got %d", initialActive-1, got)
+	}
+
+	vl.accessLock.RLock()
+	dn := vl.vid2location[1].list[0]
+	vl.accessLock.RUnlock()
+
+	if !vl.SetVolumeAvailable(dn, 1, false, false) {
+		t.Fatalf("SetVolumeAvailable should report the volume became writable")
+	}
+	if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive {
+		t.Fatalf("expected SetVolumeAvailable to restore activeVolumeCount to %d, got %d", initialActive, got)
+	}
+	if !vl.sizeTracking[1].fullSince.IsZero() {
+		t.Fatalf("expected SetVolumeAvailable to clear fullSince after restoring activeVolumeCount")
+	}
+
+	advanceSizeTrackingClock(vl, 1, capacityRecoveryDelay+time.Second)
+	if vl.UpdateVolumeSize(1, 4000, 0, true) {
+		t.Fatalf("heartbeat recovery should not fire after SetVolumeAvailable already restored the volume")
+	}
+	if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive {
+		t.Fatalf("expected activeVolumeCount to remain %d, got %d", initialActive, got)
+	}
+}
+
+// A volume marked full by pending estimates takes no writes, so no heartbeat
+// reports it again and only the periodic decay can bring it back.
+func TestDecayQuietVolumeSizesRecoversPhantomFullVolume(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":4000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	topo, vl := setupPickTest(t, layout, 10000)
+	VolumeGrowStrategy.Threshold = 0.9
+
+	initialActive := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount
+	if !vl.RecordAssign(1, 20000) {
+		t.Fatalf("expected RecordAssign to remove the volume from writable")
+	}
+	vl.AdjustActiveVolumeCountForFull(1)
+	if w, _ := vl.GetWritableVolumeCount(); w != 0 {
+		t.Fatalf("expected 0 writable after the pending estimate filled the volume, got %d", w)
+	}
+
+	recovered := false
+	for i := 0; i < 10 && !recovered; i++ {
+		advanceSizeTrackingClock(vl, 1, capacityRecoveryDelay+time.Second)
+		topo.DecayQuietVolumeSizes()
+		w, _ := vl.GetWritableVolumeCount()
+		recovered = w == 1
+	}
+	if !recovered {
+		t.Fatalf("the periodic decay never returned the volume to the writable list")
+	}
+	if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive {
+		t.Fatalf("expected activeVolumeCount restored to %d, got %d", initialActive, got)
+	}
+	if !vl.sizeTracking[1].fullSince.IsZero() {
+		t.Fatalf("expected fullSince cleared after recovery")
+	}
+}
+
+// The decay must not consume the replica-dedup window: a real report landing
+// right behind it carries a size no volume server will send again, since only
+// a volume whose content changed is reported at all.
+func TestDecayQuietVolumeSizesKeepsTheNextHeartbeat(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":4000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	topo, vl := setupPickTest(t, layout, 10000)
+
+	vl.RecordAssign(1, 2000)
+	advanceSizeTrackingClock(vl, 1, 30*time.Second)
+	topo.DecayQuietVolumeSizes()
+
+	vl.UpdateVolumeSize(1, 6000, 7, true)
+
+	vl.accessLock.RLock()
+	defer vl.accessLock.RUnlock()
+	if got := vl.sizeTracking[1].reportedSize; got != 6000 {
+		t.Errorf("the heartbeat after a decay left reportedSize at %d, want the reported 6000", got)
+	}
+	if got := vl.sizeTracking[1].compactRevision; got != 7 {
+		t.Errorf("the heartbeat after a decay left compactRevision at %d, want the reported 7", got)
+	}
+}
+
+// The decay picks its volumes under a read lock and replays them under a write
+// one. A compaction heartbeat landing in that gap must survive, so the replay
+// takes the record as it stands rather than the size the pass set out with —
+// the arguments below stand in for that older snapshot.
+func TestDecayQuietVolumeSizesDoesNotRollBackAHeartbeat(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":9000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	vl.RecordAssign(1, 500)
+	// The heartbeat that wins the race: a compaction shrank the volume.
+	vl.UpdateVolumeSize(1, 2000, 3, true)
+	// Far enough behind that the dedup window no longer covers for the replay,
+	// which is the only case where reading the record under the lock is what
+	// saves the report.
+	advanceSizeTrackingClock(vl, 1, 3*time.Second)
+
+	vl.UpdateVolumeSize(1, 9000, 0, false)
+
+	vl.accessLock.RLock()
+	defer vl.accessLock.RUnlock()
+	if got := vl.sizeTracking[1].reportedSize; got != 2000 {
+		t.Errorf("the decay rolled reportedSize back to %d, want the compacted 2000", got)
+	}
+	if got := vl.sizeTracking[1].compactRevision; got != 3 {
+		t.Errorf("the decay rolled compactRevision back to %d, want the reported 3", got)
+	}
+	if got := vl.sizeTracking[1].effectiveSize; got > 2000 {
+		t.Errorf("effectiveSize %d outgrew the compacted size the heartbeat reported", got)
+	}
+}
+
+// The decay stands in for a report that never came, so a heartbeat arriving
+// after the pass chose the volume takes its place rather than adding to it:
+// halving twice in one cycle forgets pending bytes the volume has yet to write.
+func TestDecayQuietVolumeSizesYieldsToAHeartbeatItRaced(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":4000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	vl.RecordAssign(1, 4000)
+	advanceSizeTrackingClock(vl, 1, 30*time.Second)
+
+	// The heartbeat wins the race and does the one decay this cycle owes.
+	vl.UpdateVolumeSize(1, 4000, 0, true)
+	vl.accessLock.RLock()
+	afterHeartbeat := vl.sizeTracking[1].effectiveSize
+	vl.accessLock.RUnlock()
+	if afterHeartbeat != 6000 {
+		t.Fatalf("the heartbeat left effectiveSize at %d, want 6000", afterHeartbeat)
+	}
+
+	vl.UpdateVolumeSize(1, 0, 0, false)
+
+	vl.accessLock.RLock()
+	defer vl.accessLock.RUnlock()
+	if got := vl.sizeTracking[1].effectiveSize; got != afterHeartbeat {
+		t.Errorf("the decay halved again to %d, want the heartbeat's %d left alone", got, afterHeartbeat)
+	}
+}
+
+// A volume the disk really did fill can never recover through the decay, so it
+// must leave the candidate set instead of costing a write lock every pulse for
+// the rest of its life.
+func TestDecayQuietVolumeSizesSkipsAGenuinelyFullVolume(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":10000, "replication":"000"},
+          {"id":2, "size":4000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	// Volume 1 is full on disk; volume 2 only looks full to the estimate.
+	vl.SetVolumeCapacityFull(1)
+	vl.RecordAssign(2, 6000)
+	advanceSizeTrackingClock(vl, 1, 30*time.Second)
+	advanceSizeTrackingClock(vl, 2, 30*time.Second)
+
+	candidates := vl.quietDecayCandidates(10 * time.Second)
+	for _, vid := range candidates {
+		if vid == 1 {
+			t.Errorf("the decay keeps taking the write lock for volume 1, which the disk really did fill")
+		}
+	}
+	if len(candidates) != 1 || candidates[0] != 2 {
+		t.Errorf("candidates %v, want only the phantom-full volume 2", candidates)
+	}
+
+	vl.DecayQuietVolumeSizes(10 * time.Second)
+
+	vl.accessLock.RLock()
+	defer vl.accessLock.RUnlock()
+	if got := vl.sizeTracking[2].effectiveSize; got != 7000 {
+		t.Errorf("the phantom-full volume was left at %d, want it decayed to 7000", got)
+	}
+}

weed/topology/volume_layout_drain_test.go

@@ -0,0 +1,261 @@
+package topology
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func TestGetPendingSize(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":1000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	// Initially no pending
+	if p := vl.GetPendingSize(1); p != 0 {
+		t.Fatalf("expected 0 pending, got %d", p)
+	}
+
+	// RecordAssign increases pending
+	vl.RecordAssign(1, 5000)
+	if p := vl.GetPendingSize(1); p != 5000 {
+		t.Fatalf("expected 5000 pending, got %d", p)
+	}
+
+	// UpdateVolumeSize (heartbeat) decays pending
+	vl.UpdateVolumeSize(1, 3000, 0, true)
+	// effective was 6000, reported 3000 → decay to 3000 + (6000-3000)/2 = 4500
+	// pending = 4500 - 3000 = 1500
+	if p := vl.GetPendingSize(1); p != 1500 {
+		t.Fatalf("expected 1500 pending after decay, got %d", p)
+	}
+}
+
+func TestGetPendingSize_CompactionResets(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":5000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	// Add large pending
+	vl.RecordAssign(1, 4000)
+	if p := vl.GetPendingSize(1); p != 4000 {
+		t.Fatalf("expected 4000 pending, got %d", p)
+	}
+
+	// Compaction happens — size drops from 5000 to 2000, revision changes.
+	// Without compaction awareness, decay would give: 2000 + (9000-2000)/2 = 5500.
+	// With compaction awareness, vid2size resets to 2000 (the real size).
+	vl.UpdateVolumeSize(1, 2000, 1, true) // revision 0 → 1
+
+	if p := vl.GetPendingSize(1); p != 0 {
+		t.Errorf("expected 0 pending after compaction reset, got %d", p)
+	}
+
+	// Verify vid2size is the reported size, not a decayed value
+	vl.accessLock.RLock()
+	if vl.sizeTracking[1].effectiveSize != 2000 {
+		t.Errorf("expected vid2size=2000 after compaction, got %d", vl.sizeTracking[1].effectiveSize)
+	}
+	vl.accessLock.RUnlock()
+}
+
+func TestDrainAndRemoveFromWritable_NoPending(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":1000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	// No pending — drain should return immediately
+	start := time.Now()
+	vl.DrainAndRemoveFromWritable(1)
+	if elapsed := time.Since(start); elapsed > 500*time.Millisecond {
+		t.Errorf("drain with no pending took %v, expected near-instant", elapsed)
+	}
+
+	// Verify volume is no longer writable
+	writable, _ := vl.GetWritableVolumeCount()
+	if writable != 0 {
+		t.Errorf("expected 0 writable after drain, got %d", writable)
+	}
+}
+
+func TestDrainAndRemoveFromWritable_WithPending(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":1000, "replication":"000"},
+          {"id":2, "size":1000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	// Add pending below threshold
+	vl.RecordAssign(1, int64(pendingSizeThreshold-1))
+
+	start := time.Now()
+	vl.DrainAndRemoveFromWritable(1)
+	elapsed := time.Since(start)
+
+	// Should return quickly since pending is below threshold
+	if elapsed > 500*time.Millisecond {
+		t.Errorf("drain with pending below threshold took %v", elapsed)
+	}
+
+	// Volume removed from writable
+	writables := vl.CloneWritableVolumes()
+	for _, vid := range writables {
+		if vid == 1 {
+			t.Error("volume 1 should not be writable after drain")
+		}
+	}
+	// Volume 2 still writable
+	found := false
+	for _, vid := range writables {
+		if vid == 2 {
+			found = true
+		}
+	}
+	if !found {
+		t.Error("volume 2 should still be writable")
+	}
+}
+
+func TestDrainAndRemoveFromWritable_DecaysViaConcurrentHeartbeat(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":1000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	// Add large pending (well above threshold)
+	vl.RecordAssign(1, 100*1024*1024) // 100 MB
+
+	// Simulate heartbeats in background that will decay the pending.
+	// Advance lastUpdateTime before each call to bypass the 2s replica dedup.
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		for i := 0; i < 10; i++ {
+			time.Sleep(500 * time.Millisecond)
+			vl.accessLock.Lock()
+			if st := vl.sizeTracking[1]; st != nil {
+				st.lastUpdateTime = time.Time{} // reset to allow update
+			}
+			vl.accessLock.Unlock()
+			vl.UpdateVolumeSize(1, 1000+uint64(i+1)*1000, 0, true)
+		}
+	}()
+
+	start := time.Now()
+	vl.DrainAndRemoveFromWritable(1)
+	elapsed := time.Since(start)
+
+	// Should have drained within a few seconds (heartbeats every 500ms).
+	// Use generous margin for slow CI.
+	if elapsed > 15*time.Second {
+		t.Errorf("drain took %v, expected faster with concurrent heartbeats", elapsed)
+	}
+
+	<-done
+
+	// Verify volume is no longer writable
+	writable, _ := vl.GetWritableVolumeCount()
+	if writable != 0 {
+		t.Errorf("expected 0 writable after drain, got %d", writable)
+	}
+}
+
+func TestSetVolumeReadOnly_PreservesPending(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":1000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	topo := setupWithLimit(t, layout, 10000)
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
+	dn := vl.Lookup(1)[0]
+
+	// Add some pending
+	vl.RecordAssign(1, 5000)
+
+	// SetVolumeReadOnly should succeed immediately (non-blocking)
+	result := vl.SetVolumeReadOnly(dn, 1)
+	if !result {
+		t.Error("expected SetVolumeReadOnly to return true")
+	}
+
+	// Pending is still there (not drained), but volume is readonly
+	writable, _ := vl.GetWritableVolumeCount()
+	if writable != 0 {
+		t.Errorf("expected 0 writable after readonly, got %d", writable)
+	}
+	if p := vl.GetPendingSize(1); p != 5000 {
+		t.Errorf("expected 5000 pending (not drained), got %d", p)
+	}
+}

weed/topology/volume_layout_grow_test.go

@@ -0,0 +1,386 @@
+package topology
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// Reproduces https://github.com/seaweedfs/seaweedfs/issues/8986
+//
+// Topology: 1 DC with 3 racks, replication "010" (different-rack).
+// Each volume has replicas on 2 of the 3 racks.
+// The third rack has no replica, so ShouldGrowVolumesByDcAndRack
+// incorrectly returns true for that rack, causing endless volume creation
+// even though the DC already has plenty of non-crowded writable volumes.
+var topologyLayout3Racks = `
+{
+  "dc_edge":{
+    "rack1":{
+      "server1":{
+        "ip":"10.0.0.1",
+        "volumes":[
+          {"id":1, "size":12312, "replication":"010"},
+          {"id":2, "size":12312, "replication":"010"}
+        ],
+        "limit":30
+      }
+    },
+    "rack2":{
+      "server2":{
+        "ip":"10.0.0.2",
+        "volumes":[
+          {"id":1, "size":12312, "replication":"010"},
+          {"id":2, "size":12312, "replication":"010"}
+        ],
+        "limit":30
+      }
+    },
+    "rack3":{
+      "server3":{
+        "ip":"10.0.0.3",
+        "volumes":[],
+        "limit":30
+      }
+    }
+  }
+}
+`
+
+func TestShouldGrowVolumesByDcAndRack_Issue8986(t *testing.T) {
+	topo := setup(topologyLayout3Racks)
+
+	rp, _ := super_block.NewReplicaPlacementFromString("010")
+	vl := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
+
+	writables := vl.CloneWritableVolumes()
+	if len(writables) != 2 {
+		t.Fatalf("expected 2 writable volumes, got %d", len(writables))
+	}
+
+	// rack1 and rack2 each have replicas of the writable volumes.
+	// ShouldGrowVolumesByDcAndRack should return false for them.
+	if vl.ShouldGrowVolumesByDcAndRack(&writables, "dc_edge", "rack1") {
+		t.Error("rack1 should NOT need growth — it has writable volumes")
+	}
+	if vl.ShouldGrowVolumesByDcAndRack(&writables, "dc_edge", "rack2") {
+		t.Error("rack2 should NOT need growth — it has writable volumes")
+	}
+
+	// rack3 has no replicas. With the old logic, this returns true (should grow),
+	// but since the DC already has non-crowded writable volumes with "010"
+	// replication, growing a new volume is unnecessary — the existing volumes
+	// can serve any write request that doesn't pin to a specific rack.
+	if vl.ShouldGrowVolumesByDcAndRack(&writables, "dc_edge", "rack3") {
+		t.Error("rack3 should NOT need growth — the DC already has non-crowded writable volumes that can serve writes")
+	}
+}
+
+// Topology from https://github.com/seaweedfs/seaweedfs/issues/9832
+// 1 DC, 2 racks, 1 server each, replication 010, one crowded volume (28.65 GB)
+// near the 30 GB size limit.
+var topologyLayout9832 = `
+{
+  "datacenter1":{
+    "rack1":{
+      "node-a":{
+        "ip":"10.0.0.1",
+        "volumes":[ {"id":12, "size":28650, "replication":"010", "collection":"bucket-nexus"} ],
+        "limit":30
+      }
+    },
+    "rack2":{
+      "node-b":{
+        "ip":"10.0.0.2",
+        "volumes":[ {"id":12, "size":28650, "replication":"010", "collection":"bucket-nexus"} ],
+        "limit":30
+      }
+    }
+  }
+}
+`
+
+// Reproduces https://github.com/seaweedfs/seaweedfs/issues/9832
+//
+// A crowded "010" volume made the periodic rack-aware scan return true for
+// every rack in the DC, so it grew once per rack with a hardcoded step of 2 —
+// 2 racks × 2 = 4 logical volumes (8 physical) — and ignored a lowered
+// master.volume_growth.copy_2. PlanRackAwareGrowth now plans a single DC-wide
+// grow capped at copy_N.
+func TestPlanRackAwareGrowth_Issue9832(t *testing.T) {
+	defer restoreCopyCounts(VolumeGrowStrategy.Copy1Count, VolumeGrowStrategy.Copy2Count)
+	VolumeGrowStrategy.Copy2Count = 1 // user's master.volume_growth.copy_2
+
+	topo := setupWithLimit(t, topologyLayout9832, 30000)
+	rp, _ := super_block.NewReplicaPlacementFromString("010")
+	vl := topo.GetVolumeLayout("bucket-nexus", rp, needle.EMPTY_TTL, types.HardDriveType)
+
+	if writables := vl.CloneWritableVolumes(); len(writables) != 1 {
+		t.Fatalf("expected 1 writable volume, got %d", len(writables))
+	}
+
+	// The crowded volume makes both racks report "should grow" — the source of
+	// the old per-rack multiplication.
+	writables := vl.CloneWritableVolumes()
+	if !vl.ShouldGrowVolumesByDcAndRack(&writables, "datacenter1", "rack1") ||
+		!vl.ShouldGrowVolumesByDcAndRack(&writables, "datacenter1", "rack2") {
+		t.Fatal("expected both racks to report should-grow for the crowded volume")
+	}
+
+	plans := vl.PlanRackAwareGrowth(topo.ListDCAndRacks(), 0, 2)
+	if len(plans) != 1 {
+		t.Fatalf("expected 1 DC-wide grow, got %d: %+v", len(plans), plans)
+	}
+	if plans[0].Rack != "" {
+		t.Errorf("expected DC-wide grow (empty rack), got rack %q", plans[0].Rack)
+	}
+	if plans[0].WritableVolumeCount != 1 {
+		t.Errorf("expected copy_2=1 logical volume, got %d", plans[0].WritableVolumeCount)
+	}
+}
+
+// With the default copy_2 the per-event step is preserved (not increased): the
+// fix only removes the per-rack multiplication.
+func TestPlanRackAwareGrowth_DefaultStepNotMultiplied(t *testing.T) {
+	defer restoreCopyCounts(VolumeGrowStrategy.Copy1Count, VolumeGrowStrategy.Copy2Count)
+	VolumeGrowStrategy.Copy2Count = 6 // default
+
+	topo := setupWithLimit(t, topologyLayout9832, 30000)
+	rp, _ := super_block.NewReplicaPlacementFromString("010")
+	vl := topo.GetVolumeLayout("bucket-nexus", rp, needle.EMPTY_TTL, types.HardDriveType)
+
+	plans := vl.PlanRackAwareGrowth(topo.ListDCAndRacks(), 0, 2)
+	if len(plans) != 1 {
+		t.Fatalf("expected 1 DC-wide grow, got %d: %+v", len(plans), plans)
+	}
+	if plans[0].WritableVolumeCount != 2 {
+		t.Errorf("expected step count 2 (min of step 2 and copy_2 6), got %d", plans[0].WritableVolumeCount)
+	}
+}
+
+// A non-crowded "010" volume needs no growth at all.
+func TestPlanRackAwareGrowth_NotCrowdedNoGrowth(t *testing.T) {
+	layout := `
+{
+  "datacenter1":{
+    "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":12, "size":1000, "replication":"010", "collection":"c"} ], "limit":30 } },
+    "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[ {"id":12, "size":1000, "replication":"010", "collection":"c"} ], "limit":30 } }
+  }
+}
+`
+	topo := setupWithLimit(t, layout, 30000)
+	rp, _ := super_block.NewReplicaPlacementFromString("010")
+	vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
+
+	if plans := vl.PlanRackAwareGrowth(topo.ListDCAndRacks(), 0, 2); len(plans) != 0 {
+		t.Fatalf("expected no growth for non-crowded volume, got %+v", plans)
+	}
+}
+
+// Non-rack-spanning replication ("000") still grows per rack: a rack without a
+// writable volume gets its own grow.
+func TestPlanRackAwareGrowth_PerRackForNonRackSpanning(t *testing.T) {
+	layout := `
+{
+  "datacenter1":{
+    "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":1, "size":1000, "replication":"000", "collection":"c"} ], "limit":30 } },
+    "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[], "limit":30 } }
+  }
+}
+`
+	topo := setupWithLimit(t, layout, 30000)
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
+
+	plans := vl.PlanRackAwareGrowth(topo.ListDCAndRacks(), 0, 2)
+	if len(plans) != 1 {
+		t.Fatalf("expected 1 per-rack grow, got %d: %+v", len(plans), plans)
+	}
+	if plans[0].Rack != "rack2" {
+		t.Errorf("expected grow pinned to empty rack2, got %q", plans[0].Rack)
+	}
+}
+
+// lastGrowCount is spread evenly across all grow targets even when DCs have
+// different rack counts: each crowded "000" rack gets ceilDiv(lastGrowCount,
+// totalRacks), so the total matches the request rather than over-growing per DC.
+func TestPlanRackAwareGrowth_EvenDistributionAcrossUnevenDCs(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":1, "size":28650, "replication":"000", "collection":"c"} ], "limit":30 } },
+    "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[ {"id":2, "size":28650, "replication":"000", "collection":"c"} ], "limit":30 } }
+  },
+  "dc2":{
+    "rack3":{ "node-c":{ "ip":"10.0.0.3", "volumes":[ {"id":3, "size":28650, "replication":"000", "collection":"c"} ], "limit":30 } },
+    "rack4":{ "node-d":{ "ip":"10.0.0.4", "volumes":[ {"id":4, "size":28650, "replication":"000", "collection":"c"} ], "limit":30 } },
+    "rack5":{ "node-e":{ "ip":"10.0.0.5", "volumes":[ {"id":5, "size":28650, "replication":"000", "collection":"c"} ], "limit":30 } }
+  }
+}
+`
+	topo := setupWithLimit(t, layout, 30000)
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
+
+	plans := vl.PlanRackAwareGrowth(topo.ListDCAndRacks(), 10, 2)
+	if len(plans) != 5 {
+		t.Fatalf("expected a grow per crowded rack (5), got %d: %+v", len(plans), plans)
+	}
+	total := uint32(0)
+	for _, p := range plans {
+		if p.WritableVolumeCount != 2 { // ceilDiv(10, 5 racks)
+			t.Errorf("expected even per-rack count 2, got %d for %s/%s", p.WritableVolumeCount, p.DataCenter, p.Rack)
+		}
+		total += p.WritableVolumeCount
+	}
+	if total != 10 {
+		t.Errorf("expected total grow 10, got %d", total)
+	}
+}
+
+// A collection pinned to one data center (fs.configure -dataCenter) lives only
+// there, yet the scan used to plan growth for every data center in the
+// topology, spreading the collection's volumes across all of them. Data
+// centers hosting none of the layout's volumes are not the scan's to fill.
+func TestPlanRackAwareGrowth_SkipsDataCentersWithoutLayoutVolumes(t *testing.T) {
+	layout := `
+{
+  "dc1":{ "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":1, "size":%d, "replication":"000", "collection":"pinned"} ], "limit":30 } } },
+  "dc2":{ "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[], "limit":30 } } },
+  "dc3":{ "rack3":{ "node-c":{ "ip":"10.0.0.3", "volumes":[], "limit":30 } } },
+  "dc4":{ "rack4":{ "node-d":{ "ip":"10.0.0.4", "volumes":[], "limit":30 } } }
+}
+`
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+
+	topo := setupWithLimit(t, fmt.Sprintf(layout, 1000), 30000)
+	vl := topo.GetVolumeLayout("pinned", rp, needle.EMPTY_TTL, types.HardDriveType)
+	if plans := vl.PlanRackAwareGrowth(topo.ListDCAndRacks(), 0, 2); len(plans) != 0 {
+		t.Fatalf("healthy dc1 volume: expected no growth, got %+v", plans)
+	}
+
+	topo = setupWithLimit(t, fmt.Sprintf(layout, 28650), 30000)
+	vl = topo.GetVolumeLayout("pinned", rp, needle.EMPTY_TTL, types.HardDriveType)
+	plans := vl.PlanRackAwareGrowth(topo.ListDCAndRacks(), 0, 2)
+	if len(plans) != 1 {
+		t.Fatalf("crowded dc1 volume: expected 1 grow, got %d: %+v", len(plans), plans)
+	}
+	if plans[0].DataCenter != "dc1" || plans[0].Rack != "rack1" {
+		t.Errorf("expected grow pinned to dc1/rack1, got %s/%s", plans[0].DataCenter, plans[0].Rack)
+	}
+}
+
+// The lastGrowCount divisor counts only the racks the scan can actually plan
+// for; racks of skipped data centers would dilute every grow.
+func TestPlanRackAwareGrowth_DivisorCountsOnlyHostingRacks(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":1, "size":28650, "replication":"000", "collection":"pinned"} ], "limit":30 } },
+    "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[ {"id":2, "size":28650, "replication":"000", "collection":"pinned"} ], "limit":30 } }
+  },
+  "dc2":{
+    "rack3":{ "node-c":{ "ip":"10.0.0.3", "volumes":[], "limit":30 } },
+    "rack4":{ "node-d":{ "ip":"10.0.0.4", "volumes":[], "limit":30 } }
+  }
+}
+`
+	topo := setupWithLimit(t, layout, 30000)
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := topo.GetVolumeLayout("pinned", rp, needle.EMPTY_TTL, types.HardDriveType)
+
+	plans := vl.PlanRackAwareGrowth(topo.ListDCAndRacks(), 4, 2)
+	if len(plans) != 2 {
+		t.Fatalf("expected a grow per crowded dc1 rack, got %d: %+v", len(plans), plans)
+	}
+	for _, p := range plans {
+		if p.DataCenter != "dc1" {
+			t.Errorf("expected grows pinned to dc1, got %s/%s", p.DataCenter, p.Rack)
+		}
+		if p.WritableVolumeCount != 2 { // ceilDiv(4, 2 hosting racks), not ceilDiv(4, 4)
+			t.Errorf("expected per-rack count 2, got %d", p.WritableVolumeCount)
+		}
+	}
+}
+
+// The periodic must-grow and crowded growth paths build their request from
+// ToVolumeGrowRequest. For a layout living in exactly one data center the
+// request carries that DC, so those paths do not scatter a pinned collection
+// either; a layout spanning DCs keeps unconstrained growth.
+func TestToVolumeGrowRequest_SingleDataCenterLayout(t *testing.T) {
+	layout := `
+{
+  "dc1":{ "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":1, "size":1000, "replication":"000", "collection":"pinned"} ], "limit":30 } } },
+  "dc2":{ "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[ {"id":2, "size":1000, "replication":"000", "collection":"spread"} ], "limit":30 } } },
+  "dc3":{ "rack3":{ "node-c":{ "ip":"10.0.0.3", "volumes":[ {"id":3, "size":1000, "replication":"000", "collection":"spread"} ], "limit":30 } } }
+}
+`
+	topo := setupWithLimit(t, layout, 30000)
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+
+	pinned := &VolumeLayoutCollection{"pinned", topo.GetVolumeLayout("pinned", rp, needle.EMPTY_TTL, types.HardDriveType)}
+	if vgr := pinned.ToVolumeGrowRequest(); vgr.DataCenter != "dc1" {
+		t.Errorf("expected growth pinned to dc1, got %q", vgr.DataCenter)
+	}
+
+	spread := &VolumeLayoutCollection{"spread", topo.GetVolumeLayout("spread", rp, needle.EMPTY_TTL, types.HardDriveType)}
+	if vgr := spread.ToVolumeGrowRequest(); vgr.DataCenter != "" {
+		t.Errorf("expected unconstrained growth for a multi-DC layout, got %q", vgr.DataCenter)
+	}
+}
+
+// Cross-DC replication can never legitimately live in one data center: a
+// single hosting DC there means the other DCs are down, and pinning growth to
+// the survivor would encode the outage as a placement constraint.
+func TestToVolumeGrowRequest_CrossDCReplicationNeverPinned(t *testing.T) {
+	layout := `
+{
+  "dc1":{ "rack1":{ "node-a":{ "ip":"10.0.0.1", "volumes":[ {"id":1, "size":1000, "replication":"100", "collection":"c"} ], "limit":30 } } },
+  "dc2":{ "rack2":{ "node-b":{ "ip":"10.0.0.2", "volumes":[], "limit":30 } } }
+}
+`
+	topo := setupWithLimit(t, layout, 30000)
+	rp, _ := super_block.NewReplicaPlacementFromString("100")
+
+	vlc := &VolumeLayoutCollection{"c", topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)}
+	if vgr := vlc.ToVolumeGrowRequest(); vgr.DataCenter != "" {
+		t.Errorf("expected unconstrained growth for cross-DC replication, got %q", vgr.DataCenter)
+	}
+}
+
+// Volumes packed to capacity (e.g. by fs.mergeVolumes) go crowded and then
+// unwritable, but stay in the crowded map. ShouldGrowVolumes must count only
+// writable crowded volumes, or those leftovers keep writable <= crowded true
+// forever and every assign-path grow request passes the gate.
+func TestShouldGrowVolumes_UnwritableCrowdedVolumes(t *testing.T) {
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := NewVolumeLayout(rp, needle.EMPTY_TTL, types.HardDriveType, 30000, false)
+
+	vl.accessLock.Lock()
+	vl.setVolumeWritable(1)
+	vl.setVolumeWritable(2)
+	vl.accessLock.Unlock()
+
+	vl.SetVolumeCrowded(1)
+	vl.SetVolumeCapacityFull(1)
+
+	if _, crowded := vl.GetWritableVolumeCount(); crowded != 0 {
+		t.Fatalf("expected 0 writable crowded volumes, got %d", crowded)
+	}
+	if vl.ShouldGrowVolumes() {
+		t.Fatal("volume 2 still has room, growth is not needed")
+	}
+
+	vl.SetVolumeCrowded(2)
+	if !vl.ShouldGrowVolumes() {
+		t.Fatal("every writable volume is crowded, growth is needed")
+	}
+}
+
+func restoreCopyCounts(copy1, copy2 uint32) {
+	VolumeGrowStrategy.Copy1Count = copy1
+	VolumeGrowStrategy.Copy2Count = copy2
+}

weed/topology/volume_layout_oversized_heartbeat_test.go

@@ -0,0 +1,318 @@
+package topology
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// A volume that grew past the limit keeps bouncing between writable and
+// unwritable when the oversized mark is not refreshed on heartbeat: RegisterVolume
+// sets it once, but the delta heartbeat path (ApplyVolumeChanges) never re-sets
+// it, so ensureCorrectWritables sees a stale "not oversized" mark and re-adds
+// the volume to writables on the next heartbeat, while RecordAssign removes it
+// on the next assign. UpdateOversizedState must keep the mark current.
+func TestUpdateOversizedStateKeepsOversizedVolumeUnwritable(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":4000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	// Grow the volume past the limit, as RecordAssign would see it.
+	if !vl.RecordAssign(1, 9000) {
+		t.Fatalf("RecordAssign should report the volume reached capacity")
+	}
+	if w, _ := vl.GetWritableVolumeCount(); w != 0 {
+		t.Fatalf("expected 0 writable after RecordAssign, got %d", w)
+	}
+
+	// A heartbeat that reports the now-oversized volume must refresh the mark
+	// so ensureCorrectWritables does not re-add it to writables. Mirrors the
+	// heartbeat order: refresh, then decay, then correct.
+	dn := vl.vid2location[1].list[0]
+	vi, err := dn.GetVolumesById(1)
+	if err != nil {
+		t.Fatalf("GetVolumesById: %v", err)
+	}
+	oversized := vi
+	oversized.Size = 12000
+	vl.UpdateOversizedState(&oversized, dn)
+	vl.UpdateVolumeSize(1, oversized.Size, 0, true)
+	vl.EnsureCorrectWritables(&oversized)
+
+	if w, _ := vl.GetWritableVolumeCount(); w != 0 {
+		t.Fatalf("expected volume to stay unwritable after heartbeat refresh, got %d writable", w)
+	}
+	if !vl.vid2location[1].AnyOversized() {
+		t.Fatalf("expected AnyOversized to be true after heartbeat refresh")
+	}
+}
+
+// A volume that shrank back under the limit must have its oversized mark
+// cleared by the heartbeat refresh, or it stays locked out of writables
+// forever.
+func TestUpdateOversizedStateClearsMarkWhenVolumeShrinks(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":4000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	dn := vl.vid2location[1].list[0]
+	vi, err := dn.GetVolumesById(1)
+	if err != nil {
+		t.Fatalf("GetVolumesById: %v", err)
+	}
+
+	// Mark oversized via a heartbeat that reports a huge size.
+	big := vi
+	big.Size = 12000
+	vl.UpdateOversizedState(&big, dn)
+	if !vl.vid2location[1].AnyOversized() {
+		t.Fatalf("expected AnyOversized after a huge report")
+	}
+
+	// A later heartbeat with a normal size clears the mark.
+	vl.UpdateOversizedState(&vi, dn)
+	if vl.vid2location[1].AnyOversized() {
+		t.Fatalf("expected AnyOversized cleared after the volume shrank")
+	}
+}
+
+// A volume removed for capacity must not be re-added to writables by
+// ensureCorrectWritables before capacityRecoveryDelay elapses, even after its
+// oversized mark is cleared by a heartbeat that reports a smaller size.
+func TestEnsureCorrectWritablesHonorsRecoveryCooldown(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":4000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	// Remove the volume for capacity, as RecordAssign would.
+	if !vl.RecordAssign(1, 9000) {
+		t.Fatalf("RecordAssign should report the volume reached capacity")
+	}
+	if w, _ := vl.GetWritableVolumeCount(); w != 0 {
+		t.Fatalf("expected 0 writable after RecordAssign, got %d", w)
+	}
+
+	dn := vl.vid2location[1].list[0]
+	vi, err := dn.GetVolumesById(1)
+	if err != nil {
+		t.Fatalf("GetVolumesById: %v", err)
+	}
+
+	// Heartbeat reports the volume shrank back under the limit: the oversized
+	// mark clears and effectiveSize decays, but the volume is still within
+	// capacityRecoveryDelay.
+	vi.Size = 4000
+	vl.UpdateOversizedState(&vi, dn)
+	vl.UpdateVolumeSize(1, vi.Size, 0, true)
+	vl.EnsureCorrectWritables(&vi)
+	if w, _ := vl.GetWritableVolumeCount(); w != 0 {
+		t.Fatalf("expected volume to stay unwritable during the cooldown, got %d writable", w)
+	}
+
+	// After the cooldown, a heartbeat with the shrunken size recovers it.
+	advanceSizeTrackingClock(vl, 1, capacityRecoveryDelay+time.Second)
+	if !vl.UpdateVolumeSize(1, vi.Size, 0, true) {
+		t.Fatalf("expected volume to recover to writable after the cooldown")
+	}
+	vl.EnsureCorrectWritables(&vi)
+	if w, _ := vl.GetWritableVolumeCount(); w != 1 {
+		t.Fatalf("expected volume writable after the cooldown, got %d", w)
+	}
+}
+
+// After the cooldown elapses, a volume whose effective size is still at the
+// limit must not be restored by ensureCorrectWritables: the next assign would
+// remove it again.
+func TestEnsureCorrectWritablesDoesNotRestoreVolumeStillAtLimit(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":8000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	// Remove the volume for capacity, as RecordAssign would. effectiveSize
+	// lands at 13000, past the limit (10000); after the decay against a
+	// reported size of 8000 it stays at 10500, still past the limit.
+	if !vl.RecordAssign(1, 5000) {
+		t.Fatalf("RecordAssign should report the volume reached capacity")
+	}
+	if w, _ := vl.GetWritableVolumeCount(); w != 0 {
+		t.Fatalf("expected 0 writable after RecordAssign, got %d", w)
+	}
+
+	dn := vl.vid2location[1].list[0]
+	vi, err := dn.GetVolumesById(1)
+	if err != nil {
+		t.Fatalf("GetVolumesById: %v", err)
+	}
+
+	// Heartbeat reports a size below the limit: the oversized mark clears,
+	// but the decay only halves the pending estimate, so effectiveSize stays
+	// past the limit and UpdateVolumeSize refuses recovery.
+	vi.Size = 8000
+	vl.UpdateOversizedState(&vi, dn)
+	vl.UpdateVolumeSize(1, vi.Size, 0, true)
+	if vl.vid2location[1].AnyOversized() {
+		t.Fatalf("expected oversized mark cleared after the shrink report")
+	}
+
+	// After the cooldown, the volume is still at the limit:
+	// ensureCorrectWritables must not override UpdateVolumeSize's refusal.
+	advanceSizeTrackingClock(vl, 1, capacityRecoveryDelay+time.Second)
+	vl.EnsureCorrectWritables(&vi)
+	if w, _ := vl.GetWritableVolumeCount(); w != 0 {
+		t.Fatalf("expected volume at the limit to stay unwritable, got %d writable", w)
+	}
+}
+
+// A crowded volume is not a full one: it is above the growth threshold but
+// still has room. One that drops out of writables while a replica is away must
+// come back when the replica returns -- nothing writes to a volume that is not
+// writable, so its size can never fall on its own and the lockout would be
+// permanent.
+func TestEnsureCorrectWritablesRestoresCrowdedVolumeAfterReplicaReturns(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "ip":"10.0.0.1",
+        "volumes":[
+          {"id":1, "size":9500, "replication":"001"}
+        ],
+        "limit":10
+      },
+      "server2":{
+        "ip":"10.0.0.2",
+        "volumes":[
+          {"id":1, "size":9500, "replication":"001"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	topo := setupWithLimit(t, layout, 10000)
+	rp, _ := super_block.NewReplicaPlacementFromString("001")
+	vl := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
+
+	// 9500 is past the growth threshold (9000) but under the limit (10000).
+	vl.UpdateVolumeSize(1, 9500, 0, true)
+	if _, crowded := vl.crowded[1]; !crowded {
+		t.Fatalf("expected the volume to be crowded")
+	}
+	if w, _ := vl.GetWritableVolumeCount(); w != 1 {
+		t.Fatalf("a crowded volume is still writable, got %d writable", w)
+	}
+
+	dn := vl.vid2location[1].list[1]
+	vi, err := dn.GetVolumesById(1)
+	if err != nil {
+		t.Fatalf("GetVolumesById: %v", err)
+	}
+	vl.UnRegisterVolume(&vi, dn)
+	if w, _ := vl.GetWritableVolumeCount(); w != 0 {
+		t.Fatalf("expected 0 writable while a replica is missing, got %d", w)
+	}
+
+	// The replica comes back on the next heartbeat.
+	topo.RegisterVolumeLayout(vi, dn)
+	advanceSizeTrackingClock(vl, 1, 5*time.Second)
+	vl.UpdateOversizedState(&vi, dn)
+	vl.UpdateVolumeSize(1, vi.Size, 0, true)
+	vl.EnsureCorrectWritables(&vi)
+	if w, _ := vl.GetWritableVolumeCount(); w != 1 {
+		t.Fatalf("expected the volume writable again, got %d", w)
+	}
+}
+
+// An incremental heartbeat announces an arrival with a short message that
+// carries no size. Registering from it must not clear the oversized mark the
+// full heartbeat set, or the volume is handed back to the writable list until
+// the next full report.
+func TestIncrementalRegistrationKeepsOversizedMark(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", map[string]uint32{"": 25})
+
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
+
+	// A full heartbeat reports the volume past the 32 KB limit.
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{{
+		Id:               1,
+		Size:             uint64(64 * 1024),
+		ReplicaPlacement: uint32(0),
+		Version:          uint32(needle.GetCurrentVersion()),
+	}}, dn)
+	if w, _ := vl.GetWritableVolumeCount(); w != 0 {
+		t.Fatalf("expected the oversized volume out of writables, got %d", w)
+	}
+
+	// The same volume is announced again as an arrival.
+	topo.IncrementalSyncDataNodeRegistration([]*master_pb.VolumeShortInformationMessage{{
+		Id:               1,
+		ReplicaPlacement: uint32(0),
+		Version:          uint32(needle.GetCurrentVersion()),
+	}}, nil, dn)
+
+	if !vl.vid2location[1].AnyOversized() {
+		t.Fatalf("expected the oversized mark to survive the arrival announcement")
+	}
+	if w, _ := vl.GetWritableVolumeCount(); w != 0 {
+		t.Fatalf("expected the oversized volume to stay out of writables, got %d", w)
+	}
+}

weed/topology/volume_layout_pick_bench_test.go

@@ -0,0 +1,134 @@
+package topology
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+var benchLayoutSmall = `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":2000, "replication":"000"},
+          {"id":2, "size":5000, "replication":"000"},
+          {"id":3, "size":8000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+
+var benchLayoutMedium = `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1,  "size":1000, "replication":"000"},
+          {"id":2,  "size":2000, "replication":"000"},
+          {"id":3,  "size":3000, "replication":"000"},
+          {"id":4,  "size":4000, "replication":"000"},
+          {"id":5,  "size":5000, "replication":"000"},
+          {"id":6,  "size":6000, "replication":"000"},
+          {"id":7,  "size":7000, "replication":"000"},
+          {"id":8,  "size":8000, "replication":"000"},
+          {"id":9,  "size":9000, "replication":"000"},
+          {"id":10, "size":9500, "replication":"000"}
+        ],
+        "limit":20
+      }
+    }
+  }
+}
+`
+
+var benchLayoutLarge = `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1,  "size":500,  "replication":"000"},
+          {"id":2,  "size":1000, "replication":"000"},
+          {"id":3,  "size":1500, "replication":"000"},
+          {"id":4,  "size":2000, "replication":"000"},
+          {"id":5,  "size":2500, "replication":"000"},
+          {"id":6,  "size":3000, "replication":"000"},
+          {"id":7,  "size":3500, "replication":"000"},
+          {"id":8,  "size":4000, "replication":"000"},
+          {"id":9,  "size":4500, "replication":"000"},
+          {"id":10, "size":5000, "replication":"000"},
+          {"id":11, "size":5500, "replication":"000"},
+          {"id":12, "size":6000, "replication":"000"},
+          {"id":13, "size":6500, "replication":"000"},
+          {"id":14, "size":7000, "replication":"000"},
+          {"id":15, "size":7500, "replication":"000"},
+          {"id":16, "size":8000, "replication":"000"},
+          {"id":17, "size":8500, "replication":"000"},
+          {"id":18, "size":9000, "replication":"000"},
+          {"id":19, "size":9200, "replication":"000"},
+          {"id":20, "size":9500, "replication":"000"}
+        ],
+        "limit":30
+      }
+    }
+  }
+}
+`
+
+func benchPickForWrite(b *testing.B, layout string, volumeSizeLimit uint64) {
+	topo := setupWithLimit(b, layout, volumeSizeLimit)
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
+	option := &VolumeGrowOption{}
+
+	b.ResetTimer()
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		vl.PickForWrite(1, option)
+	}
+}
+
+func BenchmarkPickForWrite_3Volumes(b *testing.B) {
+	benchPickForWrite(b, benchLayoutSmall, 10000)
+}
+
+func BenchmarkPickForWrite_10Volumes(b *testing.B) {
+	benchPickForWrite(b, benchLayoutMedium, 10000)
+}
+
+func BenchmarkPickForWrite_20Volumes(b *testing.B) {
+	benchPickForWrite(b, benchLayoutLarge, 10000)
+}
+
+func benchPickForWriteConstrained(b *testing.B, layout string, volumeSizeLimit uint64) {
+	topo := setupWithLimit(b, layout, volumeSizeLimit)
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
+	option := &VolumeGrowOption{DataCenter: "dc1"}
+
+	b.ResetTimer()
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		vl.PickForWrite(1, option)
+	}
+}
+
+func BenchmarkPickForWriteConstrained_3Volumes(b *testing.B) {
+	benchPickForWriteConstrained(b, benchLayoutSmall, 10000)
+}
+
+func BenchmarkPickForWriteConstrained_10Volumes(b *testing.B) {
+	benchPickForWriteConstrained(b, benchLayoutMedium, 10000)
+}
+
+func BenchmarkPickForWriteConstrained_20Volumes(b *testing.B) {
+	benchPickForWriteConstrained(b, benchLayoutLarge, 10000)
+}

weed/topology/volume_layout_pick_test.go

@@ -0,0 +1,786 @@
+package topology
+
+import (
+	"encoding/json"
+	"math"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+// setupWithLimit is like setup() but allows specifying the volumeSizeLimit
+// so that VolumeLayouts are created with the correct limit from the start.
+func setupWithLimit(t testing.TB, topologyLayout string, volumeSizeLimit uint64) *Topology {
+	t.Helper()
+	var data interface{}
+	if err := json.Unmarshal([]byte(topologyLayout), &data); err != nil {
+		t.Fatalf("setupWithLimit: json.Unmarshal: %v", err)
+	}
+	mTopology, ok := data.(map[string]interface{})
+	if !ok {
+		t.Fatalf("setupWithLimit: expected map[string]interface{}, got %T", data)
+	}
+
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), volumeSizeLimit, 5, false)
+	for dcKey, dcValue := range mTopology {
+		dc := NewDataCenter(dcKey)
+		dcMap := dcValue.(map[string]interface{})
+		topo.LinkChildNode(dc)
+		for rackKey, rackValue := range dcMap {
+			dcRack := NewRack(rackKey)
+			rackMap := rackValue.(map[string]interface{})
+			dc.LinkChildNode(dcRack)
+			for serverKey, serverValue := range rackMap {
+				server := NewDataNode(serverKey)
+				serverMap := serverValue.(map[string]interface{})
+				if ip, ok := serverMap["ip"]; ok {
+					server.Ip = ip.(string)
+				}
+				dcRack.LinkChildNode(server)
+				for _, v := range serverMap["volumes"].([]interface{}) {
+					m := v.(map[string]interface{})
+					vi := storage.VolumeInfo{
+						Id:      needle.VolumeId(int64(m["id"].(float64))),
+						Size:    uint64(m["size"].(float64)),
+						Version: needle.GetCurrentVersion(),
+					}
+					if mVal, ok := m["fileCount"]; ok {
+						vi.FileCount = uint32(mVal.(float64))
+					}
+					if mVal, ok := m["collection"]; ok {
+						vi.Collection = mVal.(string)
+					}
+					if mVal, ok := m["replication"]; ok {
+						rp, _ := super_block.NewReplicaPlacementFromString(mVal.(string))
+						vi.ReplicaPlacement = rp
+					}
+					if vi.ReplicaPlacement != nil {
+						vl := topo.GetVolumeLayout(vi.Collection, vi.ReplicaPlacement, needle.EMPTY_TTL, types.HardDriveType)
+						vl.RegisterVolume(&vi, server)
+						vl.setVolumeWritable(vi.Id)
+					}
+					server.AddOrUpdateVolume(vi)
+				}
+				disk := server.getOrCreateDisk("")
+				disk.UpAdjustDiskUsageDelta("", &DiskUsageCounts{
+					maxVolumeCount: int64(serverMap["limit"].(float64)),
+				})
+			}
+		}
+	}
+	return topo
+}
+
+func setupPickTest(t testing.TB, layout string, volumeSizeLimit uint64) (*Topology, *VolumeLayout) {
+	t.Helper()
+	topo := setupWithLimit(t, layout, volumeSizeLimit)
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	vl := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
+	return topo, vl
+}
+
+func TestPickForWriteWeightedDistribution(t *testing.T) {
+	// 3 volumes at 20%, 50%, 80% full (sizes 2000, 5000, 8000 of limit 10000)
+	// remaining: 8000, 5000, 2000 => ratios ~53%, 33%, 13%
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":2000, "replication":"000"},
+          {"id":2, "size":5000, "replication":"000"},
+          {"id":3, "size":8000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	counts := make(map[needle.VolumeId]int)
+	option := &VolumeGrowOption{}
+	n := 60000
+
+	for i := 0; i < n; i++ {
+		vid, _, _, _, err := vl.PickForWrite(1, option)
+		if err != nil {
+			t.Fatalf("PickForWrite: %v", err)
+		}
+		counts[vid]++
+	}
+
+	// vid 1 (remaining 8000) > vid 2 (remaining 5000) > vid 3 (remaining 2000)
+	if counts[1] <= counts[3] {
+		t.Errorf("expected vid 1 picked more than vid 3: vid1=%d, vid3=%d", counts[1], counts[3])
+	}
+	if counts[2] <= counts[3] {
+		t.Errorf("expected vid 2 picked more than vid 3: vid2=%d, vid3=%d", counts[2], counts[3])
+	}
+
+	// Check proportions: expected 8000/5000/2000 out of 15000
+	expected := map[needle.VolumeId]float64{
+		1: 8000.0 / 15000.0,
+		2: 5000.0 / 15000.0,
+		3: 2000.0 / 15000.0,
+	}
+	for vid, expectedPct := range expected {
+		actualPct := float64(counts[vid]) / float64(n)
+		if math.Abs(actualPct-expectedPct) > 0.03 {
+			t.Errorf("vid %d: expected ~%.1f%%, got %.1f%%", vid, expectedPct*100, actualPct*100)
+		}
+	}
+}
+
+func TestPickForWriteWithPendingSize(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":1000, "replication":"000"},
+          {"id":2, "size":1000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	// Add large pending to vid 1, making it effectively 9000/10000
+	vl.RecordAssign(1, 8000)
+
+	counts := make(map[needle.VolumeId]int)
+	option := &VolumeGrowOption{}
+	n := 10000
+
+	for i := 0; i < n; i++ {
+		vid, _, _, _, err := vl.PickForWrite(1, option)
+		if err != nil {
+			t.Fatalf("PickForWrite: %v", err)
+		}
+		counts[vid]++
+	}
+
+	// vid 2 (remaining ~9000) should be picked much more than vid 1 (remaining ~1000)
+	ratio := float64(counts[2]) / float64(counts[1])
+	if ratio < 3.0 {
+		t.Errorf("vid2/vid1 ratio %.2f expected >= 3.0 (vid1=%d, vid2=%d)", ratio, counts[1], counts[2])
+	}
+}
+
+// A flat 1MB charge per hintless file id overcharges a small-file workload by
+// orders of magnitude, marking volumes full while they hold a fraction of the
+// limit.
+func TestPickForWriteEstimatesPendingSizeFromVolumeAverage(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":409600, "fileCount":100, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	topo, vl := setupPickTest(t, layout, 30*1024*1024)
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	option := &VolumeGrowOption{ReplicaPlacement: rp}
+
+	pendingAfterAssign := func(expectedDataSize uint64) uint64 {
+		vl.accessLock.RLock()
+		before := vl.sizeTracking[1].effectiveSize
+		vl.accessLock.RUnlock()
+		if _, _, _, _, err := topo.PickForWrite(1, option, vl, expectedDataSize); err != nil {
+			t.Fatalf("PickForWrite: %v", err)
+		}
+		vl.accessLock.RLock()
+		defer vl.accessLock.RUnlock()
+		return vl.sizeTracking[1].effectiveSize - before
+	}
+
+	if got := pendingAfterAssign(0); got != 4096 {
+		t.Errorf("a hintless assign charged %d, want the volume's 4096-byte average", got)
+	}
+	if got := pendingAfterAssign(8192); got != 8192 {
+		t.Errorf("a hinted assign charged %d, want the 8192-byte hint to win over the average", got)
+	}
+}
+
+// A volume with no files yet has no average to draw on, so the 1MB fallback
+// still applies.
+func TestPickForWriteEstimateFallsBackWithoutHistory(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":0, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	topo, vl := setupPickTest(t, layout, 30*1024*1024)
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	option := &VolumeGrowOption{ReplicaPlacement: rp}
+
+	if _, _, _, _, err := topo.PickForWrite(1, option, vl, 0); err != nil {
+		t.Fatalf("PickForWrite: %v", err)
+	}
+	vl.accessLock.RLock()
+	pending := vl.sizeTracking[1].effectiveSize - vl.sizeTracking[1].reportedSize
+	vl.accessLock.RUnlock()
+	if pending != DefaultNeedleSizeEstimate {
+		t.Errorf("a hintless assign on an empty volume charged %d, want the %d fallback", pending, DefaultNeedleSizeEstimate)
+	}
+}
+
+func TestPickForWriteSingleWritable(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":5000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	option := &VolumeGrowOption{}
+	for i := 0; i < 100; i++ {
+		vid, _, _, _, err := vl.PickForWrite(1, option)
+		if err != nil {
+			t.Fatalf("PickForWrite: %v", err)
+		}
+		if vid != 1 {
+			t.Fatalf("expected vid 1, got %d", vid)
+		}
+	}
+}
+
+func TestPickForWriteAllNearFull(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":9999, "replication":"000"},
+          {"id":2, "size":9999, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	option := &VolumeGrowOption{}
+	for i := 0; i < 100; i++ {
+		vid, _, _, _, err := vl.PickForWrite(1, option)
+		if err != nil {
+			t.Fatalf("PickForWrite: %v", err)
+		}
+		if vid != 1 && vid != 2 {
+			t.Fatalf("expected vid 1 or 2, got %d", vid)
+		}
+	}
+}
+
+func TestPickForWriteConstrainedWeighted(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "ip":"10.0.0.1",
+        "volumes":[
+          {"id":1, "size":2000, "replication":"000"},
+          {"id":2, "size":8000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    },
+    "rack2":{
+      "server2":{
+        "ip":"10.0.0.2",
+        "volumes":[
+          {"id":3, "size":5000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	counts := make(map[needle.VolumeId]int)
+	option := &VolumeGrowOption{DataCenter: "dc1"}
+	n := 20000
+
+	for i := 0; i < n; i++ {
+		vid, _, _, _, err := vl.PickForWrite(1, option)
+		if err != nil {
+			t.Fatalf("PickForWrite: %v", err)
+		}
+		counts[vid]++
+	}
+
+	// vid 1 (remaining 8000) should be picked most, vid 2 (remaining 2000) least
+	if counts[1] <= counts[2] {
+		t.Errorf("expected vid 1 picked more than vid 2: vid1=%d, vid2=%d", counts[1], counts[2])
+	}
+}
+
+func TestRecordAssignMarksCrowded(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":8500, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	// Volume at 85% — not crowded yet (threshold is 90%)
+	_, crowded := vl.GetWritableVolumeCount()
+	if crowded != 0 {
+		t.Fatalf("expected 0 crowded, got %d", crowded)
+	}
+
+	// Add pending that pushes past 90%
+	vl.RecordAssign(1, 1000)
+
+	_, crowded = vl.GetWritableVolumeCount()
+	if crowded != 1 {
+		t.Errorf("expected 1 crowded after pending push past 90%%, got %d", crowded)
+	}
+}
+
+func TestRecordAssignReachingCapacityRemovesFromWritable(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":5000, "replication":"000"},
+          {"id":2, "size":5000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	topo, vl := setupPickTest(t, layout, 10000)
+
+	writable, _ := vl.GetWritableVolumeCount()
+	if writable != 2 {
+		t.Fatalf("expected 2 writable volumes initially, got %d", writable)
+	}
+	// Each volume counts as active initially.
+	initialActive := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount
+
+	// Push vid 1 past the hard limit (5000 + 5000 = 10000 == limit).
+	reachedCapacity := vl.RecordAssign(1, 5000)
+	if !reachedCapacity {
+		t.Fatalf("RecordAssign should return true when effectiveSize reaches limit")
+	}
+	vl.AdjustActiveVolumeCountForFull(1)
+
+	writable, _ = vl.GetWritableVolumeCount()
+	if writable != 1 {
+		t.Errorf("expected 1 writable after eager removal, got %d", writable)
+	}
+	// activeVolumeCount should be decremented for the data node holding vid 1.
+	afterActive := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount
+	if afterActive != initialActive-1 {
+		t.Errorf("expected activeVolumeCount=%d, got %d", initialActive-1, afterActive)
+	}
+
+	// A second RecordAssign on the already-removed volume should not return
+	// true again (no double accounting).
+	if vl.RecordAssign(1, 10000) {
+		t.Errorf("RecordAssign should not report reachedCapacity twice for the same removal")
+	}
+	afterSecond := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount
+	if afterSecond != afterActive {
+		t.Errorf("activeVolumeCount changed on second RecordAssign: before=%d after=%d", afterActive, afterSecond)
+	}
+}
+
+// advanceSizeTrackingClock backdates a volume's time-sensitive fields by d
+// so heartbeat decay and the recovery delay fire on the next update.
+func advanceSizeTrackingClock(vl *VolumeLayout, vid needle.VolumeId, d time.Duration) {
+	vl.accessLock.Lock()
+	defer vl.accessLock.Unlock()
+	st := vl.sizeTracking[vid]
+	if st == nil {
+		return
+	}
+	if !st.lastUpdateTime.IsZero() {
+		st.lastUpdateTime = st.lastUpdateTime.Add(-d)
+	}
+	if !st.fullSince.IsZero() {
+		st.fullSince = st.fullSince.Add(-d)
+	}
+}
+
+func TestUpdateVolumeSizeRecoversEagerlyRemovedVolume(t *testing.T) {
+	// Two writable volumes, each at 40% of a 10000-byte limit. Push vid 1
+	// past the hard limit via RecordAssign, then heartbeat with an
+	// unchanged reported size so decay shrinks effectiveSize below the
+	// crowded threshold (90% of limit). After the recovery delay, the
+	// volume should be re-added to writables and activeVolumeCount
+	// restored.
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":4000, "replication":"000"},
+          {"id":2, "size":4000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	topo, vl := setupPickTest(t, layout, 10000)
+
+	initialActive := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount
+	initialWritables, _ := vl.GetWritableVolumeCount()
+
+	// Push vid 1 past the limit (effective = 4000 + 6000 = 10000).
+	if !vl.RecordAssign(1, 6000) {
+		t.Fatalf("RecordAssign should return true at the limit")
+	}
+	vl.AdjustActiveVolumeCountForFull(1)
+
+	w, _ := vl.GetWritableVolumeCount()
+	if w != initialWritables-1 {
+		t.Fatalf("expected %d writables after eager removal, got %d", initialWritables-1, w)
+	}
+
+	// Before the recovery delay, a heartbeat that lets decay run should
+	// *not* re-add the volume (even though effectiveSize would now be
+	// under the threshold).
+	advanceSizeTrackingClock(vl, 1, 3*time.Second) // past the 2s dedup window, but before 30s delay
+	if vl.UpdateVolumeSize(1, 4000, 0, true) {
+		t.Fatalf("recovery should not fire before capacityRecoveryDelay")
+	}
+	w, _ = vl.GetWritableVolumeCount()
+	if w != initialWritables-1 {
+		t.Errorf("writable count should not change before delay, got %d", w)
+	}
+
+	// Now skip past the recovery delay. Decay the gap further until
+	// effectiveSize drops below the crowded threshold (9000).
+	for i := 0; i < 6; i++ {
+		advanceSizeTrackingClock(vl, 1, 10*time.Second)
+		recovered := vl.UpdateVolumeSize(1, 4000, 0, true)
+		if recovered {
+			vl.AdjustActiveVolumeCountAfterRecovery(1)
+			break
+		}
+	}
+
+	w, _ = vl.GetWritableVolumeCount()
+	if w != initialWritables {
+		t.Errorf("expected recovery to restore %d writables, got %d", initialWritables, w)
+	}
+	if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive {
+		t.Errorf("expected activeVolumeCount restored to %d, got %d", initialActive, got)
+	}
+
+	// fullSince should have been cleared so a subsequent heartbeat doesn't
+	// try to recover again.
+	advanceSizeTrackingClock(vl, 1, 60*time.Second)
+	if vl.UpdateVolumeSize(1, 4000, 0, true) {
+		t.Errorf("recovery should not re-fire after the volume is already writable")
+	}
+}
+
+func TestUpdateVolumeSizeNoRecoveryWhenDiskStillOversized(t *testing.T) {
+	// Volume reported at 95% of limit and pushed past limit by RecordAssign.
+	// Even after the recovery delay and decay, reportedSize remains >= limit
+	// (real on-disk size is over limit), so recovery must not fire.
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":9500, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	if !vl.RecordAssign(1, 500) {
+		t.Fatalf("RecordAssign should hit the limit (9500 + 500 = 10000)")
+	}
+	vl.AdjustActiveVolumeCountForFull(1)
+
+	// Plenty of time elapsed — but reported stays at 10500 (over limit).
+	for i := 0; i < 5; i++ {
+		advanceSizeTrackingClock(vl, 1, 10*time.Second)
+		if vl.UpdateVolumeSize(1, 10500, 0, true) {
+			t.Fatalf("recovery must not fire when reported >= limit")
+		}
+	}
+	w, _ := vl.GetWritableVolumeCount()
+	if w != 0 {
+		t.Errorf("expected 0 writables (volume legitimately full), got %d", w)
+	}
+}
+
+func TestHeartbeatDecaysPendingSize(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":1000, "replication":"000"},
+          {"id":2, "size":1000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	// vid2size starts at 1000 (reported). Add 8000 pending → 9000.
+	vl.RecordAssign(1, 8000)
+
+	vl.accessLock.RLock()
+	if vl.sizeTracking[1].effectiveSize != 9000 {
+		t.Fatalf("expected vid2size=9000 after RecordAssign, got %d", vl.sizeTracking[1].effectiveSize)
+	}
+	vl.accessLock.RUnlock()
+
+	// Helper to simulate a new heartbeat cycle (advance past dedup window)
+	advanceCycle := func() {
+		vl.accessLock.Lock()
+		vl.sizeTracking[1].lastUpdateTime = time.Now().Add(-3 * time.Second)
+		vl.accessLock.Unlock()
+	}
+
+	// Heartbeat: volume server reports size=3000 (some writes landed).
+	// Old effective=9000, new reported=3000 → excess=6000 → decayed to 3000.
+	// So vid2size should become 3000 + 6000/2 = 6000, not just 3000.
+	vl.UpdateVolumeSize(1, 3000, 0, true)
+
+	vl.accessLock.RLock()
+	if vl.sizeTracking[1].effectiveSize != 6000 {
+		t.Errorf("expected vid2size=6000 after decay (3000 + 6000/2), got %d", vl.sizeTracking[1].effectiveSize)
+	}
+	vl.accessLock.RUnlock()
+
+	// Second heartbeat: size=5000. Old effective=6000 → excess=1000 → decay to 500.
+	// vid2size should become 5000 + 1000/2 = 5500.
+	advanceCycle()
+	vl.UpdateVolumeSize(1, 5000, 0, true)
+
+	vl.accessLock.RLock()
+	if vl.sizeTracking[1].effectiveSize != 5500 {
+		t.Errorf("expected vid2size=5500 after second decay (5000 + 1000/2), got %d", vl.sizeTracking[1].effectiveSize)
+	}
+	vl.accessLock.RUnlock()
+
+	// Third heartbeat: size=5500. Old effective=5500 → no excess.
+	// vid2size should be exactly 5500.
+	advanceCycle()
+	vl.UpdateVolumeSize(1, 5500, 0, true)
+
+	vl.accessLock.RLock()
+	if vl.sizeTracking[1].effectiveSize != 5500 {
+		t.Errorf("expected vid2size=5500 (no excess), got %d", vl.sizeTracking[1].effectiveSize)
+	}
+	vl.accessLock.RUnlock()
+
+	// vid 2 (remaining 9000) should be picked more than vid 1 (remaining 4500)
+	counts := make(map[needle.VolumeId]int)
+	option := &VolumeGrowOption{}
+	for i := 0; i < 10000; i++ {
+		vid, _, _, _, err := vl.PickForWrite(1, option)
+		if err != nil {
+			t.Fatalf("PickForWrite: %v", err)
+		}
+		counts[vid]++
+	}
+	if counts[2] <= counts[1] {
+		t.Errorf("vid 2 (remaining 9000) should be picked more than vid 1 (remaining 4500): vid1=%d, vid2=%d", counts[1], counts[2])
+	}
+}
+
+func TestHeartbeatDecayDedupReplicas(t *testing.T) {
+	// Volume 1 replicated on server1 and server2.
+	// Both servers report size=3000 in the same heartbeat cycle.
+	// Decay should run only once, not once per replica.
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "ip":"10.0.0.1",
+        "volumes":[
+          {"id":1, "size":1000, "replication":"001"}
+        ],
+        "limit":10
+      },
+      "server2":{
+        "ip":"10.0.0.2",
+        "volumes":[
+          {"id":1, "size":1000, "replication":"001"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	topo := setupWithLimit(t, layout, 10000)
+	rp, _ := super_block.NewReplicaPlacementFromString("001")
+	vl := topo.GetVolumeLayout("", rp, needle.EMPTY_TTL, types.HardDriveType)
+
+	// Add pending: effective = 1000 + 8000 = 9000
+	vl.RecordAssign(1, 8000)
+
+	vl.accessLock.RLock()
+	if vl.sizeTracking[1].effectiveSize != 9000 {
+		t.Fatalf("expected vid2size=9000, got %d", vl.sizeTracking[1].effectiveSize)
+	}
+	vl.accessLock.RUnlock()
+
+	// Both replicas report size=3000. Decay should happen once: 3000 + (9000-3000)/2 = 6000.
+	// Calling UpdateVolumeSize twice simulates two replicas reporting in the same cycle.
+	vl.UpdateVolumeSize(1, 3000, 0, true)
+	vl.UpdateVolumeSize(1, 3000, 0, true) // second replica, same size — should be a no-op
+
+	vl.accessLock.RLock()
+	got := vl.sizeTracking[1].effectiveSize
+	vl.accessLock.RUnlock()
+	// Without dedup: would be 3000 + (6000-3000)/2 = 4500 (double decay).
+	// With dedup: should be 6000 (single decay).
+	if got != 6000 {
+		t.Errorf("expected vid2size=6000 (single decay), got %d (double decay would give 4500)", got)
+	}
+}
+
+func TestUpdateVolumeSize_DecaysEvenWhenReportedSizeUnchanged(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "volumes":[
+          {"id":1, "size":1000, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	// Add pending: effective = 1000 + 8000 = 9000
+	vl.RecordAssign(1, 8000)
+	if p := vl.GetPendingSize(1); p != 8000 {
+		t.Fatalf("expected 8000 pending, got %d", p)
+	}
+
+	// First heartbeat: reported size unchanged at 1000 (writes haven't landed).
+	// Decay should still run: 1000 + (9000-1000)/2 = 5000.
+	vl.UpdateVolumeSize(1, 1000, 0, true)
+	if p := vl.GetPendingSize(1); p != 4000 {
+		t.Errorf("expected 4000 pending after first decay, got %d", p)
+	}
+
+	// Simulate next heartbeat cycle (>2s later) with same reported size.
+	// Need to advance lastUpdateTime — manipulate directly under lock.
+	vl.accessLock.Lock()
+	vl.sizeTracking[1].lastUpdateTime = time.Now().Add(-3 * time.Second)
+	vl.accessLock.Unlock()
+
+	// Second heartbeat: still 1000. Decay again: 1000 + (5000-1000)/2 = 3000.
+	vl.UpdateVolumeSize(1, 1000, 0, true)
+	if p := vl.GetPendingSize(1); p != 2000 {
+		t.Errorf("expected 2000 pending after second decay, got %d", p)
+	}
+}
+
+func TestShouldGrowVolumesByDcAndRack_WithPendingSize(t *testing.T) {
+	layout := `
+{
+  "dc1":{
+    "rack1":{
+      "server1":{
+        "ip":"10.0.0.1",
+        "volumes":[
+          {"id":1, "size":8500, "replication":"000"}
+        ],
+        "limit":10
+      }
+    }
+  }
+}
+`
+	_, vl := setupPickTest(t, layout, 10000)
+
+	writables := vl.CloneWritableVolumes()
+	if vl.ShouldGrowVolumesByDcAndRack(&writables, "dc1", "rack1") {
+		t.Error("should not grow before pending makes volume crowded")
+	}
+
+	// Add pending that pushes effective size past 9000 threshold
+	vl.RecordAssign(1, 600)
+
+	if !vl.ShouldGrowVolumesByDcAndRack(&writables, "dc1", "rack1") {
+		t.Error("should grow after pending pushes volume past crowded threshold")
+	}
+}

weed/topology/volume_layout_size_tracking_test.go

@@ -0,0 +1,163 @@
+package topology
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func sizeTrackingLayout(t *testing.T) (*Topology, *DataNode, *VolumeLayout) {
+	t.Helper()
+	topo := NewTopology("st", nil, 32*1024, 5, false)
+	dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
+	rp, _ := super_block.NewReplicaPlacementFromString("000")
+	return topo, dn, topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
+}
+
+func sizeTrackingVolume(id uint32, size uint64, readOnly bool) *master_pb.VolumeInformationMessage {
+	return &master_pb.VolumeInformationMessage{
+		Id: id, Size: size, Collection: "c", Version: 3, ReadOnly: readOnly,
+	}
+}
+
+func TestSizeTrackingSkipsReadOnlyVolumes(t *testing.T) {
+	topo, dn, vl := sizeTrackingLayout(t)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		sizeTrackingVolume(1, 1000, false), sizeTrackingVolume(2, 1000, true),
+	}, dn)
+
+	vl.accessLock.RLock()
+	_, writableTracked := vl.sizeTracking[needle.VolumeId(1)]
+	_, readOnlyTracked := vl.sizeTracking[needle.VolumeId(2)]
+	vl.accessLock.RUnlock()
+
+	if !writableTracked {
+		t.Error("a writable volume lost its size tracking, so assignments stop being accounted for")
+	}
+	if readOnlyTracked {
+		t.Error("a read-only volume is tracked for writes it can never take")
+	}
+}
+
+// A volume that goes read-only after the master already tracked it has to give
+// the entry back, or a cluster that tiers its volumes keeps paying for them.
+func TestSizeTrackingReleasedWhenAVolumeGoesReadOnly(t *testing.T) {
+	topo, dn, vl := sizeTrackingLayout(t)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{sizeTrackingVolume(1, 1000, false)}, dn)
+
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{sizeTrackingVolume(1, 1000, true)}, dn)
+
+	vl.accessLock.RLock()
+	_, tracked := vl.sizeTracking[needle.VolumeId(1)]
+	vl.accessLock.RUnlock()
+	if tracked {
+		t.Error("a volume that became read-only kept its size tracking")
+	}
+}
+
+func TestSizeTrackingReturnsWhenAVolumeBecomesWritable(t *testing.T) {
+	topo, dn, vl := sizeTrackingLayout(t)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{sizeTrackingVolume(1, 1000, true)}, dn)
+
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{sizeTrackingVolume(1, 2000, false)}, dn)
+
+	vl.accessLock.RLock()
+	st, tracked := vl.sizeTracking[needle.VolumeId(1)]
+	vl.accessLock.RUnlock()
+	if !tracked {
+		t.Fatal("a volume that became writable was not tracked again")
+	}
+	if st.reportedSize != 2000 {
+		t.Errorf("tracking resumed at size %d, want the reported 2000", st.reportedSize)
+	}
+}
+
+// A volume held out of the writable list for capacity is not read-only, and its
+// entry is what enforces the recovery delay.
+func TestSizeTrackingSurvivesAFullVolume(t *testing.T) {
+	topo, dn, vl := sizeTrackingLayout(t)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{sizeTrackingVolume(1, 1000, false)}, dn)
+
+	vl.RecordAssign(needle.VolumeId(1), int64(32*1024))
+
+	vl.accessLock.RLock()
+	st, tracked := vl.sizeTracking[needle.VolumeId(1)]
+	vl.accessLock.RUnlock()
+	if !tracked {
+		t.Fatal("a full volume lost its tracking, so nothing holds it out of the writable list")
+	}
+	if st.fullSince.IsZero() {
+		t.Error("a full volume did not record when it filled, so the recovery delay cannot apply")
+	}
+}
+
+// A volume is unwritable if any replica is read-only, so the answer must not
+// depend on which replica's heartbeat arrives first.
+func TestSizeTrackingIgnoresReplicaReportOrder(t *testing.T) {
+	for _, readOnlyFirst := range []bool{true, false} {
+		name := "WritableFirst"
+		if readOnlyFirst {
+			name = "ReadOnlyFirst"
+		}
+		t.Run(name, func(t *testing.T) {
+			topo := NewTopology("st", nil, 32*1024, 5, false)
+			rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
+			writableNode := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "a", map[string]uint32{"": 100})
+			readOnlyNode := rack.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "", "b", map[string]uint32{"": 100})
+
+			report := func(dn *DataNode, readOnly bool) {
+				topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+					{Id: 1, Size: 1000, Collection: "c", Version: 3, ReplicaPlacement: 1, ReadOnly: readOnly},
+				}, dn)
+			}
+			if readOnlyFirst {
+				report(readOnlyNode, true)
+				report(writableNode, false)
+			} else {
+				report(writableNode, false)
+				report(readOnlyNode, true)
+			}
+
+			rp, _ := super_block.NewReplicaPlacementFromString("001")
+			vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType)
+			vl.accessLock.RLock()
+			_, tracked := vl.sizeTracking[needle.VolumeId(1)]
+			_, crowded := vl.crowded[needle.VolumeId(1)]
+			vl.accessLock.RUnlock()
+
+			if tracked {
+				t.Error("a volume with a read-only replica is tracked for writes it cannot take")
+			}
+			if crowded {
+				t.Error("a volume with a read-only replica was left in the crowded set")
+			}
+		})
+	}
+}
+
+// The crowded entry has to go with the tracking, or the memory this releases
+// is only moved.
+func TestCrowdedEntryReleasedWithSizeTracking(t *testing.T) {
+	topo, dn, vl := sizeTrackingLayout(t)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{sizeTrackingVolume(1, 31000, false)}, dn)
+
+	vl.accessLock.RLock()
+	_, crowdedWhileWritable := vl.crowded[needle.VolumeId(1)]
+	vl.accessLock.RUnlock()
+	if !crowdedWhileWritable {
+		t.Fatal("expected a nearly full writable volume to be crowded")
+	}
+
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{sizeTrackingVolume(1, 31000, true)}, dn)
+
+	vl.accessLock.RLock()
+	_, stillCrowded := vl.crowded[needle.VolumeId(1)]
+	vl.accessLock.RUnlock()
+	if stillCrowded {
+		t.Error("a volume that became read-only kept its crowded entry")
+	}
+}

weed/topology/volume_location_flags_test.go

@@ -0,0 +1,120 @@
+package topology
+
+import (
+	"testing"
+)
+
+func flagNode(ip string) *DataNode {
+	dn := NewDataNode(ip)
+	dn.Ip, dn.Port = ip, 8080
+	return dn
+}
+
+func TestLocationFlagsFollowTheirNode(t *testing.T) {
+	a, b, c := flagNode("10.0.0.1"), flagNode("10.0.0.2"), flagNode("10.0.0.3")
+	list := NewVolumeLocationList()
+	for _, dn := range []*DataNode{a, b, c} {
+		list.Set(dn)
+	}
+
+	list.SetReadOnly(b, true)
+	list.SetOversized(c, true)
+	if !list.AnyReadOnly() || !list.AnyOversized() {
+		t.Fatal("flags did not register")
+	}
+
+	// Removing an earlier node shifts the rest down; the flags have to move with
+	// them or they end up describing the wrong server.
+	list.Remove(a)
+	list.SetReadOnly(b, false)
+	if list.AnyReadOnly() {
+		t.Error("clearing the flag on its node left it set, so it had shifted onto another")
+	}
+	if !list.AnyOversized() {
+		t.Error("removing an unrelated node dropped another node's flag")
+	}
+	list.SetOversized(c, false)
+	if list.AnyOversized() {
+		t.Error("clearing the flag on its node left it set")
+	}
+}
+
+func TestLocationFlagsClearedWithTheirNode(t *testing.T) {
+	a, b := flagNode("10.0.0.1"), flagNode("10.0.0.2")
+	list := NewVolumeLocationList()
+	list.Set(a)
+	list.Set(b)
+	list.SetReadOnly(a, true)
+
+	list.Remove(a)
+	if list.AnyReadOnly() {
+		t.Error("a departed node left its read-only marking behind")
+	}
+}
+
+// A node that shares an address replaces the entry, so it inherits the slot.
+func TestLocationFlagsSurviveAReplacedNode(t *testing.T) {
+	a, replacement := flagNode("10.0.0.1"), flagNode("10.0.0.1")
+	list := NewVolumeLocationList()
+	list.Set(a)
+	list.SetReadOnly(a, true)
+
+	list.Set(replacement)
+	if !list.AnyReadOnly() {
+		t.Error("replacing the node at an address dropped what was known about the volume there")
+	}
+	list.SetReadOnly(replacement, false)
+	if list.AnyReadOnly() {
+		t.Error("the replacement could not clear the flag it inherited")
+	}
+}
+
+func TestLocationFlagsIgnoreUntrackableReplicaCounts(t *testing.T) {
+	list := NewVolumeLocationList()
+	nodes := make([]*DataNode, 0, maxTrackedLocations+2)
+	for i := 0; i < maxTrackedLocations+2; i++ {
+		dn := NewDataNode("n")
+		dn.Ip, dn.Port = "10.0.0.1", 9000+i
+		nodes = append(nodes, dn)
+		list.Set(dn)
+	}
+
+	// Past the tracked width the flag is not recorded rather than landing on
+	// some other node's slot.
+	list.SetReadOnly(nodes[maxTrackedLocations+1], true)
+	if list.AnyReadOnly() {
+		t.Error("a flag beyond the tracked width was recorded against another node")
+	}
+	list.SetReadOnly(nodes[0], true)
+	if !list.AnyReadOnly() {
+		t.Error("a flag within the tracked width was lost")
+	}
+}
+
+// Refresh drops stale locations, so it has to rebuild the flags with them.
+func TestLocationFlagsRebuiltByRefresh(t *testing.T) {
+	stale, fresh, alsoFresh := flagNode("10.0.0.1"), flagNode("10.0.0.2"), flagNode("10.0.0.3")
+	stale.LastSeen, fresh.LastSeen, alsoFresh.LastSeen = 100, 500, 500
+
+	list := NewVolumeLocationList()
+	for _, dn := range []*DataNode{stale, fresh, alsoFresh} {
+		list.Set(dn)
+	}
+	list.SetReadOnly(alsoFresh, true)
+
+	list.Refresh(400)
+
+	if list.Length() != 2 {
+		t.Fatalf("expected the stale location to be dropped, got %d", list.Length())
+	}
+	// Both halves are needed: that the flag survived at all, and that it
+	// survived against the right location. Rebuilding the mask as empty would
+	// pass the second on its own.
+	if !list.AnyReadOnly() {
+		t.Fatal("the flag was dropped rather than carried across the rebuild")
+	}
+	list.SetReadOnly(alsoFresh, false)
+	if list.AnyReadOnly() {
+		t.Error("the flag did not move with its location, so it now describes another server")
+	}
+}

weed/topology/volume_location_list.go

@@ -0,0 +1,194 @@
+package topology
+
+import (
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// maxTrackedLocations is how many replicas of one volume can carry per-node
+// state. Beyond that the flags are simply not recorded, which loses a read-only
+// or oversized marking rather than misreporting one.
+const maxTrackedLocations = 32
+
+type VolumeLocationList struct {
+	list []*DataNode
+	// readOnly and oversized mirror list by index, so the per-node state that
+	// used to need an index of its own rides along with the location it
+	// describes. Both are only ever asked whether any node reports them.
+	readOnly  uint32
+	oversized uint32
+}
+
+func NewVolumeLocationList() *VolumeLocationList {
+	return &VolumeLocationList{}
+}
+
+func (dnll *VolumeLocationList) String() string {
+	return fmt.Sprintf("%v", dnll.list)
+}
+
+func (dnll *VolumeLocationList) Copy() *VolumeLocationList {
+	list := make([]*DataNode, len(dnll.list))
+	copy(list, dnll.list)
+	return &VolumeLocationList{
+		list:      list,
+		readOnly:  dnll.readOnly,
+		oversized: dnll.oversized,
+	}
+}
+
+func (dnll *VolumeLocationList) indexOf(loc *DataNode) int {
+	for i, dnl := range dnll.list {
+		if loc.Ip == dnl.Ip && loc.Port == dnl.Port {
+			return i
+		}
+	}
+	return -1
+}
+
+func setFlag(flags *uint32, index int, on bool) {
+	if index < 0 || index >= maxTrackedLocations {
+		return
+	}
+	if on {
+		*flags |= 1 << uint(index)
+	} else {
+		*flags &^= 1 << uint(index)
+	}
+}
+
+func hasFlag(flags uint32, index int) bool {
+	return index >= 0 && index < maxTrackedLocations && flags&(1<<uint(index)) != 0
+}
+
+// removeFlag drops the bit at index and closes the gap, keeping the flags
+// aligned with the locations after one is removed.
+func removeFlag(flags *uint32, index int) {
+	if index < 0 || index >= maxTrackedLocations {
+		return
+	}
+	low := *flags & (1<<uint(index) - 1)
+	high := *flags >> uint(index+1) << uint(index)
+	*flags = low | high
+}
+
+// SetReadOnly records whether loc reports the volume read-only.
+func (dnll *VolumeLocationList) SetReadOnly(loc *DataNode, readOnly bool) {
+	setFlag(&dnll.readOnly, dnll.indexOf(loc), readOnly)
+}
+
+// AnyReadOnly reports whether any location has the volume read-only.
+func (dnll *VolumeLocationList) AnyReadOnly() bool {
+	return dnll != nil && dnll.readOnly != 0
+}
+
+// SetOversized records whether loc reports the volume past the size limit.
+func (dnll *VolumeLocationList) SetOversized(loc *DataNode, oversized bool) {
+	setFlag(&dnll.oversized, dnll.indexOf(loc), oversized)
+}
+
+// AnyOversized reports whether any location has the volume past the size limit.
+func (dnll *VolumeLocationList) AnyOversized() bool {
+	return dnll != nil && dnll.oversized != 0
+}
+
+func (dnll *VolumeLocationList) Head() *DataNode {
+	//mark first node as master volume
+	if dnll.Length() == 0 {
+		return nil
+	}
+	return dnll.list[0]
+}
+
+func (dnll *VolumeLocationList) Rest() []*DataNode {
+	//mark first node as master volume
+	return dnll.list[1:]
+}
+
+func (dnll *VolumeLocationList) Length() int {
+	if dnll == nil {
+		return 0
+	}
+	return len(dnll.list)
+}
+
+// Set adds loc, or replaces the entry at the same address, returning the node it
+// displaced. Two volume servers can share an address -- GetOrCreateDataNode keys
+// on the reported id and refuses to merge a new id onto an address an older node
+// still claims -- so the displaced node is not necessarily loc, and callers
+// tracking which node a volume is reachable through must move their bookkeeping
+// off it rather than assume loc already owned the entry.
+func (dnll *VolumeLocationList) Set(loc *DataNode) (displaced *DataNode) {
+	for i := 0; i < len(dnll.list); i++ {
+		if loc.Ip == dnll.list[i].Ip && loc.Port == dnll.list[i].Port {
+			displaced = dnll.list[i]
+			dnll.list[i] = loc
+			return displaced
+		}
+	}
+	dnll.list = append(dnll.list, loc)
+	return nil
+}
+
+// Remove drops the entry at loc's address and returns the node removed, or nil
+// if the volume was not reachable there. As with Set, the removed node is
+// matched by address and need not be loc.
+func (dnll *VolumeLocationList) Remove(loc *DataNode) (removed *DataNode) {
+	for i, dnl := range dnll.list {
+		if loc.Ip == dnl.Ip && loc.Port == dnl.Port {
+			dnll.list = append(dnll.list[:i], dnll.list[i+1:]...)
+			removeFlag(&dnll.readOnly, i)
+			removeFlag(&dnll.oversized, i)
+			return dnl
+		}
+	}
+	return nil
+}
+
+func (dnll *VolumeLocationList) Refresh(freshThreshHold int64) {
+	var changed bool
+	for _, dnl := range dnll.list {
+		if dnl.LastSeen < freshThreshHold {
+			changed = true
+			break
+		}
+	}
+	if !changed {
+		return
+	}
+	// The flags index the locations, so dropping some means rebuilding both
+	// rather than leaving bits describing whoever moved into their place.
+	var l []*DataNode
+	var readOnly, oversized uint32
+	for i, dnl := range dnll.list {
+		if dnl.LastSeen < freshThreshHold {
+			continue
+		}
+		setFlag(&readOnly, len(l), hasFlag(dnll.readOnly, i))
+		setFlag(&oversized, len(l), hasFlag(dnll.oversized, i))
+		l = append(l, dnl)
+	}
+	dnll.list, dnll.readOnly, dnll.oversized = l, readOnly, oversized
+}
+
+// Stats returns logic size and count. Both subtractions are clamped: the
+// deleted counters are maintained apart from the totals they come off and can
+// transiently exceed them.
+func (dnll *VolumeLocationList) Stats(vid needle.VolumeId, freshThreshHold int64) (size uint64, fileCount int) {
+	for _, dnl := range dnll.list {
+		if dnl.LastSeen < freshThreshHold {
+			vinfo, err := dnl.GetVolumesById(vid)
+			if err == nil {
+				if vinfo.Size > vinfo.DeletedByteCount {
+					size = vinfo.Size - vinfo.DeletedByteCount
+				}
+				if vinfo.FileCount > vinfo.DeleteCount {
+					fileCount = int(vinfo.FileCount - vinfo.DeleteCount)
+				}
+				return size, fileCount
+			}
+		}
+	}
+	return 0, 0
+}

weed/topology/volume_stream.go

@@ -0,0 +1,137 @@
+package topology
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// StreamVolumes hands every volume the filter selects to send, in batches, so
+// that neither end holds the whole cluster to move it. Batches are built under
+// their disk's lock and handed over outside it, so a slow reader stalls the
+// stream rather than the topology.
+//
+// Only the disks `listed` names are streamed. That listing went out first, and
+// the topology is walked again here, so without it a disk registering in
+// between would have its volumes sent to a client with nowhere to put them.
+// Bounding the walk by what was already announced makes the two agree by
+// construction: a disk arriving mid-listing is in neither, and is reported by
+// the next one.
+//
+// The batches still do not share one instant. Neither did a single listing,
+// which takes each disk's lock in turn, so a volume that moves while either is
+// running can be seen twice or not at all.
+func (t *Topology) StreamVolumes(listed *master_pb.TopologyInfo, filter VolumeFilter, batchSize int, send func(*master_pb.VolumeListStreamResponse) error) error {
+	if batchSize <= 0 {
+		batchSize = defaultVolumeStreamBatch
+	}
+	announced := announcedDisks(listed)
+	for _, dcNode := range t.Children() {
+		dc := dcNode.(*DataCenter)
+		for _, rackNode := range dc.Children() {
+			rack := rackNode.(*Rack)
+			for _, dnNode := range rack.Children() {
+				dn := dnNode.(*DataNode)
+				for _, diskNode := range dn.Children() {
+					disk := diskNode.(*Disk)
+					if !announced[[4]string{string(dc.Id()), string(rack.Id()), string(dn.Id()), string(disk.Id())}] {
+						continue
+					}
+					batch := func() *master_pb.VolumeListStreamResponse {
+						return &master_pb.VolumeListStreamResponse{
+							DataCenter: string(dc.Id()),
+							Rack:       string(rack.Id()),
+							DataNode:   string(dn.Id()),
+							DiskType:   string(disk.Id()),
+						}
+					}
+					if err := disk.streamVolumes(filter, batchSize, batch, send); err != nil {
+						return err
+					}
+					if err := disk.streamEcShards(filter, batchSize, batch, send); err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+const defaultVolumeStreamBatch = 10000
+
+// announcedDisks names the disks a listing carried.
+func announcedDisks(listed *master_pb.TopologyInfo) map[[4]string]bool {
+	announced := make(map[[4]string]bool)
+	if listed == nil {
+		return announced
+	}
+	for _, dc := range listed.DataCenterInfos {
+		for _, rack := range dc.RackInfos {
+			for _, node := range rack.DataNodeInfos {
+				for diskType := range node.DiskInfos {
+					announced[[4]string{dc.Id, rack.Id, node.Id, diskType}] = true
+				}
+			}
+		}
+	}
+	return announced
+}
+
+// streamVolumes sends this disk's volumes a batch at a time. The ids are taken
+// in one pass and the messages built in later ones, so the lock is held for a
+// batch rather than for the disk, and only 4 bytes per volume are carried
+// between passes. A volume that leaves in between is simply not sent.
+func (d *Disk) streamVolumes(filter VolumeFilter, batchSize int, newBatch func() *master_pb.VolumeListStreamResponse, send func(*master_pb.VolumeListStreamResponse) error) error {
+	d.RLock()
+	ids := make([]needle.VolumeId, 0, len(d.volumes))
+	for id := range d.volumes {
+		ids = append(ids, id)
+	}
+	d.RUnlock()
+
+	for start := 0; start < len(ids); start += batchSize {
+		end := min(start+batchSize, len(ids))
+		batch := newBatch()
+
+		d.RLock()
+		for _, id := range ids[start:end] {
+			v, found := d.volumes[id]
+			if !found || !filter.matches(v) {
+				continue
+			}
+			batch.VolumeInfos = append(batch.VolumeInfos, v.ToVolumeInformationMessage())
+		}
+		d.RUnlock()
+
+		if len(batch.VolumeInfos) == 0 {
+			continue
+		}
+		if err := send(batch); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (d *Disk) streamEcShards(filter VolumeFilter, batchSize int, newBatch func() *master_pb.VolumeListStreamResponse, send func(*master_pb.VolumeListStreamResponse) error) error {
+	// GetEcShards already copies under the lock, and a cluster holds far fewer
+	// ec shards than volumes, so these only need cutting into batches.
+	shards := d.GetEcShards()
+	for start := 0; start < len(shards); start += batchSize {
+		end := min(start+batchSize, len(shards))
+		batch := newBatch()
+		for _, ecv := range shards[start:end] {
+			if !filter.matches(ecv) {
+				continue
+			}
+			batch.EcShardInfos = append(batch.EcShardInfos, ecv.ToVolumeEcShardInformationMessage())
+		}
+		if len(batch.EcShardInfos) == 0 {
+			continue
+		}
+		if err := send(batch); err != nil {
+			return err
+		}
+	}
+	return nil
+}

weed/topology/volume_stream_bounds_test.go

@@ -0,0 +1,45 @@
+package topology
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+// A disk that registers after the listing went out has nowhere to go in it, so
+// its volumes must not be streamed into a client that cannot place them. It is
+// reported by the next listing instead.
+func TestStreamVolumesStaysInsideTheAnnouncedTopology(t *testing.T) {
+	topo := NewTopology("bounds", nil, 32*1024*1024*1024, 5, false)
+	rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
+	known := rack.GetOrCreateDataNode("10.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100})
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		{Id: 1, Collection: "c", Size: 100, Version: 3},
+		{Id: 2, Collection: "c", Size: 200, Version: 3},
+	}, known)
+
+	listed := topo.ToTopologyInfo(NoVolumes())
+
+	// The heartbeat that lands between the listing and the walk below.
+	late := rack.GetOrCreateDataNode("10.0.0.2", 8080, 18080, "", "", map[string]uint32{"": 100})
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		{Id: 3, Collection: "c", Size: 300, Version: 3},
+	}, late)
+
+	var streamed []uint32
+	err := topo.StreamVolumes(listed, VolumeFilter{}, 10, func(b *master_pb.VolumeListStreamResponse) error {
+		if b.DataNode == string(late.Id()) {
+			t.Errorf("streamed a batch for %s, which the listing never named", b.DataNode)
+		}
+		for _, v := range b.VolumeInfos {
+			streamed = append(streamed, v.Id)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !equalIds(streamed, []uint32{1, 2}) {
+		t.Errorf("streamed %v, want the two volumes the listing named", streamed)
+	}
+}

weed/topology/volume_stream_test.go

@@ -0,0 +1,134 @@
+package topology
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func streamTestTopology(t *testing.T) *Topology {
+	t.Helper()
+	topo := NewTopology("stream", nil, 32*1024*1024*1024, 5, false)
+	for dc := 1; dc <= 2; dc++ {
+		rack := topo.GetOrCreateDataCenter(fmt.Sprintf("dc%d", dc)).GetOrCreateRack("rack1")
+		for n := 1; n <= 2; n++ {
+			dn := rack.GetOrCreateDataNode(fmt.Sprintf("10.0.%d.%d", dc, n), 8080, 18080, "", "",
+				map[string]uint32{"": 10000})
+			var volumes []*master_pb.VolumeInformationMessage
+			for i := 0; i < 250; i++ {
+				volumes = append(volumes, &master_pb.VolumeInformationMessage{
+					Id: uint32(dc*10000 + n*1000 + i), Size: uint64(i) * 100,
+					Collection: fmt.Sprintf("c%d", i%3), Version: 3, DiskId: uint32(i % 2),
+				})
+			}
+			topo.SyncDataNodeRegistration(volumes, dn)
+			topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{
+				{Id: uint32(dc*100 + n), Collection: "c1", EcIndexBits: 0x3fff, DiskId: 1},
+			}, dn)
+		}
+	}
+	return topo
+}
+
+// summarise reduces a listing to what a caller reads off it, so a streamed one
+// can be held against an unstreamed one without depending on ordering.
+func summarise(info *master_pb.TopologyInfo) []string {
+	var lines []string
+	for _, dc := range info.DataCenterInfos {
+		for _, rack := range dc.RackInfos {
+			for _, node := range rack.DataNodeInfos {
+				for diskType, disk := range node.DiskInfos {
+					where := fmt.Sprintf("%s/%s/%s/%s", dc.Id, rack.Id, node.Id, diskType)
+					lines = append(lines,
+						fmt.Sprintf("%s disk id=%d max=%d count=%d", where, disk.DiskId, disk.MaxVolumeCount, disk.VolumeCount))
+					for _, v := range disk.VolumeInfos {
+						lines = append(lines, fmt.Sprintf("%s vol %d size=%d collection=%s disk=%d",
+							where, v.Id, v.Size, v.Collection, v.DiskId))
+					}
+					for _, ec := range disk.EcShardInfos {
+						lines = append(lines, fmt.Sprintf("%s ec %d collection=%s bits=%d",
+							where, ec.Id, ec.Collection, ec.EcIndexBits))
+					}
+				}
+			}
+		}
+	}
+	sort.Strings(lines)
+	return lines
+}
+
+// Every caller that moved onto the stream rebuilds the listing from a topology
+// sent without volumes plus the batches that follow. That has to come out the
+// same as the listing they used to be sent whole.
+func TestStreamedListingRebuildsToTheSameThing(t *testing.T) {
+	topo := streamTestTopology(t)
+
+	for _, batchSize := range []int{1, 7, 250, 100000} {
+		t.Run(fmt.Sprintf("batch=%d", batchSize), func(t *testing.T) {
+			rebuilt := topo.ToTopologyInfo(NoVolumes())
+			byPlace := map[[4]string]*master_pb.DiskInfo{}
+			for _, dc := range rebuilt.DataCenterInfos {
+				for _, rack := range dc.RackInfos {
+					for _, node := range rack.DataNodeInfos {
+						for diskType, disk := range node.DiskInfos {
+							byPlace[[4]string{dc.Id, rack.Id, node.Id, diskType}] = disk
+							if len(disk.VolumeInfos) != 0 || len(disk.EcShardInfos) != 0 {
+								t.Fatalf("the header listed volumes on %s", node.Id)
+							}
+						}
+					}
+				}
+			}
+
+			err := topo.StreamVolumes(rebuilt, VolumeFilter{}, batchSize, func(b *master_pb.VolumeListStreamResponse) error {
+				if batchSize < 100000 && len(b.VolumeInfos) > batchSize {
+					t.Errorf("batch carried %d volumes, more than the %d asked for", len(b.VolumeInfos), batchSize)
+				}
+				disk := byPlace[[4]string{b.DataCenter, b.Rack, b.DataNode, b.DiskType}]
+				if disk == nil {
+					t.Fatalf("batch named a disk the header did not: %s/%s", b.DataNode, b.DiskType)
+				}
+				disk.VolumeInfos = append(disk.VolumeInfos, b.VolumeInfos...)
+				disk.EcShardInfos = append(disk.EcShardInfos, b.EcShardInfos...)
+				return nil
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			want := summarise(topo.ToTopologyInfo(VolumeFilter{}))
+			got := summarise(rebuilt)
+			if len(got) != len(want) {
+				t.Fatalf("rebuilt %d lines, want %d", len(got), len(want))
+			}
+			for i := range want {
+				if got[i] != want[i] {
+					t.Fatalf("line %d:\n got %q\nwant %q", i, got[i], want[i])
+				}
+			}
+		})
+	}
+}
+
+// A filter must select the same volumes streamed as it does whole.
+func TestStreamedListingHonoursTheFilter(t *testing.T) {
+	topo := streamTestTopology(t)
+	name := "c1"
+
+	var streamed []uint32
+	err := topo.StreamVolumes(topo.ToTopologyInfo(NoVolumes()), VolumeFilter{Collection: &name}, 16, func(b *master_pb.VolumeListStreamResponse) error {
+		for _, v := range b.VolumeInfos {
+			streamed = append(streamed, v.Id)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	whole, _ := listed(topo.ToTopologyInfo(VolumeFilter{Collection: &name}))
+	if !equalIds(streamed, whole) {
+		t.Errorf("streamed %d volumes, whole listing had %d", len(streamed), len(whole))
+	}
+}