snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/topology/balancer/density.go

@@ -0,0 +1,52 @@
+package balancer
+
+import "math"
+
+// zeroVolumeDensityWeight is the ratio given to a server holding no data, so it
+// still ranks below any loaded server (which has ratio >= 1) but is distinguished
+// from a server with unknown capacity.
+const zeroVolumeDensityWeight = 0.5
+
+// UsedVolumeEquivalents converts the bytes a server holds into whole-volume
+// equivalents (volumeBytes / volumeSizeLimit). Returns 0 when the limit is unset.
+func UsedVolumeEquivalents(volumeBytes, volumeSizeLimitBytes uint64) uint64 {
+	if volumeSizeLimitBytes == 0 {
+		return 0
+	}
+	return volumeBytes / volumeSizeLimitBytes
+}
+
+// VolumeDensity is the bytes-aware capacity view balancing ranks servers by:
+// used = volumeBytes / volumeSizeLimit, capacity = maxVolumeCount - used. Using
+// real data (not just volume count) means an over-configured maxVolumeCount can't
+// make a byte-full server look empty. capacity may be <= 0 for a server already
+// past its configured slots.
+func VolumeDensity(maxVolumeCount int64, volumeBytes, volumeSizeLimitBytes uint64) (capacity float64, usedVolumes uint64) {
+	usedVolumes = UsedVolumeEquivalents(volumeBytes, volumeSizeLimitBytes)
+	return float64(maxVolumeCount - int64(usedVolumes)), usedVolumes
+}
+
+// DensityRatio is a server's load: usedVolumes / capacity, higher = fuller. An
+// empty server gets a small positive ratio so it sorts below loaded servers.
+// A server with no free capacity (capacity <= 0: full, or over its configured
+// slots so capacity is negative) ranks as the fullest (+Inf), so ascending
+// consumers treat it as a move source and never as the emptiest target.
+func DensityRatio(capacity float64, usedVolumes uint64) float64 {
+	if capacity <= 0 {
+		return math.Inf(1)
+	}
+	if usedVolumes == 0 {
+		return zeroVolumeDensityWeight / capacity
+	}
+	return float64(usedVolumes) / capacity
+}
+
+// DensityNextRatio is a server's load after one more volume lands on it:
+// (usedVolumes+1) / capacity. Used to test whether a move would overshoot. A
+// server with no free capacity (capacity <= 0) ranks as the fullest (+Inf).
+func DensityNextRatio(capacity float64, usedVolumes uint64) float64 {
+	if capacity <= 0 {
+		return math.Inf(1)
+	}
+	return float64(usedVolumes+1) / capacity
+}

weed/topology/balancer/density_test.go

@@ -0,0 +1,81 @@
+package balancer
+
+import (
+	"math"
+	"sort"
+	"testing"
+)
+
+func TestVolumeDensity(t *testing.T) {
+	const gb = uint64(1) << 30
+	// 20 full-ish volumes (~1900 GB) on a disk with a 30 GB limit and Max 1000.
+	cap, used := VolumeDensity(1000, 1900*gb, 30*gb)
+	if used != 63 { // 1900/30 = 63.33 -> 63
+		t.Errorf("usedVolumes = %d, want 63", used)
+	}
+	if cap != float64(1000-63) {
+		t.Errorf("capacity = %v, want %v", cap, float64(1000-63))
+	}
+	// Unset limit -> 0 used, capacity = maxVolumeCount.
+	if c, u := VolumeDensity(10, 1000*gb, 0); u != 0 || c != 10 {
+		t.Errorf("unset limit: got cap=%v used=%d, want 10/0", c, u)
+	}
+}
+
+func TestDensityRatio(t *testing.T) {
+	// Empty server: small positive ratio (below any loaded server).
+	empty := DensityRatio(1000, 0)
+	loaded := DensityRatio(1000, 1)
+	if !(empty > 0 && empty < loaded) {
+		t.Errorf("empty ratio %v should be >0 and < loaded %v", empty, loaded)
+	}
+	// Loaded: used/capacity.
+	if got := DensityRatio(3, 7); got != 7.0/3.0 {
+		t.Errorf("DensityRatio(3,7) = %v, want %v", got, 7.0/3.0)
+	}
+	// No free capacity (full) or over the slot limit (negative capacity) -> fullest.
+	if got := DensityRatio(0, 5); !math.IsInf(got, 1) {
+		t.Errorf("DensityRatio(0,5) = %v, want +Inf", got)
+	}
+	if got := DensityRatio(-4, 20); !math.IsInf(got, 1) {
+		t.Errorf("DensityRatio(-4,20) = %v, want +Inf (overfull ranks fullest, not negative)", got)
+	}
+}
+
+func TestDensityNextRatio(t *testing.T) {
+	if got := DensityNextRatio(3, 7); got != 8.0/3.0 {
+		t.Errorf("DensityNextRatio(3,7) = %v, want %v", got, 8.0/3.0)
+	}
+	if got := DensityNextRatio(0, 7); !math.IsInf(got, 1) {
+		t.Errorf("DensityNextRatio(0,7) = %v, want +Inf", got)
+	}
+	if got := DensityNextRatio(-1, 7); !math.IsInf(got, 1) {
+		t.Errorf("DensityNextRatio(-1,7) = %v, want +Inf", got)
+	}
+}
+
+// An overfull server (negative capacity) must sort as the fullest, not the
+// emptiest, when consumers rank ascending by DensityRatio.
+func TestDensityRatio_OverfullSortsFullest(t *testing.T) {
+	// empty (cap 100, used 0), half (cap 5, used 5), overfull (cap -3, used 13).
+	type server struct {
+		name        string
+		capacity    float64
+		usedVolumes uint64
+	}
+	servers := []server{
+		{"overfull", -3, 13},
+		{"empty", 100, 0},
+		{"half", 5, 5},
+	}
+	sort.Slice(servers, func(i, j int) bool {
+		return DensityRatio(servers[i].capacity, servers[i].usedVolumes) <
+			DensityRatio(servers[j].capacity, servers[j].usedVolumes)
+	})
+	if servers[0].name != "empty" {
+		t.Errorf("emptiest (ascending first) = %q, want empty", servers[0].name)
+	}
+	if servers[len(servers)-1].name != "overfull" {
+		t.Errorf("fullest (ascending last) = %q, want overfull", servers[len(servers)-1].name)
+	}
+}

weed/topology/balancer/disk_fullness.go

@@ -0,0 +1,33 @@
+// Package balancer holds balancing policy shared by the shell volume.balance
+// command and the maintenance balance worker so the two implementations do not
+// drift. It is dependency-light and takes plain values, so both the raw
+// master_pb world (shell) and the ActiveTopology world (worker) can call it.
+package balancer
+
+// DefaultMaxDiskUsagePercent is the default physical-disk high-water mark for
+// balancing: a move target whose disk is at or above this used percentage is
+// skipped. It guards against an over-configured maxVolumeCount making a
+// physically full server look under-utilized by slot count.
+const DefaultMaxDiskUsagePercent = 90
+
+// DiskTooFullAfter reports whether landing incomingBytes on a disk would push its
+// physical used space to or above maxUsagePercent of totalBytes. It judges the
+// disk against its own total, so heterogeneous disk sizes are handled correctly.
+//
+// It returns false (no opinion) when the gate is disabled (maxUsagePercent <= 0
+// or >= 100) or the disk reports no capacity (totalBytes == 0), so callers fall
+// back to slot-only behavior for servers that do not report disk bytes.
+//
+// Pass incomingBytes = 0 to test current fullness, or a volume's worth of bytes
+// to test fullness after the move.
+func DiskTooFullAfter(totalBytes, freeBytes, incomingBytes uint64, maxUsagePercent int) bool {
+	if maxUsagePercent <= 0 || maxUsagePercent >= 100 || totalBytes == 0 {
+		return false
+	}
+	var used uint64
+	if totalBytes > freeBytes {
+		used = totalBytes - freeBytes
+	}
+	usedAfter := float64(used) + float64(incomingBytes)
+	return usedAfter*100 >= float64(totalBytes)*float64(maxUsagePercent)
+}

weed/topology/balancer/disk_fullness_test.go

@@ -0,0 +1,31 @@
+package balancer
+
+import "testing"
+
+func TestDiskTooFullAfter(t *testing.T) {
+	const gb = uint64(1) << 30
+	tests := []struct {
+		name                  string
+		total, free, incoming uint64
+		pct                   int
+		want                  bool
+	}{
+		{"disabled at 0", 1000 * gb, 40 * gb, 0, 0, false},
+		{"disabled at 100", 1000 * gb, 40 * gb, 0, 100, false},
+		{"not reported (total 0)", 0, 0, 0, 90, false},
+		{"below mark", 1000 * gb, 300 * gb, 0, 90, false},                           // 70% used
+		{"at mark", 1000 * gb, 100 * gb, 0, 90, true},                               // exactly 90% used
+		{"above mark", 1000 * gb, 40 * gb, 0, 90, true},                             // 96% used
+		{"below but crosses with incoming", 1000 * gb, 120 * gb, 30 * gb, 90, true}, // 88% -> 91%
+		{"below and stays below with incoming", 1000 * gb, 300 * gb, 30 * gb, 90, false},
+		{"free exceeds total (defensive)", 1000 * gb, 2000 * gb, 0, 90, false},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := DiskTooFullAfter(tc.total, tc.free, tc.incoming, tc.pct); got != tc.want {
+				t.Errorf("DiskTooFullAfter(%d,%d,%d,%d) = %v, want %v",
+					tc.total, tc.free, tc.incoming, tc.pct, got, tc.want)
+			}
+		})
+	}
+}

weed/topology/balancer/replica_placement.go

@@ -0,0 +1,176 @@
+package balancer
+
+import (
+	"slices"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+)
+
+// Location identifies where a volume replica lives, at the granularity replica
+// placement reasons about: data center, rack, data node, and physical host
+// (machine). Servers sharing a host are one fault domain. It is the shared shape
+// the shell (from master_pb) and the worker (from ActiveTopology) both adapt to.
+type Location struct {
+	DataCenter string
+	Rack       string
+	NodeID     string
+	Host       string
+}
+
+// HostFromAddress returns the physical machine (host/IP) of a server address,
+// falling back to the node id (== ip:port in the common case) when no address is
+// known.
+func HostFromAddress(address, nodeID string) string {
+	if address == "" {
+		address = nodeID
+	}
+	return pb.ServerAddress(address).ToHost()
+}
+
+type rackKey struct {
+	DataCenter string
+	Rack       string
+}
+
+type nodeKey struct {
+	DataCenter string
+	Rack       string
+	NodeID     string
+}
+
+// IsGoodMove reports whether moving a volume from sourceNodeID onto target keeps
+// the volume's replica placement policy satisfied, given the current replica
+// locations. It refuses moving onto the source itself, onto a node that already
+// holds a replica, and onto a host that already holds another replica (best-effort
+// machine anti-affinity), then defers to SatisfyReplicaPlacement.
+func IsGoodMove(rp *super_block.ReplicaPlacement, existingReplicas []Location, sourceNodeID string, target Location) bool {
+	if rp == nil || !rp.HasReplication() {
+		return true // no replication constraint
+	}
+
+	// Never move onto the source node itself.
+	if target.NodeID == sourceNodeID {
+		return false
+	}
+
+	// Build the replica set after the move: remove the one replica being moved off
+	// the source, keeping any others (an over-replicated node may hold more than
+	// one replica of this volume, and only one is moving).
+	afterMove := make([]Location, 0, len(existingReplicas))
+	sourceFound := false
+	for _, r := range existingReplicas {
+		if r.NodeID == sourceNodeID && !sourceFound {
+			sourceFound = true
+		} else {
+			afterMove = append(afterMove, r)
+		}
+	}
+	if !sourceFound {
+		// Source not in the replica list — cluster state may be inconsistent.
+		// Treat as unsafe to avoid an incorrect placement decision.
+		return false
+	}
+
+	// Best-effort machine anti-affinity: don't move a replica onto a host that
+	// already holds another replica of this volume, so a single machine failure
+	// can't take out two replicas.
+	if target.Host != "" {
+		for _, r := range afterMove {
+			if r.Host == target.Host {
+				return false
+			}
+		}
+	}
+
+	return SatisfyReplicaPlacement(rp, afterMove, target)
+}
+
+// SatisfyReplicaPlacement reports whether placing a replica at target is
+// consistent with the replication policy given the existing replicas. It is the
+// shared implementation for the shell (volume.balance / fix.replication /
+// tier.move) and the maintenance balance worker.
+func SatisfyReplicaPlacement(rp *super_block.ReplicaPlacement, replicas []Location, target Location) bool {
+	existingDCs, _, existingNodes := countReplicas(replicas)
+
+	targetNK := nodeKey{DataCenter: target.DataCenter, Rack: target.Rack, NodeID: target.NodeID}
+	if _, found := existingNodes[targetNK]; found {
+		// avoid a duplicated volume on the same data node
+		return false
+	}
+
+	primaryDCs, _ := findTopDCKeys(existingDCs)
+
+	// ensure the data center count is within limit
+	if _, found := existingDCs[target.DataCenter]; !found {
+		// different from existing dcs
+		return len(existingDCs) < rp.DiffDataCenterCount+1
+	}
+	// same as one of the existing data centers
+	if !slices.Contains(primaryDCs, target.DataCenter) {
+		return false
+	}
+
+	// on a primary dc — check racks within this DC
+	primaryDcRacks := make(map[rackKey]int)
+	for _, r := range replicas {
+		if r.DataCenter != target.DataCenter {
+			continue
+		}
+		primaryDcRacks[rackKey{DataCenter: r.DataCenter, Rack: r.Rack}]++
+	}
+
+	targetRK := rackKey{DataCenter: target.DataCenter, Rack: target.Rack}
+	primaryRacks, _ := findTopRackKeys(primaryDcRacks)
+	sameRackCount := primaryDcRacks[targetRK]
+
+	if _, found := primaryDcRacks[targetRK]; !found {
+		// different from existing racks
+		return len(primaryDcRacks) < rp.DiffRackCount+1
+	}
+	// same as one of the existing racks
+	if !slices.Contains(primaryRacks, targetRK) {
+		return false
+	}
+
+	// on the primary rack — check the same-rack count
+	return sameRackCount < rp.SameRackCount+1
+}
+
+func countReplicas(replicas []Location) (dcCounts map[string]int, rackCounts map[rackKey]int, nodeCounts map[nodeKey]int) {
+	dcCounts = make(map[string]int)
+	rackCounts = make(map[rackKey]int)
+	nodeCounts = make(map[nodeKey]int)
+	for _, r := range replicas {
+		dcCounts[r.DataCenter]++
+		rackCounts[rackKey{DataCenter: r.DataCenter, Rack: r.Rack}]++
+		nodeCounts[nodeKey{DataCenter: r.DataCenter, Rack: r.Rack, NodeID: r.NodeID}]++
+	}
+	return
+}
+
+func findTopDCKeys(m map[string]int) (topKeys []string, max int) {
+	for k, c := range m {
+		if max < c {
+			topKeys = topKeys[:0]
+			topKeys = append(topKeys, k)
+			max = c
+		} else if max == c {
+			topKeys = append(topKeys, k)
+		}
+	}
+	return
+}
+
+func findTopRackKeys(m map[rackKey]int) (topKeys []rackKey, max int) {
+	for k, c := range m {
+		if max < c {
+			topKeys = topKeys[:0]
+			topKeys = append(topKeys, k)
+			max = c
+		} else if max == c {
+			topKeys = append(topKeys, k)
+		}
+	}
+	return
+}

weed/topology/balancer/replica_placement_test.go

@@ -0,0 +1,36 @@
+package balancer
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+)
+
+func loc(dc, rack, node string) Location {
+	return Location{DataCenter: dc, Rack: rack, NodeID: node}
+}
+
+// When a node abnormally holds more than one replica of a volume, moving one of
+// them off must keep the other in the after-move set, so the placement check
+// still sees the rack's true occupancy.
+func TestIsGoodMove_OverReplicatedSourceKeepsOtherReplica(t *testing.T) {
+	rp, err := super_block.NewReplicaPlacementFromString("001") // up to 2 copies in a rack
+	if err != nil {
+		t.Fatal(err)
+	}
+	// n1 abnormally holds two replicas; n2 holds a third — all in rack r1.
+	existing := []Location{loc("dc1", "r1", "n1"), loc("dc1", "r1", "n1"), loc("dc1", "r1", "n2")}
+
+	// Moving ONE replica off n1 onto a new node n3 in r1 must be rejected: r1 would
+	// still hold two replicas (the retained n1 and n2), at the SameRackCount limit.
+	if IsGoodMove(rp, existing, "n1", loc("dc1", "r1", "n3")) {
+		t.Error("expected reject: moving one replica off an over-replicated node must keep its other replica in the rack count")
+	}
+
+	// The normal case is unaffected: two replicas correctly in r1, move one to a
+	// third node in r1 — still within the SameRackCount=1 (two-per-rack) limit.
+	normal := []Location{loc("dc1", "r1", "n1"), loc("dc1", "r1", "n2")}
+	if !IsGoodMove(rp, normal, "n1", loc("dc1", "r1", "n3")) {
+		t.Error("expected allow: a normal move within the rack limit")
+	}
+}

weed/topology/balancer/replica_selection.go

@@ -0,0 +1,161 @@
+package balancer
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+)
+
+// Replica pairs a replica's placement location with the volume attributes
+// replica-selection decisions need. Like Location for placement checks, it is
+// the shared shape the shell (from master_pb) and workers adapt to. Selection
+// functions return an index into the given slice (-1 when nothing qualifies),
+// so callers can map the choice back to their own richer replica type.
+type Replica struct {
+	Location         Location
+	Size             uint64
+	ModifiedAtSecond int64
+	CompactRevision  uint32
+	ReadOnly         bool
+}
+
+// SatisfyReplicaCurrentLocation reports whether the existing replicas already
+// span the failure domains the policy requires (enough data centers, enough
+// racks, and the same-rack count somewhere). A volume whose replica count is
+// right but whose spread falls short still needs another replica.
+func SatisfyReplicaCurrentLocation(rp *super_block.ReplicaPlacement, replicas []Location) bool {
+	dcCounts, rackCounts, _ := countReplicas(replicas)
+
+	if rp.DiffDataCenterCount+1 > len(dcCounts) {
+		return false
+	}
+	if rp.DiffRackCount+1 > len(rackCounts) {
+		return false
+	}
+	if rp.SameRackCount > 0 {
+		for _, rackCount := range rackCounts {
+			if rackCount >= rp.SameRackCount+1 {
+				return true
+			}
+		}
+		return false
+	}
+	return true
+}
+
+// PickOneReplicaToCopyFrom returns the most recently modified replica, the
+// best source for adding a new replica.
+func PickOneReplicaToCopyFrom(replicas []Replica) int {
+	mostRecent := -1
+	for i, r := range replicas {
+		if mostRecent < 0 || r.ModifiedAtSecond > replicas[mostRecent].ModifiedAtSecond {
+			mostRecent = i
+		}
+	}
+	return mostRecent
+}
+
+// IsMisplaced reports whether any replica sits at a location inconsistent
+// with the replication policy given the other replicas' locations.
+func IsMisplaced(replicas []Replica, rp *super_block.ReplicaPlacement) bool {
+	for i := range replicas {
+		others := otherThan(replicas, i)
+		locs := make([]Location, len(others))
+		for j, r := range others {
+			locs[j] = r.Location
+		}
+		if !SatisfyReplicaPlacement(rp, locs, replicas[i].Location) {
+			return true
+		}
+	}
+	return false
+}
+
+// PickSmallestReplica returns the smallest replica (ties broken by oldest
+// then lowest compact revision).
+func PickSmallestReplica(replicas []Replica) int {
+	smallest := -1
+	for i := range replicas {
+		if smallest < 0 || lessReplica(replicas[i], replicas[smallest]) {
+			smallest = i
+		}
+	}
+	return smallest
+}
+
+// PickOneReplicaToDelete selects the replica to trim when over-replicated.
+// It only ever removes the smallest of multiple healthy writable replicas: a
+// ReadOnly/integrity-flagged replica is never chosen for deletion, and the
+// trim is refused (returns -1) when removing a writable replica would leave
+// only ReadOnly survivors. Among the writable replicas it prefers one whose
+// removal still satisfies placement, so the trim does not strip the only
+// replica in a required failure domain; it falls back to the smallest
+// writable if none keeps placement (a later misplaced cycle then
+// re-balances).
+func PickOneReplicaToDelete(replicas []Replica, rp *super_block.ReplicaPlacement) int {
+	var writable []int
+	for i, r := range replicas {
+		if !r.ReadOnly {
+			writable = append(writable, i)
+		}
+	}
+	if len(writable) < 2 {
+		return -1
+	}
+	var placementSafe []int
+	for _, i := range writable {
+		if !IsMisplaced(otherThan(replicas, i), rp) {
+			placementSafe = append(placementSafe, i)
+		}
+	}
+	if len(placementSafe) > 0 {
+		return pickSmallestAmong(replicas, placementSafe)
+	}
+	return pickSmallestAmong(replicas, writable)
+}
+
+// PickOneMisplacedVolume selects the replica to delete and recreate at a
+// correct placement. This is relocation, not over-replication: unlike
+// PickOneReplicaToDelete it must still act on read-only replicas (e.g. a full
+// but misplaced volume), so it does not use the writable-survivor guard. It
+// prefers a replica whose removal leaves the others well placed, and falls
+// back to the smallest replica.
+func PickOneMisplacedVolume(replicas []Replica, rp *super_block.ReplicaPlacement) int {
+	var deletionCandidates []int
+	for i := range replicas {
+		if !IsMisplaced(otherThan(replicas, i), rp) {
+			deletionCandidates = append(deletionCandidates, i)
+		}
+	}
+	if smallest := pickSmallestAmong(replicas, deletionCandidates); smallest >= 0 {
+		return smallest
+	}
+	return PickSmallestReplica(replicas)
+}
+
+func pickSmallestAmong(replicas []Replica, candidates []int) int {
+	smallest := -1
+	for _, i := range candidates {
+		if smallest < 0 || lessReplica(replicas[i], replicas[smallest]) {
+			smallest = i
+		}
+	}
+	return smallest
+}
+
+func lessReplica(a, b Replica) bool {
+	if a.Size != b.Size {
+		return a.Size < b.Size
+	}
+	if a.ModifiedAtSecond != b.ModifiedAtSecond {
+		return a.ModifiedAtSecond < b.ModifiedAtSecond
+	}
+	return a.CompactRevision < b.CompactRevision
+}
+
+func otherThan(replicas []Replica, index int) (others []Replica) {
+	for i := range replicas {
+		if index != i {
+			others = append(others, replicas[i])
+		}
+	}
+	return
+}