snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/admin/dash/collect_collection_stats_test.go

@@ -0,0 +1,330 @@
+package dash
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+// TestCollectCollectionStatsECUnevenShards verifies that EC shards spread
+// across multiple disks are aggregated correctly, including the case where
+// the last data shard is smaller than the rest.
+func TestCollectCollectionStatsECUnevenShards(t *testing.T) {
+	// Standard 10+4 EC layout. Shards 0..9 are data, 10..13 are parity.
+	// Data shards 0..8 are 1000 bytes; data shard 9 is 500 bytes (uneven tail).
+	// Parity shards 10..13 are 1000 bytes each.
+	//
+	// Physical (raw) = 9*1000 + 500 + 4*1000 = 13500
+	// Logical (data only) = 9*1000 + 500 = 9500
+
+	nodeA := &master_pb.DataNodeInfo{
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"disk1": {
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{
+						Id:         42,
+						Collection: "bucket-a",
+						// Shards 0..6 held here.
+						EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6),
+						ShardSizes:  []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000},
+					},
+				},
+			},
+		},
+	}
+	nodeB := &master_pb.DataNodeInfo{
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"disk1": {
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{
+						Id:         42,
+						Collection: "bucket-a",
+						// Shards 7..13 held here. Shard 9 (data) is the short tail.
+						EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13),
+						ShardSizes:  []int64{1000, 1000, 500, 1000, 1000, 1000, 1000},
+					},
+				},
+			},
+		},
+	}
+
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						DataNodeInfos: []*master_pb.DataNodeInfo{nodeA, nodeB},
+					},
+				},
+			},
+		},
+	}
+
+	stats := collectCollectionStats(topo)
+
+	got, ok := stats["bucket-a"]
+	if !ok {
+		t.Fatalf("expected collection bucket-a in stats, got: %v", stats)
+	}
+
+	const wantPhysical int64 = 13500
+	const wantLogical int64 = 9500
+
+	if got.PhysicalSize != wantPhysical {
+		t.Errorf("PhysicalSize: got %d, want %d", got.PhysicalSize, wantPhysical)
+	}
+	if got.LogicalSize != wantLogical {
+		t.Errorf("LogicalSize: got %d, want %d", got.LogicalSize, wantLogical)
+	}
+}
+
+// TestCollectCollectionStatsECEmptyCollection verifies that EC shards with
+// an empty collection name are bucketed under "default".
+func TestCollectCollectionStatsECEmptyCollection(t *testing.T) {
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									"disk1": {
+										EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+											{
+												Id:          1,
+												Collection:  "",
+												EcIndexBits: (1 << 0) | (1 << 10),
+												ShardSizes:  []int64{2000, 2000},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	stats := collectCollectionStats(topo)
+
+	got, ok := stats["default"]
+	if !ok {
+		t.Fatalf("expected collection default in stats, got: %v", stats)
+	}
+	if got.PhysicalSize != 4000 {
+		t.Errorf("PhysicalSize: got %d, want 4000", got.PhysicalSize)
+	}
+	// Only shard 0 is a data shard; shard 10 is parity.
+	if got.LogicalSize != 2000 {
+		t.Errorf("LogicalSize: got %d, want 2000", got.LogicalSize)
+	}
+}
+
+// TestCollectCollectionStatsECFileAndDeleteCountAggregation verifies that
+// FileCount for an EC volume is deduped across nodes (every shard holder has
+// an identical .ecx, so the total entry count is taken once) while
+// DeleteCount is summed (each needle delete tombstones exactly one node's
+// .ecx, so the true delete total is the sum of every holder's local count).
+func TestCollectCollectionStatsECFileAndDeleteCountAggregation(t *testing.T) {
+	// Volume id=7 reported by three nodes. Every node reports file_count=100
+	// (same .ecx). Local delete counts: 5 + 3 + 2 = 10 deletes total.
+	// Expected live object count = 100 - 10 = 90.
+	makeNode := func(bits uint32, sizes []int64, deleteCount uint64) *master_pb.DataNodeInfo {
+		return &master_pb.DataNodeInfo{
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"disk1": {
+					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+						{
+							Id:          7,
+							Collection:  "bucket-a",
+							EcIndexBits: bits,
+							ShardSizes:  sizes,
+							FileCount:   100,
+							DeleteCount: deleteCount,
+						},
+					},
+				},
+			},
+		}
+	}
+
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							makeNode((1<<0)|(1<<1)|(1<<2)|(1<<3), []int64{1000, 1000, 1000, 1000}, 5),
+							makeNode((1<<4)|(1<<5)|(1<<6)|(1<<7), []int64{1000, 1000, 1000, 1000}, 3),
+							makeNode((1<<8)|(1<<9)|(1<<10)|(1<<11)|(1<<12)|(1<<13), []int64{1000, 1000, 1000, 1000, 1000, 1000}, 2),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	stats := collectCollectionStats(topo)
+
+	got, ok := stats["bucket-a"]
+	if !ok {
+		t.Fatalf("expected collection bucket-a in stats, got: %v", stats)
+	}
+	if got.FileCount != 90 {
+		t.Errorf("FileCount: got %d, want 90 (100 total - 10 deletes summed)", got.FileCount)
+	}
+	// Sanity: 14 shards × 1000 bytes physical, 10 data shards logical.
+	if got.PhysicalSize != 14000 {
+		t.Errorf("PhysicalSize: got %d, want 14000", got.PhysicalSize)
+	}
+	if got.LogicalSize != 10000 {
+		t.Errorf("LogicalSize: got %d, want 10000", got.LogicalSize)
+	}
+}
+
+// TestCollectCollectionStatsECFileCountMaxDedupe verifies that EC file_count
+// is taken as the max across reporting nodes rather than the first-seen
+// value. A node that has not yet finished loading .ecx reports file_count=0,
+// which previously poisoned the aggregate and rendered buckets backed by EC
+// volumes as "0 objects".
+func TestCollectCollectionStatsECFileCountMaxDedupe(t *testing.T) {
+	makeNode := func(bits uint32, sizes []int64, fileCount uint64) *master_pb.DataNodeInfo {
+		return &master_pb.DataNodeInfo{
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"disk1": {
+					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+						{
+							Id:          11,
+							Collection:  "bucket-b",
+							EcIndexBits: bits,
+							ShardSizes:  sizes,
+							FileCount:   fileCount,
+						},
+					},
+				},
+			},
+		}
+	}
+
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						// First-reporting node has a stale fileCount of 0,
+						// second node reports the authoritative 6.
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							makeNode((1<<0)|(1<<1)|(1<<2)|(1<<3)|(1<<4)|(1<<5)|(1<<6), []int64{1, 1, 1, 1, 1, 1, 1}, 0),
+							makeNode((1<<7)|(1<<8)|(1<<9)|(1<<10)|(1<<11)|(1<<12)|(1<<13), []int64{1, 1, 1, 1, 1, 1, 1}, 6),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	stats := collectCollectionStats(topo)
+	got, ok := stats["bucket-b"]
+	if !ok {
+		t.Fatalf("expected collection bucket-b in stats, got: %v", stats)
+	}
+	if got.FileCount != 6 {
+		t.Errorf("FileCount: got %d, want 6 (max across reporters)", got.FileCount)
+	}
+}
+
+// TestTotalCollectionFileCount verifies the cluster-wide chunk count counts a
+// chunk once per replicated volume and per EC volume, and nets out deletes.
+func TestTotalCollectionFileCount(t *testing.T) {
+	// Volume 1 has replication 001 (two copies), so both nodes report the same
+	// 101 chunks with 10 deleted: 91 live chunks in total, not 182 and not the
+	// 90 that halving each report would give.
+	replica := func() *master_pb.DiskInfo {
+		return &master_pb.DiskInfo{
+			VolumeInfos: []*master_pb.VolumeInformationMessage{
+				{
+					Id:               1,
+					Collection:       "bucket-a",
+					ReplicaPlacement: 1,
+					FileCount:        101,
+					DeleteCount:      10,
+				},
+			},
+		}
+	}
+	// EC volume 2 has its 20 chunks reported by every shard holder.
+	ecShards := func(bits uint32, sizes []int64) *master_pb.DiskInfo {
+		return &master_pb.DiskInfo{
+			EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+				{
+					Id:          2,
+					Collection:  "bucket-b",
+					EcIndexBits: bits,
+					ShardSizes:  sizes,
+					FileCount:   20,
+				},
+			},
+		}
+	}
+
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{DiskInfos: map[string]*master_pb.DiskInfo{
+								"disk1": replica(),
+								"disk2": ecShards((1<<0)|(1<<1), []int64{1, 1}),
+							}},
+							{DiskInfos: map[string]*master_pb.DiskInfo{
+								"disk1": replica(),
+								"disk2": ecShards((1<<2)|(1<<3), []int64{1, 1}),
+							}},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	if got := totalCollectionFileCount(topo); got != 111 {
+		t.Errorf("totalCollectionFileCount: got %d, want 111 (91 replicated + 20 EC)", got)
+	}
+}
+
+// TestTotalCollectionFileCountUnderReplicated verifies a volume whose second
+// replica has not reported yet still contributes its full live count.
+func TestTotalCollectionFileCountUnderReplicated(t *testing.T) {
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{DiskInfos: map[string]*master_pb.DiskInfo{
+								"disk1": {
+									VolumeInfos: []*master_pb.VolumeInformationMessage{
+										{
+											Id:               1,
+											Collection:       "bucket-a",
+											ReplicaPlacement: 1,
+											FileCount:        50,
+										},
+									},
+								},
+							}},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	if got := totalCollectionFileCount(topo); got != 50 {
+		t.Errorf("totalCollectionFileCount: got %d, want 50 (one replica reporting, not halved)", got)
+	}
+}