snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/shell/command_volume_list_test.go

@@ -0,0 +1,307 @@
+package shell
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/stretchr/testify/assert"
+
+	//"google.golang.org/protobuf/proto"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/golang/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func TestParsing(t *testing.T) {
+	topo := parseOutput(topoData)
+
+	assert.Equal(t, 5, len(topo.DataCenterInfos))
+
+	topo = parseOutput(topoData2)
+
+	dataNodes := topo.DataCenterInfos[0].RackInfos[0].DataNodeInfos
+	assert.Equal(t, 14, len(dataNodes))
+	diskInfo := dataNodes[0].DiskInfos[""]
+	assert.Equal(t, 1559, len(diskInfo.VolumeInfos))
+	assert.Equal(t, 6740, len(diskInfo.EcShardInfos))
+
+}
+
+// TODO: actually parsing all fields would be nice...
+func parseOutput(output string) *master_pb.TopologyInfo {
+	lines := strings.Split(output, "\n")
+	var topo *master_pb.TopologyInfo
+	var dc *master_pb.DataCenterInfo
+	var rack *master_pb.RackInfo
+	var dn *master_pb.DataNodeInfo
+	var disk *master_pb.DiskInfo
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		parts := strings.Split(line, " ")
+		switch parts[0] {
+		case "Topology":
+			if topo == nil {
+				topo = &master_pb.TopologyInfo{
+					Id: parts[1],
+				}
+			}
+		case "DataCenter":
+			if dc == nil {
+				dc = &master_pb.DataCenterInfo{
+					Id: parts[1],
+				}
+				topo.DataCenterInfos = append(topo.DataCenterInfos, dc)
+			} else {
+				dc = nil
+			}
+		case "Rack":
+			if rack == nil {
+				rack = &master_pb.RackInfo{
+					Id: parts[1],
+				}
+				dc.RackInfos = append(dc.RackInfos, rack)
+			} else {
+				rack = nil
+			}
+		case "DataNode":
+			if dn == nil {
+				dn = &master_pb.DataNodeInfo{
+					Id:        parts[1],
+					DiskInfos: make(map[string]*master_pb.DiskInfo),
+				}
+				rack.DataNodeInfos = append(rack.DataNodeInfos, dn)
+			} else {
+				dn = nil
+			}
+		case "Disk":
+			if disk == nil {
+				diskType := parts[1][:strings.Index(parts[1], "(")]
+				volumeCountStr := parts[1][strings.Index(parts[1], ":")+1 : strings.Index(parts[1], "/")]
+				maxVolumeCountStr := parts[1][strings.Index(parts[1], "/")+1:]
+				maxVolumeCount, _ := strconv.Atoi(maxVolumeCountStr)
+				volumeCount, _ := strconv.Atoi(volumeCountStr)
+				disk = &master_pb.DiskInfo{
+					Type:           diskType,
+					MaxVolumeCount: int64(maxVolumeCount),
+					VolumeCount:    int64(volumeCount),
+				}
+				dn.DiskInfos[types.ToDiskType(diskType).String()] = disk
+			} else {
+				disk = nil
+			}
+		case "volume":
+			volumeLine := line[len("volume "):]
+			volume := &master_pb.VolumeInformationMessage{}
+			proto.UnmarshalText(volumeLine, volume)
+			disk.VolumeInfos = append(disk.VolumeInfos, volume)
+		case "ec":
+			ecVolumeLine := line[len("ec volume "):]
+			ecShard := &master_pb.VolumeEcShardInformationMessage{}
+			for _, part := range strings.Split(ecVolumeLine, " ") {
+				if strings.HasPrefix(part, "id:") {
+					id, _ := strconv.ParseInt(part[len("id:"):], 10, 64)
+					ecShard.Id = uint32(id)
+				}
+				if strings.HasPrefix(part, "collection:") {
+					ecShard.Collection = part[len("collection:"):]
+				}
+				// TODO: we need to parse EC shard sizes as well
+				if strings.HasPrefix(part, "shards:") {
+					shards := part[len("shards:["):]
+					shards = strings.TrimRight(shards, "]")
+					shardsInfo := erasure_coding.NewShardsInfo()
+					for _, shardId := range strings.Split(shards, ",") {
+						sid, _ := strconv.Atoi(shardId)
+						shardsInfo.Set(erasure_coding.NewShardInfo(erasure_coding.ShardId(sid), 0))
+					}
+					ecShard.EcIndexBits = shardsInfo.Bitmap()
+					ecShard.ShardSizes = shardsInfo.SizesInt64()
+				}
+			}
+			disk.EcShardInfos = append(disk.EcShardInfos, ecShard)
+		}
+	}
+
+	return topo
+}
+
+// TestWriteDataNodeInfo_SplitsCollapsedDisksByPhysicalDiskId verifies that
+// the verbose Disk block in volume.list shows one entry per physical disk
+// when the master collapsed several same-type disks under a single
+// DiskInfos["hdd"] map entry. Before the split, six physical disks would
+// appear as "Disk hdd ... id:0" with all volumes stacked on it.
+func TestWriteDataNodeInfo_SplitsCollapsedDisksByPhysicalDiskId(t *testing.T) {
+	dn := &master_pb.DataNodeInfo{
+		Id: "node1:8081",
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"hdd": {
+				Type:           "hdd",
+				MaxVolumeCount: 60,
+				VolumeInfos: []*master_pb.VolumeInformationMessage{
+					{Id: 1, DiskId: 0, Collection: "c"},
+					{Id: 2, DiskId: 1, Collection: "c"},
+					{Id: 3, DiskId: 2, Collection: "c"},
+				},
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{Id: 100, DiskId: 3, Collection: "c", EcIndexBits: 1},
+					{Id: 101, DiskId: 4, Collection: "c", EcIndexBits: 1},
+					{Id: 102, DiskId: 5, Collection: "c", EcIndexBits: 1},
+				},
+			},
+		},
+	}
+
+	c := &commandVolumeList{}
+	fs := flag.NewFlagSet("volume.list", flag.ContinueOnError)
+	c.dataCenter = fs.String("dataCenter", "", "")
+	c.rack = fs.String("rack", "", "")
+	c.dataNode = fs.String("dataNode", "", "")
+	c.readonly = fs.Bool("readonly", false, "")
+	c.writable = fs.Bool("writable", false, "")
+	c.volumeId = fs.Uint64("volumeId", 0, "")
+
+	var buf bytes.Buffer
+	verbosity := 5
+	rackInvocations := 0
+	c.writeDataNodeInfo(&buf, dn, verbosity, func() { rackInvocations++ })
+
+	out := buf.String()
+	// Match the "id:N\n" line ending so substring checks don't accept
+	// unrelated tokens like "ec volume id:101" as a match for id:1.
+	for diskID := 0; diskID < 6; diskID++ {
+		needle := fmt.Sprintf("id:%d\n", diskID)
+		if !strings.Contains(out, needle) {
+			t.Errorf("output missing %q; got:\n%s", needle, out)
+		}
+	}
+
+	// The parent's outRackInfo callback rides the same code path as the
+	// DataNode header — it fires from the inner writeDiskInfo callback,
+	// once per disk before the fix. After the fix the header guard runs
+	// outRackInfo at most once per DataNode. This proxy lets us pin the
+	// "header printed once" invariant without depending on the exact
+	// format of the rendered DataNode line.
+	if rackInvocations != 1 {
+		t.Errorf("DataNode header callback ran %d times; want 1 (regression: header printed once per split disk)", rackInvocations)
+	}
+}
+
+func TestWriteTopologyInfo_PrintsParentHeadersOnce(t *testing.T) {
+	topo := topoFromNodes(
+		volNode("node1:8081", &master_pb.VolumeInformationMessage{Id: 1, Collection: "c"}),
+		volNode("node2:8081", &master_pb.VolumeInformationMessage{Id: 2, Collection: "c"}),
+	)
+	topo.DiskInfos = map[string]*master_pb.DiskInfo{"hdd": {Type: "hdd"}}
+	topo.DataCenterInfos[0].DiskInfos = map[string]*master_pb.DiskInfo{"hdd": {Type: "hdd"}}
+	topo.DataCenterInfos[0].RackInfos[0].DiskInfos = map[string]*master_pb.DiskInfo{"hdd": {Type: "hdd"}}
+
+	c := &commandVolumeList{}
+	fs := flag.NewFlagSet("volume.list", flag.ContinueOnError)
+	c.dataCenter = fs.String("dataCenter", "", "")
+	c.rack = fs.String("rack", "", "")
+	c.dataNode = fs.String("dataNode", "", "")
+	c.readonly = fs.Bool("readonly", false, "")
+	c.writable = fs.Bool("writable", false, "")
+	c.volumeId = fs.Uint64("volumeId", 0, "")
+
+	var dcBuf bytes.Buffer
+	c.writeTopologyInfo(&dcBuf, topo, 30000, 1)
+	assert.Equal(t, 1, strings.Count(dcBuf.String(), "  DataCenter dc1 hdd("))
+
+	var rackBuf bytes.Buffer
+	c.writeTopologyInfo(&rackBuf, topo, 30000, 2)
+	assert.Equal(t, 1, strings.Count(rackBuf.String(), "    Rack rack1 hdd("))
+}
+
+// volNode builds a single-node topology from (volumeId, collection) pairs so the
+// duplicate-detection tests can describe a cluster compactly.
+func volNode(nodeId string, volumes ...*master_pb.VolumeInformationMessage) *master_pb.DataNodeInfo {
+	return &master_pb.DataNodeInfo{
+		Id: nodeId,
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"hdd": {Type: "hdd", VolumeInfos: volumes},
+		},
+	}
+}
+
+func topoFromNodes(nodes ...*master_pb.DataNodeInfo) *master_pb.TopologyInfo {
+	return &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{{
+			Id:        "dc1",
+			RackInfos: []*master_pb.RackInfo{{Id: "rack1", DataNodeInfos: nodes}},
+		}},
+	}
+}
+
+// TestFindDuplicateVolumeIds covers the dangerous condition where the master
+// reused a volume id across two collections (e.g. id 59788 in both infra-backup
+// and ai-news-data-infrastructure) — the state that lets collection.delete
+// destroy the wrong collection's data. The detector must flag exactly the shared
+// ids, while same-id replicas of one volume must NOT be flagged.
+func TestFindDuplicateVolumeIds(t *testing.T) {
+	t.Run("flags id shared across collections", func(t *testing.T) {
+		topo := topoFromNodes(
+			volNode("srv0487:8082", &master_pb.VolumeInformationMessage{Id: 59788, Collection: "infra-backup"}),
+			volNode("srv0502:8083", &master_pb.VolumeInformationMessage{Id: 59788, Collection: "ai-news-data-infrastructure"}),
+			// A clean, single-collection volume that must not be flagged.
+			volNode("srv0487:8085", &master_pb.VolumeInformationMessage{Id: 7069, Collection: "ai-news-data-infrastructure"}),
+		)
+		dup := findDuplicateVolumeIds(topo)
+		assert.Equal(t, map[uint32][]string{
+			59788: {"ai-news-data-infrastructure", "infra-backup"},
+		}, dup)
+	})
+
+	t.Run("replicas of one volume are not duplicates", func(t *testing.T) {
+		// Same id on three nodes, all the same collection: a normal replicated
+		// volume, not a cross-collection clash.
+		topo := topoFromNodes(
+			volNode("n1", &master_pb.VolumeInformationMessage{Id: 42, Collection: "c"}),
+			volNode("n2", &master_pb.VolumeInformationMessage{Id: 42, Collection: "c"}),
+			volNode("n3", &master_pb.VolumeInformationMessage{Id: 42, Collection: "c"}),
+		)
+		assert.Empty(t, findDuplicateVolumeIds(topo))
+	})
+
+	t.Run("default empty collection counts as its own collection", func(t *testing.T) {
+		topo := topoFromNodes(
+			volNode("n1", &master_pb.VolumeInformationMessage{Id: 1, Collection: ""}),
+			volNode("n2", &master_pb.VolumeInformationMessage{Id: 1, Collection: "x"}),
+		)
+		assert.Equal(t, map[uint32][]string{1: {"", "x"}}, findDuplicateVolumeIds(topo))
+	})
+
+	t.Run("normal volume and ec shard sharing an id across collections", func(t *testing.T) {
+		topo := topoFromNodes(volNode("n1", &master_pb.VolumeInformationMessage{Id: 5, Collection: "a"}))
+		topo.DataCenterInfos[0].RackInfos[0].DataNodeInfos[0].DiskInfos["hdd"].EcShardInfos =
+			[]*master_pb.VolumeEcShardInformationMessage{{Id: 5, Collection: "b", EcIndexBits: 1}}
+		assert.Equal(t, map[uint32][]string{5: {"a", "b"}}, findDuplicateVolumeIds(topo))
+	})
+}
+
+func TestWriteDuplicateVolumeIdWarning(t *testing.T) {
+	var buf bytes.Buffer
+	writeDuplicateVolumeIdWarning(&buf, map[uint32][]string{
+		59788: {"ai-news-data-infrastructure", "infra-backup"},
+		1:     {"", "x"},
+	})
+	out := buf.String()
+	assert.Contains(t, out, "WARNING: 2 volume id(s) exist in more than one collection")
+	assert.Contains(t, out, "volume 59788 in collections: ai-news-data-infrastructure, infra-backup")
+	// Empty collection name is rendered readably rather than as a blank.
+	assert.Contains(t, out, `volume 1 in collections: "" (default), x`)
+	// Lowest id first.
+	assert.Less(t, strings.Index(out, "volume 1 "), strings.Index(out, "volume 59788 "))
+
+	// Clean topology prints nothing.
+	var empty bytes.Buffer
+	writeDuplicateVolumeIdWarning(&empty, nil)
+	assert.Empty(t, empty.String())
+}