snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/admin/dash/volume_management.go

@@ -0,0 +1,651 @@
+package dash
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"sort"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+// GetClusterVolumes retrieves cluster volumes data with pagination, sorting, and filtering
+func (s *AdminServer) GetClusterVolumes(page int, pageSize int, sortBy string, sortOrder string, collection string) (*ClusterVolumesData, error) {
+	// Set defaults
+	if page < 1 {
+		page = 1
+	}
+	if pageSize < 1 || pageSize > 1000 {
+		pageSize = 100
+	}
+	if sortBy == "" {
+		sortBy = "id"
+	}
+	if sortOrder == "" {
+		sortOrder = "asc"
+	}
+	var volumes []VolumeWithTopology
+	var totalSize int64
+	var cachedTopologyInfo *master_pb.TopologyInfo
+
+	// Get detailed volume information via gRPC
+	err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
+		resp, err := pb.CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{})
+		if err != nil {
+			return err
+		}
+
+		// Cache the topology info for reuse
+		cachedTopologyInfo = resp.TopologyInfo
+
+		if resp.TopologyInfo != nil {
+			for _, dc := range resp.TopologyInfo.DataCenterInfos {
+				for _, rack := range dc.RackInfos {
+					for _, node := range rack.DataNodeInfos {
+						for _, diskInfo := range node.DiskInfos {
+							// Process regular volumes
+							for _, volInfo := range diskInfo.VolumeInfos {
+								volume := VolumeWithTopology{
+									VolumeInformationMessage: volInfo,
+									Server:                   node.Id,
+									DataCenter:               dc.Id,
+									Rack:                     rack.Id,
+								}
+								volumes = append(volumes, volume)
+								totalSize += int64(volInfo.Size)
+							}
+
+							// Process EC shards in the same loop
+							for _, ecShardInfo := range diskInfo.EcShardInfos {
+								// Add all shard sizes for this EC volume
+								for _, shardSize := range ecShardInfo.ShardSizes {
+									totalSize += shardSize
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Filter by collection if specified
+	if collection != "" {
+		var filteredVolumes []VolumeWithTopology
+		var filteredTotalSize int64
+		var filteredEcTotalSize int64
+
+		for _, volume := range volumes {
+			if matchesCollection(volume.Collection, collection) {
+				filteredVolumes = append(filteredVolumes, volume)
+				filteredTotalSize += int64(volume.Size)
+			}
+		}
+
+		// Filter EC shard sizes by collection using already processed data
+		// This reuses the topology traversal done above (lines 43-71) to avoid a second pass
+		if cachedTopologyInfo != nil {
+			for _, dc := range cachedTopologyInfo.DataCenterInfos {
+				for _, rack := range dc.RackInfos {
+					for _, node := range rack.DataNodeInfos {
+						for _, diskInfo := range node.DiskInfos {
+							for _, ecShardInfo := range diskInfo.EcShardInfos {
+								if matchesCollection(ecShardInfo.Collection, collection) {
+									// Add all shard sizes for this EC volume
+									for _, shardSize := range ecShardInfo.ShardSizes {
+										filteredEcTotalSize += shardSize
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+
+		volumes = filteredVolumes
+		totalSize = filteredTotalSize + filteredEcTotalSize
+	}
+
+	// Calculate unique data center, rack, disk type, collection, and version counts from filtered volumes
+	dataCenterMap := make(map[string]bool)
+	rackMap := make(map[string]bool)
+	diskTypeMap := make(map[string]bool)
+	collectionMap := make(map[string]bool)
+	versionMap := make(map[string]bool)
+	hasRemoteVolumes := false
+	for _, volume := range volumes {
+		if volume.RemoteStorageName != "" {
+			hasRemoteVolumes = true
+		}
+		if volume.DataCenter != "" {
+			dataCenterMap[volume.DataCenter] = true
+		}
+		if volume.Rack != "" {
+			rackMap[volume.Rack] = true
+		}
+		diskType := volume.DiskType
+		if diskType == "" {
+			diskType = "hdd" // Default to hdd if not specified
+		}
+		diskTypeMap[diskType] = true
+
+		// Handle collection for display purposes
+		collectionName := volume.Collection
+		if collectionName == "" {
+			collectionName = "default"
+		}
+		collectionMap[collectionName] = true
+
+		versionMap[fmt.Sprintf("%d", volume.Version)] = true
+	}
+	dataCenterCount := len(dataCenterMap)
+	rackCount := len(rackMap)
+	diskTypeCount := len(diskTypeMap)
+	collectionCount := len(collectionMap)
+	versionCount := len(versionMap)
+
+	// Sort volumes
+	s.sortVolumes(volumes, sortBy, sortOrder)
+
+	// Get volume size limit from master
+	var volumeSizeLimit uint64
+	err = s.WithMasterClient(func(client master_pb.SeaweedClient) error {
+		resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		volumeSizeLimit = uint64(resp.VolumeSizeLimitMB) * 1024 * 1024 // Convert MB to bytes
+		return nil
+	})
+	if err != nil {
+		// If we can't get the limit, set a default
+		volumeSizeLimit = 30 * 1024 * 1024 * 1024 // 30GB default
+	}
+
+	// Calculate pagination
+	totalVolumes := len(volumes)
+	totalPages := (totalVolumes + pageSize - 1) / pageSize
+	if totalPages == 0 {
+		totalPages = 1
+	}
+
+	// Apply pagination
+	startIndex := (page - 1) * pageSize
+	endIndex := startIndex + pageSize
+	if startIndex >= totalVolumes {
+		volumes = []VolumeWithTopology{}
+	} else {
+		if endIndex > totalVolumes {
+			endIndex = totalVolumes
+		}
+		volumes = volumes[startIndex:endIndex]
+	}
+
+	// Determine conditional display flags and extract single values
+	showDataCenterColumn := dataCenterCount > 1
+	showRackColumn := rackCount > 1
+	// Remote-tiered volumes surface in the disk type column, so show it
+	// whenever any volume lives on a remote tier.
+	showDiskTypeColumn := diskTypeCount > 1 || hasRemoteVolumes
+	showCollectionColumn := collectionCount > 1 && collection == "" // Hide column when filtering by collection
+	showVersionColumn := versionCount > 1
+
+	var singleDataCenter, singleRack, singleDiskType, singleCollection, singleVersion string
+	var allVersions, allDiskTypes []string
+
+	if dataCenterCount == 1 {
+		for dc := range dataCenterMap {
+			singleDataCenter = dc
+			break
+		}
+	}
+	if rackCount == 1 {
+		for rack := range rackMap {
+			singleRack = rack
+			break
+		}
+	}
+	if diskTypeCount == 1 {
+		for diskType := range diskTypeMap {
+			singleDiskType = diskType
+			break
+		}
+	} else {
+		// Collect all disk types and sort them
+		for diskType := range diskTypeMap {
+			allDiskTypes = append(allDiskTypes, diskType)
+		}
+		sort.Strings(allDiskTypes)
+	}
+	if collectionCount == 1 {
+		for collection := range collectionMap {
+			singleCollection = collection
+			break
+		}
+	}
+	if versionCount == 1 {
+		for version := range versionMap {
+			singleVersion = "v" + version
+			break
+		}
+	} else {
+		// Collect all versions and sort them
+		for version := range versionMap {
+			allVersions = append(allVersions, "v"+version)
+		}
+		sort.Strings(allVersions)
+	}
+
+	return &ClusterVolumesData{
+		Volumes:              volumes,
+		TotalVolumes:         totalVolumes,
+		TotalSize:            totalSize,
+		VolumeSizeLimit:      volumeSizeLimit,
+		LastUpdated:          time.Now(),
+		CurrentPage:          page,
+		TotalPages:           totalPages,
+		PageSize:             pageSize,
+		SortBy:               sortBy,
+		SortOrder:            sortOrder,
+		DataCenterCount:      dataCenterCount,
+		RackCount:            rackCount,
+		DiskTypeCount:        diskTypeCount,
+		CollectionCount:      collectionCount,
+		VersionCount:         versionCount,
+		ShowDataCenterColumn: showDataCenterColumn,
+		ShowRackColumn:       showRackColumn,
+		ShowDiskTypeColumn:   showDiskTypeColumn,
+		ShowCollectionColumn: showCollectionColumn,
+		ShowVersionColumn:    showVersionColumn,
+		SingleDataCenter:     singleDataCenter,
+		SingleRack:           singleRack,
+		SingleDiskType:       singleDiskType,
+		SingleCollection:     singleCollection,
+		SingleVersion:        singleVersion,
+		AllVersions:          allVersions,
+		AllDiskTypes:         allDiskTypes,
+		FilterCollection:     collection,
+	}, nil
+}
+
+// sortVolumes sorts the volumes slice based on the specified field and order
+func (s *AdminServer) sortVolumes(volumes []VolumeWithTopology, sortBy string, sortOrder string) {
+	sort.Slice(volumes, func(i, j int) bool {
+		var less bool
+
+		switch sortBy {
+		case "id":
+			less = volumes[i].Id < volumes[j].Id
+		case "server":
+			less = volumes[i].Server < volumes[j].Server
+		case "datacenter":
+			less = volumes[i].DataCenter < volumes[j].DataCenter
+		case "rack":
+			less = volumes[i].Rack < volumes[j].Rack
+		case "collection":
+			less = volumes[i].Collection < volumes[j].Collection
+		case "size":
+			less = volumes[i].Size < volumes[j].Size
+		case "filecount":
+			less = volumes[i].FileCount < volumes[j].FileCount
+		case "replication":
+			less = volumes[i].ReplicaPlacement < volumes[j].ReplicaPlacement
+		case "disktype":
+			less = volumes[i].DiskType < volumes[j].DiskType
+		case "version":
+			less = volumes[i].Version < volumes[j].Version
+		default:
+			less = volumes[i].Id < volumes[j].Id
+		}
+
+		if sortOrder == "desc" {
+			return !less
+		}
+		return less
+	})
+}
+
+// GetVolumeDetails retrieves detailed information about a specific volume
+func (s *AdminServer) GetVolumeDetails(volumeID uint32, server string) (*VolumeDetailsData, error) {
+	var primaryVolume VolumeWithTopology
+	var replicas []VolumeWithTopology
+	var volumeSizeLimit uint64
+	var found bool
+
+	// Find the volume and all its replicas in the cluster
+	err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
+		resp, err := client.VolumeList(context.Background(), &master_pb.VolumeListRequest{VolumeIds: []uint32{volumeID}})
+		if err != nil {
+			return err
+		}
+
+		if resp.TopologyInfo != nil {
+			for _, dc := range resp.TopologyInfo.DataCenterInfos {
+				for _, rack := range dc.RackInfos {
+					for _, node := range rack.DataNodeInfos {
+						for _, diskInfo := range node.DiskInfos {
+							for _, volInfo := range diskInfo.VolumeInfos {
+								// An older master ignores the filter.
+								if volInfo.Id == volumeID {
+									diskType := volInfo.DiskType
+									if diskType == "" {
+										diskType = "hdd"
+									}
+
+									volume := VolumeWithTopology{
+										VolumeInformationMessage: volInfo,
+										Server:                   node.Id,
+										DataCenter:               dc.Id,
+										Rack:                     rack.Id,
+									}
+
+									// If this is the requested server, it's the primary volume
+									if node.Id == server {
+										primaryVolume = volume
+										found = true
+									} else {
+										// This is a replica on another server
+										replicas = append(replicas, volume)
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	if !found {
+		return nil, fmt.Errorf("volume %d not found on server %s", volumeID, server)
+	}
+
+	// Get volume size limit from master
+	err = s.WithMasterClient(func(client master_pb.SeaweedClient) error {
+		resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		volumeSizeLimit = uint64(resp.VolumeSizeLimitMB) * 1024 * 1024 // Convert MB to bytes
+		return nil
+	})
+
+	if err != nil {
+		// If we can't get the limit, set a default
+		volumeSizeLimit = 30 * 1024 * 1024 * 1024 // 30GB default
+	}
+
+	return &VolumeDetailsData{
+		Volume:           primaryVolume,
+		Replicas:         replicas,
+		VolumeSizeLimit:  volumeSizeLimit,
+		ReplicationCount: len(replicas) + 1, // Include the primary volume
+		LastUpdated:      time.Now(),
+	}, nil
+}
+
+// SetVolumeReadOnly changes the access mode of a single volume replica.
+func (s *AdminServer) SetVolumeReadOnly(ctx context.Context, volumeID uint32, server string, readOnly bool) error {
+	var address pb.ServerAddress
+	err := s.masterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {
+		resp, err := client.VolumeList(ctx, &master_pb.VolumeListRequest{VolumeIds: []uint32{volumeID}})
+		if err != nil {
+			return err
+		}
+		address, err = volumeReplicaAddress(resp.GetTopologyInfo(), volumeID, server)
+		return err
+	})
+	if err != nil {
+		return err
+	}
+
+	return s.WithVolumeServerClient(address, func(client volume_server_pb.VolumeServerClient) error {
+		if readOnly {
+			_, err := client.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
+				VolumeId: volumeID,
+				Persist:  true,
+			})
+			return err
+		}
+		_, err := client.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: volumeID})
+		return err
+	})
+}
+
+// Resolve the selected replica through the master rather than dialing a
+// user-supplied address. Node IDs can differ from their network addresses.
+func volumeReplicaAddress(topology *master_pb.TopologyInfo, volumeID uint32, server string) (pb.ServerAddress, error) {
+	for _, dc := range topology.GetDataCenterInfos() {
+		for _, rack := range dc.GetRackInfos() {
+			for _, node := range rack.GetDataNodeInfos() {
+				if node.GetId() != server {
+					continue
+				}
+				for _, disk := range node.GetDiskInfos() {
+					for _, volume := range disk.GetVolumeInfos() {
+						// Older masters may ignore the volume ID filter.
+						if volume.GetId() == volumeID {
+							return pb.NewServerAddressFromDataNode(node), nil
+						}
+					}
+				}
+			}
+		}
+	}
+	return "", fmt.Errorf("volume %d not found on server %s", volumeID, server)
+}
+
+// VacuumVolume performs a vacuum operation on a specific volume
+func (s *AdminServer) VacuumVolume(volumeID int, server string) error {
+	// Validate volumeID range before converting to uint32
+	if volumeID < 0 || uint64(volumeID) > math.MaxUint32 {
+		return fmt.Errorf("volume ID out of range: %d", volumeID)
+	}
+	return s.WithMasterClient(func(client master_pb.SeaweedClient) error {
+		_, err := client.VacuumVolume(context.Background(), &master_pb.VacuumVolumeRequest{
+			// lgtm[go/incorrect-integer-conversion]
+			// Safe conversion: volumeID has been validated to be in range [0, 0xFFFFFFFF] above
+			VolumeId:         uint32(volumeID),
+			GarbageThreshold: 0.0001, // A very low threshold to ensure all garbage is collected
+			Collection:       "",     // Empty for all collections
+		})
+		return err
+	})
+}
+
+// GetClusterVolumeServers retrieves cluster volume servers data including EC shard information
+func (s *AdminServer) GetClusterVolumeServers() (*ClusterVolumeServersData, error) {
+	var volumeServerMap map[string]*VolumeServer
+
+	// Fetch public URL mapping from master HTTP API
+	publicUrls := s.fetchPublicUrlMap()
+
+	// Make only ONE VolumeList call and use it for both topology building AND EC shard processing
+	err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
+		resp, err := pb.CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{})
+		if err != nil {
+			return err
+		}
+
+		// Get volume size limit from response, default to 30GB if not set
+		volumeSizeLimitMB := resp.VolumeSizeLimitMb
+		if volumeSizeLimitMB == 0 {
+			volumeSizeLimitMB = 30000 // default to 30000MB (30GB)
+		}
+
+		// Build basic topology from the VolumeList response (replaces GetClusterTopology call)
+		volumeServerMap = make(map[string]*VolumeServer)
+
+		if resp.TopologyInfo != nil {
+			// Process topology to build basic volume server info (similar to cluster_topology.go logic)
+			for _, dc := range resp.TopologyInfo.DataCenterInfos {
+				for _, rack := range dc.RackInfos {
+					for _, node := range rack.DataNodeInfos {
+						// Initialize volume server if not exists
+						if volumeServerMap[node.Id] == nil {
+							// Look up PublicUrl from master HTTP API
+							nodeAddr := node.Address
+							if nodeAddr == "" {
+								nodeAddr = node.Id
+							}
+							publicUrl := publicUrls[nodeAddr]
+							if publicUrl == "" {
+								publicUrl = nodeAddr
+							}
+							volumeServerMap[node.Id] = &VolumeServer{
+								Address:        node.Id,
+								PublicURL:      publicUrl,
+								DataCenter:     dc.Id,
+								Rack:           rack.Id,
+								Volumes:        0,
+								DiskUsage:      0,
+								DiskCapacity:   0,
+								EcVolumes:      0,
+								EcShards:       0,
+								EcShardDetails: []VolumeServerEcInfo{},
+							}
+						}
+						vs := volumeServerMap[node.Id]
+
+						// Process EC shard information for this server at volume server level (not per-disk)
+						ecVolumeMap := make(map[uint32]*VolumeServerEcInfo)
+						// Temporary map to accumulate shard info across disks
+						ecShardAccumulator := make(map[uint32][]*master_pb.VolumeEcShardInformationMessage)
+
+						// Process disk information
+						for _, diskInfo := range node.DiskInfos {
+							vs.MaxVolumes += int(diskInfo.MaxVolumeCount)
+							// Prefer the real physical disk capacity the volume server
+							// reports; the slot-based estimate overstates capacity when
+							// maxVolumeCount is configured higher than the disk holds.
+							if diskInfo.DiskTotalBytes > 0 {
+								vs.DiskCapacity += int64(diskInfo.DiskTotalBytes)
+							} else {
+								vs.DiskCapacity += int64(diskInfo.MaxVolumeCount) * int64(volumeSizeLimitMB) * 1024 * 1024
+							}
+
+							// Count regular volumes and calculate disk usage.
+							// A remote-tiered volume reports its cloud object's
+							// size; keep those bytes out of the local disk usage
+							// that is compared against DiskCapacity.
+							for _, volInfo := range diskInfo.VolumeInfos {
+								vs.Volumes++
+								if volInfo.RemoteStorageName != "" {
+									vs.RemoteSize += int64(volInfo.Size)
+								} else {
+									vs.DiskUsage += int64(volInfo.Size)
+								}
+							}
+
+							// Accumulate EC shard information across all disks for this volume server
+							for _, ecShardInfo := range diskInfo.EcShardInfos {
+								volumeId := ecShardInfo.Id
+								ecShardAccumulator[volumeId] = append(ecShardAccumulator[volumeId], ecShardInfo)
+							}
+						}
+
+						// Process accumulated EC shard information per volume
+						for volumeId, ecShardInfos := range ecShardAccumulator {
+							if len(ecShardInfos) == 0 {
+								continue
+							}
+
+							// Initialize EC volume info
+							ecInfo := &VolumeServerEcInfo{
+								VolumeID:     volumeId,
+								Collection:   ecShardInfos[0].Collection,
+								ShardCount:   0,
+								EcIndexBits:  0,
+								ShardNumbers: []int{},
+								ShardSizes:   make(map[int]int64),
+								TotalSize:    0,
+							}
+
+							// Merge EcIndexBits from all disks and collect shard sizes
+							allShardSizes := make(map[erasure_coding.ShardId]int64)
+							for _, ecShardInfo := range ecShardInfos {
+								si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo)
+								ecInfo.EcIndexBits |= si.Bitmap()
+
+								// Collect shard sizes from this disk
+								for _, id := range si.Ids() {
+									allShardSizes[id] += int64(si.Size(id))
+								}
+							}
+
+							// Process final merged shard information
+							for shardId := range allShardSizes {
+								ecInfo.ShardCount++
+								ecInfo.ShardNumbers = append(ecInfo.ShardNumbers, int(shardId))
+								vs.EcShards++
+
+								// Add shard size if available
+								if shardSize, exists := allShardSizes[shardId]; exists {
+									ecInfo.ShardSizes[int(shardId)] = shardSize
+									ecInfo.TotalSize += shardSize
+									vs.DiskUsage += shardSize // Add EC shard size to total disk usage
+								}
+							}
+
+							ecVolumeMap[volumeId] = ecInfo
+						}
+
+						// Convert EC volume map to slice and update volume server (after processing all disks)
+						for _, ecInfo := range ecVolumeMap {
+							vs.EcShardDetails = append(vs.EcShardDetails, *ecInfo)
+							vs.EcVolumes++
+						}
+					}
+				}
+			}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert map back to slice
+	var volumeServers []VolumeServer
+	for _, vs := range volumeServerMap {
+		volumeServers = append(volumeServers, *vs)
+	}
+
+	// Sort volume servers by address for consistent ordering on page refresh
+	sort.Slice(volumeServers, func(i, j int) bool {
+		return volumeServers[i].GetDisplayAddress() < volumeServers[j].GetDisplayAddress()
+	})
+
+	var totalCapacity int64
+	var totalVolumes int
+	for _, vs := range volumeServers {
+		totalCapacity += vs.DiskCapacity
+		totalVolumes += vs.Volumes
+	}
+
+	return &ClusterVolumeServersData{
+		VolumeServers:      volumeServers,
+		TotalVolumeServers: len(volumeServers),
+		TotalVolumes:       totalVolumes,
+		TotalCapacity:      totalCapacity,
+		LastUpdated:        time.Now(),
+	}, nil
+}