snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,135 @@
package balance
import (
"context"
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation/volume_move"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
"github.com/seaweedfs/seaweedfs/weed/worker/types/base"
"google.golang.org/grpc"
)
// BalanceTask implements the Task interface
type BalanceTask struct {
*base.BaseTask
server string
volumeID uint32
collection string
progress float64
grpcDialOption grpc.DialOption
}
// NewBalanceTask creates a new balance task instance
func NewBalanceTask(id string, server string, volumeID uint32, collection string, grpcDialOption grpc.DialOption) *BalanceTask {
return &BalanceTask{
BaseTask: base.NewBaseTask(id, types.TaskTypeBalance),
server: server,
volumeID: volumeID,
collection: collection,
grpcDialOption: grpcDialOption,
}
}
// Execute implements the Task interface
func (t *BalanceTask) Execute(ctx context.Context, params *worker_pb.TaskParams) error {
if params == nil {
return fmt.Errorf("task parameters are required")
}
balanceParams := params.GetBalanceParams()
if balanceParams == nil {
return fmt.Errorf("balance parameters are required")
}
// Get source and destination from unified arrays
if len(params.Sources) == 0 {
return fmt.Errorf("source is required for balance task")
}
if len(params.Targets) == 0 {
return fmt.Errorf("target is required for balance task")
}
sourceNode := params.Sources[0].Node
destNode := params.Targets[0].Node
if sourceNode == "" {
return fmt.Errorf("source node is required for balance task")
}
if destNode == "" {
return fmt.Errorf("destination node is required for balance task")
}
t.GetLogger().WithFields(map[string]interface{}{
"volume_id": t.volumeID,
"source": sourceNode,
"destination": destNode,
"collection": t.collection,
}).Info("Starting balance task - moving volume")
// The move sequence — freeze the source, copy, tail, verify the target
// matches the source before the destructive delete — is shared with the
// shell's volume.move/volume.balance commands.
mover := volume_move.NewMover(t.grpcDialOption)
err := mover.LiveMoveVolume(ctx, needle.VolumeId(t.volumeID), pb.ServerAddress(sourceNode), pb.ServerAddress(destNode), volume_move.VolumeMoveOptions{
IdleTimeout: 60 * time.Second,
IoBytePerSecond: balanceParams.IoBytePerSecond,
Progress: func(percent float64, stage string) {
t.ReportProgress(percent)
t.GetLogger().Info("move stage: %s", stage)
},
})
if err != nil {
return fmt.Errorf("move volume %d from %s to %s: %w", t.volumeID, sourceNode, destNode, err)
}
glog.Infof("Balance task completed successfully: volume %d moved from %s to %s",
t.volumeID, sourceNode, destNode)
return nil
}
// Validate implements the UnifiedTask interface
func (t *BalanceTask) Validate(params *worker_pb.TaskParams) error {
if params == nil {
return fmt.Errorf("task parameters are required")
}
balanceParams := params.GetBalanceParams()
if balanceParams == nil {
return fmt.Errorf("balance parameters are required")
}
if params.VolumeId != t.volumeID {
return fmt.Errorf("volume ID mismatch: expected %d, got %d", t.volumeID, params.VolumeId)
}
// Validate that at least one source matches our server
found := false
for _, source := range params.Sources {
if source.Node == t.server {
found = true
break
}
}
if !found {
return fmt.Errorf("no source matches expected server %s", t.server)
}
return nil
}
// EstimateTime implements the UnifiedTask interface
func (t *BalanceTask) EstimateTime(params *worker_pb.TaskParams) time.Duration {
// Basic estimate based on simulated steps
return 14 * time.Second // Sum of all step durations
}
// GetProgress returns current progress
func (t *BalanceTask) GetProgress() float64 {
return t.progress
}

View File

@@ -0,0 +1,204 @@
package balance
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/admin/config"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
)
// Config extends BaseConfig with balance-specific settings
type Config struct {
base.BaseConfig
ImbalanceThreshold float64 `json:"imbalance_threshold"`
MinServerCount int `json:"min_server_count"`
IoBytePerSecond int64 `json:"io_byte_per_second"`
DataCenterFilter string `json:"-"` // per-detection-run, not persisted
RackFilter string `json:"-"` // per-detection-run, not persisted
NodeFilter string `json:"-"` // per-detection-run, not persisted
}
// NewDefaultConfig creates a new default balance configuration
func NewDefaultConfig() *Config {
return &Config{
BaseConfig: base.BaseConfig{
Enabled: true,
ScanIntervalSeconds: 30 * 60, // 30 minutes
MaxConcurrent: 1,
},
ImbalanceThreshold: 0.2, // 20%
MinServerCount: 2,
}
}
// GetConfigSpec returns the configuration schema for balance tasks
func GetConfigSpec() base.ConfigSpec {
return base.ConfigSpec{
Fields: []*config.Field{
{
Name: "enabled",
JSONName: "enabled",
Type: config.FieldTypeBool,
DefaultValue: true,
Required: false,
DisplayName: "Enable Balance Tasks",
Description: "Whether balance tasks should be automatically created",
HelpText: "Toggle this to enable or disable automatic balance task generation",
InputType: "checkbox",
CSSClasses: "form-check-input",
},
{
Name: "scan_interval_seconds",
JSONName: "scan_interval_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 30 * 60,
MinValue: 5 * 60,
MaxValue: 2 * 60 * 60,
Required: true,
DisplayName: "Scan Interval",
Description: "How often to scan for volume distribution imbalances",
HelpText: "The system will check for volume distribution imbalances at this interval",
Placeholder: "30",
Unit: config.UnitMinutes,
InputType: "interval",
CSSClasses: "form-control",
},
{
Name: "max_concurrent",
JSONName: "max_concurrent",
Type: config.FieldTypeInt,
DefaultValue: 1,
MinValue: 1,
MaxValue: 3,
Required: true,
DisplayName: "Max Concurrent Tasks",
Description: "Maximum number of balance tasks that can run simultaneously",
HelpText: "Limits the number of balance operations running at the same time",
Placeholder: "1 (default)",
Unit: config.UnitCount,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "imbalance_threshold",
JSONName: "imbalance_threshold",
Type: config.FieldTypeFloat,
DefaultValue: 0.2,
MinValue: 0.05,
MaxValue: 0.5,
Required: true,
DisplayName: "Imbalance Threshold",
Description: "Minimum imbalance ratio to trigger balancing",
HelpText: "Volume distribution imbalances above this threshold will trigger balancing",
Placeholder: "0.20 (20%)",
Unit: config.UnitNone,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "io_byte_per_second",
JSONName: "io_byte_per_second",
Type: config.FieldTypeInt,
DefaultValue: 0,
MinValue: 0,
Required: false,
DisplayName: "Move IO Limit (bytes/sec)",
Description: "Limit each volume move's copy rate",
HelpText: "0 falls back to each volume server's own maintenance rate (-maintenanceBytePerSecond)",
Placeholder: "0 (server maintenance rate)",
Unit: config.UnitNone,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "min_server_count",
JSONName: "min_server_count",
Type: config.FieldTypeInt,
DefaultValue: 2,
MinValue: 2,
MaxValue: 10,
Required: true,
DisplayName: "Minimum Server Count",
Description: "Minimum number of servers required for balancing",
HelpText: "Balancing will only occur if there are at least this many servers",
Placeholder: "2 (default)",
Unit: config.UnitCount,
InputType: "number",
CSSClasses: "form-control",
},
},
}
}
// ToTaskPolicy converts configuration to a TaskPolicy protobuf message
func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
return &worker_pb.TaskPolicy{
Enabled: c.Enabled,
MaxConcurrent: int32(c.MaxConcurrent),
RepeatIntervalSeconds: int32(c.ScanIntervalSeconds),
CheckIntervalSeconds: int32(c.ScanIntervalSeconds),
TaskConfig: &worker_pb.TaskPolicy_BalanceConfig{
BalanceConfig: &worker_pb.BalanceTaskConfig{
ImbalanceThreshold: float64(c.ImbalanceThreshold),
MinServerCount: int32(c.MinServerCount),
IoBytePerSecond: c.IoBytePerSecond,
},
},
}
}
// FromTaskPolicy loads configuration from a TaskPolicy protobuf message
func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
if policy == nil {
return fmt.Errorf("policy is nil")
}
// Set general TaskPolicy fields
c.Enabled = policy.Enabled
c.MaxConcurrent = int(policy.MaxConcurrent)
c.ScanIntervalSeconds = int(policy.RepeatIntervalSeconds) // Direct seconds-to-seconds mapping
// Set balance-specific fields from the task config
if balanceConfig := policy.GetBalanceConfig(); balanceConfig != nil {
c.ImbalanceThreshold = float64(balanceConfig.ImbalanceThreshold)
c.MinServerCount = int(balanceConfig.MinServerCount)
c.IoBytePerSecond = balanceConfig.IoBytePerSecond
}
return nil
}
// LoadConfigFromPersistence loads configuration from the persistence layer if available
func LoadConfigFromPersistence(configPersistence interface{}) *Config {
config := NewDefaultConfig()
// Try to load from persistence if available
if persistence, ok := configPersistence.(interface {
LoadBalanceTaskPolicy() (*worker_pb.TaskPolicy, error)
}); ok {
policy, err := persistence.LoadBalanceTaskPolicy()
switch {
case err != nil:
glog.Warningf("Could not read the persisted balance configuration, falling back to defaults: %v", err)
case policy == nil:
glog.V(1).Infof("No balance configuration persisted yet, using defaults")
default:
if err := config.FromTaskPolicy(policy); err != nil {
glog.Warningf("Could not apply the persisted balance configuration, falling back to defaults: %v", err)
} else {
glog.V(1).Infof("Loaded balance configuration from persistence")
return config
}
}
} else if configPersistence != nil {
// A store was handed in but does not expose the accessor, so the persisted
// settings are silently ignored - always a wiring bug, never a normal state.
glog.Warningf("%T cannot provide the persisted balance configuration: it has no LoadBalanceTaskPolicy() method, "+
"so the compiled-in defaults are used and any saved balance settings are ignored", configPersistence)
}
glog.V(1).Infof("Using default balance configuration")
return config
}

View File

@@ -0,0 +1,839 @@
package balance
import (
"fmt"
"math"
"sort"
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/topology/balancer"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/util"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// Detection implements the detection logic for balance tasks.
// maxResults limits how many balance operations are returned per invocation.
// A non-positive maxResults means no explicit limit (uses a large default).
// The returned truncated flag is true when detection stopped because it hit
// maxResults rather than running out of work.
func Detection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo, config base.TaskConfig, maxResults int) ([]*types.TaskDetectionResult, bool, error) {
if !config.IsEnabled() {
return nil, false, nil
}
if clusterInfo == nil {
return nil, false, nil
}
balanceConfig := config.(*Config)
if maxResults <= 0 {
maxResults = math.MaxInt32
}
// Group volumes by disk type to ensure we compare apples to apples.
// Remote-tiered volumes are skipped: a "move" copies .idx/.vif then
// Volume.Destroy on the source deletes the remote object the destination
// now points at. Mirrors shell/command_volume_balance.go.
volumesByDiskType := make(map[string][]*types.VolumeHealthMetrics)
for _, metric := range metrics {
if metric.HasRemoteCopy {
continue
}
volumesByDiskType[metric.DiskType] = append(volumesByDiskType[metric.DiskType], metric)
}
// Sort disk types for deterministic iteration order when maxResults
// spans multiple disk types.
diskTypes := make([]string, 0, len(volumesByDiskType))
for dt := range volumesByDiskType {
diskTypes = append(diskTypes, dt)
}
sort.Strings(diskTypes)
var allParams []*types.TaskDetectionResult
truncated := false
for _, diskType := range diskTypes {
remaining := maxResults - len(allParams)
if remaining <= 0 {
truncated = true
break
}
tasks, diskTruncated := detectForDiskType(diskType, volumesByDiskType[diskType], balanceConfig, clusterInfo, remaining)
allParams = append(allParams, tasks...)
if diskTruncated {
truncated = true
}
}
return allParams, truncated, nil
}
// detectForDiskType performs balance detection for a specific disk type,
// returning up to maxResults balance tasks and whether it was truncated by the limit.
func detectForDiskType(diskType string, diskMetrics []*types.VolumeHealthMetrics, balanceConfig *Config, clusterInfo *types.ClusterInfo, maxResults int) ([]*types.TaskDetectionResult, bool) {
// Skip if cluster segment is too small
minVolumeCount := 2 // More reasonable for small clusters
if len(diskMetrics) < minVolumeCount {
// Only log at verbose level to avoid spamming for small/empty disk types
glog.V(1).Infof("BALANCE [%s]: No tasks created - cluster too small (%d volumes, need ≥%d)", diskType, len(diskMetrics), minVolumeCount)
return nil, false
}
// Analyze volume distribution across servers.
// Seed from ActiveTopology so servers with matching disk type but zero
// volumes are included in the count and imbalance calculation.
// Also collect MaxVolumeCount per server to compute utilization ratios.
serverVolumeCounts := make(map[string]int)
serverMaxVolumes := make(map[string]int64)
serverDiskTotalBytes := make(map[string]uint64)
serverDiskFreeBytes := make(map[string]uint64)
if clusterInfo.ActiveTopology != nil {
topologyInfo := clusterInfo.ActiveTopology.GetTopologyInfo()
if topologyInfo != nil {
dcMatchers := wildcard.CompileWildcardMatchers(balanceConfig.DataCenterFilter)
rackMatchers := wildcard.CompileWildcardMatchers(balanceConfig.RackFilter)
nodeMatchers := wildcard.CompileWildcardMatchers(balanceConfig.NodeFilter)
for _, dc := range topologyInfo.DataCenterInfos {
if !wildcard.MatchesAnyWildcard(dcMatchers, dc.Id) {
continue
}
for _, rack := range dc.RackInfos {
if !wildcard.MatchesAnyWildcard(rackMatchers, rack.Id) {
continue
}
for _, node := range rack.DataNodeInfos {
if !wildcard.MatchesAnyWildcard(nodeMatchers, node.Id) {
continue
}
for diskTypeName, diskInfo := range node.DiskInfos {
if diskTypeName == diskType {
serverVolumeCounts[node.Id] = 0
serverMaxVolumes[node.Id] += diskInfo.MaxVolumeCount
serverDiskTotalBytes[node.Id] += diskInfo.DiskTotalBytes
serverDiskFreeBytes[node.Id] += diskInfo.DiskFreeBytes
}
}
}
}
}
}
}
hasLocationFilter := balanceConfig.DataCenterFilter != "" || balanceConfig.RackFilter != "" || balanceConfig.NodeFilter != ""
for _, metric := range diskMetrics {
if hasLocationFilter {
// Only count metrics for servers that passed filtering.
// Without this guard, out-of-scope servers are re-introduced.
if _, allowed := serverVolumeCounts[metric.Server]; !allowed {
continue
}
}
serverVolumeCounts[metric.Server]++
}
if len(serverVolumeCounts) < balanceConfig.MinServerCount {
glog.V(1).Infof("BALANCE [%s]: No tasks created - too few servers (%d servers, need ≥%d)", diskType, len(serverVolumeCounts), balanceConfig.MinServerCount)
return nil, false
}
// Seed adjustments from existing pending/assigned balance tasks so that
// effectiveCounts reflects in-flight moves and prevents over-scheduling.
var adjustments map[string]int
if clusterInfo.ActiveTopology != nil {
adjustments = clusterInfo.ActiveTopology.GetTaskServerAdjustments(topology.TaskTypeBalance)
}
if adjustments == nil {
adjustments = make(map[string]int)
}
// Servers where we can no longer find eligible volumes or plan destinations
exhaustedServers := make(map[string]bool)
// Sort servers for deterministic iteration and tie-breaking
sortedServers := make([]string, 0, len(serverVolumeCounts))
for server := range serverVolumeCounts {
sortedServers = append(sortedServers, server)
}
sort.Strings(sortedServers)
// Pre-index volumes by server with cursors to avoid O(maxResults * volumes) scanning.
// Sort each server's volumes by VolumeID for deterministic selection.
volumesByServer := make(map[string][]*types.VolumeHealthMetrics, len(serverVolumeCounts))
for _, metric := range diskMetrics {
volumesByServer[metric.Server] = append(volumesByServer[metric.Server], metric)
}
for _, vols := range volumesByServer {
sort.Slice(vols, func(i, j int) bool {
return vols[i].VolumeID < vols[j].VolumeID
})
}
serverCursors := make(map[string]int, len(serverVolumeCounts))
var results []*types.TaskDetectionResult
balanced := false
// Decide upfront whether all servers have MaxVolumeCount info.
// If any server is missing it, fall back to raw counts for ALL servers
// to avoid mixing utilization ratios (0.0–1.0) with raw counts.
allServersHaveMaxInfo := true
for _, server := range sortedServers {
if maxVol, ok := serverMaxVolumes[server]; !ok || maxVol <= 0 {
allServersHaveMaxInfo = false
glog.V(1).Infof("BALANCE [%s]: Server %s is missing MaxVolumeCount info, falling back to raw volume counts for balancing", diskType, server)
break
}
}
var serverUtilization func(server string, effectiveCount int) float64
if allServersHaveMaxInfo {
serverUtilization = func(server string, effectiveCount int) float64 {
return float64(effectiveCount) / float64(serverMaxVolumes[server])
}
} else {
serverUtilization = func(_ string, effectiveCount int) float64 {
return float64(effectiveCount)
}
}
// plannedBytes tracks data already routed to each destination earlier in this
// detection cycle, so the disk-fullness gate sees a target fill up as moves are
// planned instead of only its heartbeat-time free space (the shell equivalent
// is adjustAfterMove decrementing DiskFreeBytes).
plannedBytes := make(map[string]uint64)
// destinationFull excludes servers a projected move would push over, for the cycle.
destinationFull := make(map[string]bool)
// destinationDiskTooFull reports whether landing incomingBytes on server would put
// its disk at/over the mark, net of bytes already planned this cycle; unreported
// disks are never gated.
destinationDiskTooFull := func(server string, incomingBytes uint64) bool {
free := serverDiskFreeBytes[server]
if planned := plannedBytes[server]; planned < free {
free -= planned
} else {
free = 0
}
return balancer.DiskTooFullAfter(serverDiskTotalBytes[server], free, incomingBytes, balancer.DefaultMaxDiskUsagePercent)
}
for len(results) < maxResults {
// Compute effective volume counts with adjustments from planned moves
effectiveCounts := make(map[string]int, len(serverVolumeCounts))
for server, count := range serverVolumeCounts {
effective := count + adjustments[server]
if effective < 0 {
effective = 0
}
effectiveCounts[server] = effective
}
// Find the most and least utilized servers using utilization ratio
// (volumes / maxVolumes) so that servers with higher capacity are
// expected to hold proportionally more volumes.
maxUtilization := -1.0
minUtilization := math.Inf(1)
maxServer := ""
minServer := ""
for _, server := range sortedServers {
count := effectiveCounts[server]
util := serverUtilization(server, count)
// Min is the emptiest server that can actually receive a volume, so a
// physically full server (whose over-set maxVolumeCount makes its slot
// utilization look low) is never chosen as the destination.
if !destinationFull[server] && !destinationDiskTooFull(server, 0) && util < minUtilization {
minUtilization = util
minServer = server
}
// Max is only among non-exhausted servers since we can only move from them
if exhaustedServers[server] {
continue
}
if util > maxUtilization {
maxUtilization = util
maxServer = server
}
}
if maxServer == "" {
// All servers exhausted
glog.V(1).Infof("BALANCE [%s]: All overloaded servers exhausted after %d task(s)", diskType, len(results))
break
}
if minServer == "" {
// Every candidate destination is at/above the physical disk high-water
// mark, so no move can safely improve balance.
glog.V(1).Infof("BALANCE [%s]: No eligible destination - all candidates at/above %d%% disk usage after %d task(s)", diskType, balancer.DefaultMaxDiskUsagePercent, len(results))
break
}
// Check if utilization imbalance exceeds threshold.
// imbalanceRatio is the difference between the most and least utilized
// servers, expressed as a fraction of mean utilization.
avgUtilization := (maxUtilization + minUtilization) / 2.0
var imbalanceRatio float64
if avgUtilization > 0 {
imbalanceRatio = (maxUtilization - minUtilization) / avgUtilization
}
if imbalanceRatio <= balanceConfig.ImbalanceThreshold {
if len(results) == 0 {
glog.Infof("BALANCE [%s]: No tasks created - cluster well balanced. Imbalance=%.1f%% (threshold=%.1f%%). MaxUtil=%.1f%% on %s, MinUtil=%.1f%% on %s",
diskType, imbalanceRatio*100, balanceConfig.ImbalanceThreshold*100, maxUtilization*100, maxServer, minUtilization*100, minServer)
} else {
glog.Infof("BALANCE [%s]: Created %d task(s), cluster now balanced. Imbalance=%.1f%% (threshold=%.1f%%)",
diskType, len(results), imbalanceRatio*100, balanceConfig.ImbalanceThreshold*100)
}
balanced = true
break
}
// When the global max and min effective counts differ by at most 1,
// no single move can improve balance — it would just swap which server
// is min vs max. Stop here to avoid infinite oscillation when the
// threshold is unachievable (e.g., 11 vols across 4 servers: best is
// 3/3/3/2, imbalance=36%). We scan ALL servers' effective counts so the
// check works regardless of whether utilization or raw counts are used.
globalMaxCount, globalMinCount := 0, math.MaxInt
for _, c := range effectiveCounts {
if c > globalMaxCount {
globalMaxCount = c
}
if c < globalMinCount {
globalMinCount = c
}
}
if globalMaxCount-globalMinCount <= 1 {
if len(results) == 0 {
glog.Infof("BALANCE [%s]: No tasks created - cluster as balanced as possible. Imbalance=%.1f%% (threshold=%.1f%%), but max-min diff is %d",
diskType, imbalanceRatio*100, balanceConfig.ImbalanceThreshold*100, globalMaxCount-globalMinCount)
} else {
glog.Infof("BALANCE [%s]: Created %d task(s), cluster as balanced as possible. Imbalance=%.1f%% (threshold=%.1f%%), max-min diff=%d",
diskType, len(results), imbalanceRatio*100, balanceConfig.ImbalanceThreshold*100, globalMaxCount-globalMinCount)
}
balanced = true
break
}
// Per-move convergence guard: mirrors weed/shell/command_volume_balance.go
// to prevent oscillation and destination overshoot when MaxVolumeCount
// values are heterogeneous. Without this guard, the greedy max→min
// algorithm can schedule moves that flip which server is most-utilized
// (source becomes min, destination becomes max), producing A→B, B→A
// oscillation within a single detection cycle.
//
// Check: after the move, the destination's utilization must not strictly
// exceed the source's utilization. If it would, no single move can
// improve balance — stop here. This also handles heterogeneous capacity
// correctly by comparing post-move utilization ratios rather than raw
// counts. The integer discretization is handled automatically: when
// counts cannot match the ideal exactly, the check still admits moves
// that reduce the max/min gap without flipping it.
maxCap := serverMaxVolumes[maxServer]
minCap := serverMaxVolumes[minServer]
if allServersHaveMaxInfo && maxCap > 0 && minCap > 0 {
newSrcUtil := float64(effectiveCounts[maxServer]-1) / float64(maxCap)
newDstUtil := float64(effectiveCounts[minServer]+1) / float64(minCap)
if newDstUtil > newSrcUtil {
if len(results) == 0 {
glog.Infof("BALANCE [%s]: No tasks created - no beneficial move available. After move, dest %s util would be %.1f%% vs source %s util %.1f%%",
diskType, minServer, newDstUtil*100, maxServer, newSrcUtil*100)
} else {
glog.Infof("BALANCE [%s]: Created %d task(s), no more beneficial moves available. After move, dest %s util would be %.1f%% vs source %s util %.1f%%",
diskType, len(results), minServer, newDstUtil*100, maxServer, newSrcUtil*100)
}
balanced = true
break
}
}
// Select a volume from the overloaded server using per-server cursor
var selectedVolume *types.VolumeHealthMetrics
serverVols := volumesByServer[maxServer]
cursor := serverCursors[maxServer]
for cursor < len(serverVols) {
metric := serverVols[cursor]
cursor++
// Skip volumes that already have a task in ActiveTopology
if clusterInfo.ActiveTopology != nil && clusterInfo.ActiveTopology.HasAnyTask(metric.VolumeID) {
continue
}
selectedVolume = metric
break
}
serverCursors[maxServer] = cursor
if selectedVolume == nil {
glog.V(1).Infof("BALANCE [%s]: No more eligible volumes on overloaded server %s, trying other servers", diskType, maxServer)
exhaustedServers[maxServer] = true
continue
}
// Skip a destination this specific volume would push over the mark, and
// re-pick, rather than overshoot.
if destinationDiskTooFull(minServer, uint64(selectedVolume.Size)) {
glog.V(1).Infof("BALANCE [%s]: skip destination %s: volume %d (%d bytes) would cross %d%% disk usage",
diskType, minServer, selectedVolume.VolumeID, selectedVolume.Size, balancer.DefaultMaxDiskUsagePercent)
destinationFull[minServer] = true
serverCursors[maxServer]-- // retry this volume against another destination
continue
}
// Create task targeting minServer — the greedy algorithm's natural choice.
// Using minServer instead of letting planBalanceDestination independently
// pick a destination ensures that the detection loop's effective counts
// and the destination selection stay in sync. Without this, the topology's
// LoadCount-based scoring can diverge from the adjustment-based effective
// counts, causing moves to pile onto one server or oscillate (A→B, B→A).
//
// Constrain the destination (including the score-based fallback inside
// createBalanceTask) to servers whose physical disk is not near full;
// sources are unaffected, so a full server can still be drained.
eligibleTargets := make(map[string]int, len(serverVolumeCounts))
for s, c := range serverVolumeCounts {
if !destinationFull[s] && !destinationDiskTooFull(s, 0) {
eligibleTargets[s] = c
}
}
task, destServerID := createBalanceTask(diskType, selectedVolume, clusterInfo, minServer, eligibleTargets, balanceConfig.IoBytePerSecond)
if task == nil {
glog.V(1).Infof("BALANCE [%s]: Cannot plan task for volume %d on server %s, trying next volume", diskType, selectedVolume.VolumeID, maxServer)
continue
}
results = append(results, task)
// Adjust effective counts for the next iteration.
adjustments[maxServer]--
if destServerID != "" {
adjustments[destServerID]++
// Charge the moved volume's bytes to the destination so the disk-fullness
// gate sees it fill up over the course of this cycle.
plannedBytes[destServerID] += uint64(selectedVolume.Size)
// If the destination server wasn't in serverVolumeCounts (e.g., a
// server with 0 volumes not seeded from topology), add it so
// subsequent iterations include it in effective/average/min/max.
if _, exists := serverVolumeCounts[destServerID]; !exists {
serverVolumeCounts[destServerID] = 0
sortedServers = append(sortedServers, destServerID)
sort.Strings(sortedServers)
}
}
}
// Truncated only if we hit maxResults and detection didn't naturally finish
truncated := len(results) >= maxResults && !balanced
return results, truncated
}
// createBalanceTask creates a single balance task for the selected volume.
// targetServer is the server ID chosen by the detection loop's greedy algorithm.
// Returns (nil, "") if destination planning fails.
// On success, returns the task result and the canonical destination server ID.
// allowedServers is the set of servers that passed DC/rack/node filtering in
// the detection loop. When non-empty, the fallback destination planner is
// checked against this set so that filter scope cannot leak.
func createBalanceTask(diskType string, selectedVolume *types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo, targetServer string, allowedServers map[string]int, ioBytePerSecond int64) (*types.TaskDetectionResult, string) {
taskID := fmt.Sprintf("balance_vol_%d_%d", selectedVolume.VolumeID, time.Now().UnixNano())
task := &types.TaskDetectionResult{
TaskID: taskID,
TaskType: types.TaskTypeBalance,
VolumeID: selectedVolume.VolumeID,
Server: selectedVolume.Server,
Collection: selectedVolume.Collection,
Priority: types.TaskPriorityNormal,
Reason: fmt.Sprintf("Cluster imbalance detected for %s disk type",
diskType),
ScheduleAt: time.Now(),
}
// Plan destination if ActiveTopology is available
if clusterInfo.ActiveTopology == nil {
glog.Warningf("No ActiveTopology available for destination planning in balance detection")
return nil, ""
}
// Parse replica placement once for use in both scoring and validation.
var volumeRP *super_block.ReplicaPlacement
if selectedVolume.ExpectedReplicas > 0 && selectedVolume.ExpectedReplicas <= 255 {
if parsed, rpErr := super_block.NewReplicaPlacementFromByte(byte(selectedVolume.ExpectedReplicas)); rpErr == nil && parsed.HasReplication() {
volumeRP = parsed
}
}
var replicas []types.ReplicaLocation
if clusterInfo.VolumeReplicaMap != nil {
replicas = clusterInfo.VolumeReplicaMap[selectedVolume.VolumeID]
if volumeRP != nil && len(replicas) == 0 {
glog.V(1).Infof("BALANCE [%s]: No replica locations found for volume %d, skipping placement validation",
diskType, selectedVolume.VolumeID)
}
}
// Resolve the target server chosen by the detection loop's effective counts.
// This keeps destination selection in sync with the greedy algorithm rather
// than relying on topology LoadCount which can diverge across iterations.
destinationPlan, err := resolveBalanceDestination(clusterInfo.ActiveTopology, selectedVolume, targetServer)
if err != nil {
// Fall back to score-based planning if the preferred target can't be resolved
glog.V(1).Infof("BALANCE [%s]: Cannot resolve target %s for volume %d, falling back to score-based planning: %v",
diskType, targetServer, selectedVolume.VolumeID, err)
destinationPlan, err = planBalanceDestination(clusterInfo.ActiveTopology, selectedVolume, volumeRP, replicas, allowedServers)
if err != nil {
glog.Warningf("Failed to plan balance destination for volume %d: %v", selectedVolume.VolumeID, err)
return nil, ""
}
}
// Verify the resolved destination. If it falls outside the filtered scope
// or violates replica placement, fall back to the score-based planner and
// pick the best candidate that is actually valid.
if !isValidBalanceDestination(destinationPlan, allowedServers, volumeRP, replicas, selectedVolume.Server) {
switch {
case destinationPlan == nil:
glog.V(1).Infof("BALANCE [%s]: Planned destination for volume %d is nil, falling back",
diskType, selectedVolume.VolumeID)
case !isAllowedBalanceTarget(destinationPlan.TargetNode, allowedServers):
glog.V(1).Infof("BALANCE [%s]: Planned destination %s for volume %d is outside filtered scope, falling back",
diskType, destinationPlan.TargetNode, selectedVolume.VolumeID)
case volumeRP != nil && len(replicas) > 0:
glog.V(1).Infof("BALANCE [%s]: Destination %s violates replica placement for volume %d (rp=%03d), falling back",
diskType, destinationPlan.TargetNode, selectedVolume.VolumeID, selectedVolume.ExpectedReplicas)
}
destinationPlan, err = planBalanceDestination(clusterInfo.ActiveTopology, selectedVolume, volumeRP, replicas, allowedServers)
if err != nil {
glog.Warningf("BALANCE [%s]: Failed to plan fallback destination for volume %d: %v", diskType, selectedVolume.VolumeID, err)
return nil, ""
}
if !isValidBalanceDestination(destinationPlan, allowedServers, volumeRP, replicas, selectedVolume.Server) {
if destinationPlan == nil {
glog.V(1).Infof("BALANCE [%s]: Fallback destination for volume %d is nil",
diskType, selectedVolume.VolumeID)
} else {
glog.V(1).Infof("BALANCE [%s]: Fallback destination %s is not valid for volume %d",
diskType, destinationPlan.TargetNode, selectedVolume.VolumeID)
}
return nil, ""
}
}
// Find the actual disk containing the volume on the source server
sourceDisk, found := base.FindVolumeDisk(clusterInfo.ActiveTopology, selectedVolume.VolumeID, selectedVolume.Collection, selectedVolume.Server)
if !found {
glog.Warningf("BALANCE [%s]: Could not find volume %d (collection: %s) on source server %s - unable to create balance task",
diskType, selectedVolume.VolumeID, selectedVolume.Collection, selectedVolume.Server)
return nil, ""
}
// Update reason with full details now that we have destination info
task.Reason = fmt.Sprintf("Cluster imbalance detected for %s: move volume %d from %s to %s",
diskType, selectedVolume.VolumeID, selectedVolume.Server, destinationPlan.TargetNode)
// Create typed parameters with unified source and target information
task.TypedParams = &worker_pb.TaskParams{
TaskId: taskID,
VolumeId: selectedVolume.VolumeID,
Collection: selectedVolume.Collection,
VolumeSize: selectedVolume.Size,
Sources: []*worker_pb.TaskSource{
{
Node: selectedVolume.ServerAddress,
DiskId: sourceDisk,
VolumeId: selectedVolume.VolumeID,
EstimatedSize: selectedVolume.Size,
DataCenter: selectedVolume.DataCenter,
Rack: selectedVolume.Rack,
},
},
Targets: []*worker_pb.TaskTarget{
{
Node: destinationPlan.TargetAddress,
DiskId: destinationPlan.TargetDisk,
VolumeId: selectedVolume.VolumeID,
EstimatedSize: destinationPlan.ExpectedSize,
DataCenter: destinationPlan.TargetDC,
Rack: destinationPlan.TargetRack,
},
},
TaskParams: &worker_pb.TaskParams_BalanceParams{
BalanceParams: &worker_pb.BalanceTaskParams{
ForceMove: false,
TimeoutSeconds: 600, // 10 minutes default
IoBytePerSecond: ioBytePerSecond,
},
},
}
glog.V(1).Infof("Planned balance destination for volume %d: %s -> %s",
selectedVolume.VolumeID, selectedVolume.Server, destinationPlan.TargetNode)
// Add pending balance task to ActiveTopology for capacity management
targetDisk := destinationPlan.TargetDisk
err = clusterInfo.ActiveTopology.AddPendingTask(topology.TaskSpec{
TaskID: taskID,
TaskType: topology.TaskTypeBalance,
VolumeID: selectedVolume.VolumeID,
VolumeSize: int64(selectedVolume.Size),
Sources: []topology.TaskSourceSpec{
{ServerID: selectedVolume.Server, DiskID: sourceDisk},
},
Destinations: []topology.TaskDestinationSpec{
{ServerID: destinationPlan.TargetNode, DiskID: targetDisk},
},
})
if err != nil {
glog.Warningf("BALANCE [%s]: Failed to add pending task for volume %d: %v", diskType, selectedVolume.VolumeID, err)
return nil, ""
}
glog.V(2).Infof("Added pending balance task %s to ActiveTopology for volume %d: %s:%d -> %s:%d",
taskID, selectedVolume.VolumeID, selectedVolume.Server, sourceDisk, destinationPlan.TargetNode, targetDisk)
return task, destinationPlan.TargetNode
}
// resolveBalanceDestination resolves the destination for a balance operation
// when the target server is already known (chosen by the detection loop's
// effective volume counts). It finds a disk on the target server that can
// accept another volume after accounting for ALL pending and assigned tasks
// on that disk (via ActiveTopology.GetEffectiveAvailableCapacity) — not the
// static VolumeCount from the topology snapshot. This keeps destination
// planning consistent with the loop's effective-count bookkeeping and
// prevents over-scheduling when multiple moves are planned within the same
// detection cycle.
func resolveBalanceDestination(activeTopology *topology.ActiveTopology, selectedVolume *types.VolumeHealthMetrics, targetServer string) (*topology.DestinationPlan, error) {
nodeDisks := activeTopology.GetNodeDisks(targetServer)
if len(nodeDisks) == 0 {
return nil, fmt.Errorf("target server %s not found in topology", targetServer)
}
var eligibleDisk *topology.DiskInfo
for _, disk := range nodeDisks {
if disk == nil || disk.DiskInfo == nil {
continue
}
if disk.DiskType != selectedVolume.DiskType {
continue
}
// Use effective capacity so that prior moves planned in this same
// detection cycle (already registered as pending tasks) reduce the
// available slots. A disk with VolumeCount << MaxVolumeCount in the
// topology snapshot can still be effectively full if many in-flight
// tasks already target it.
if disk.DiskInfo.MaxVolumeCount > 0 {
available := activeTopology.GetEffectiveAvailableCapacity(disk.NodeID, disk.DiskID)
if available <= 0 {
continue
}
}
eligibleDisk = disk
break
}
if eligibleDisk == nil {
return nil, fmt.Errorf("target server %s has no available disk of type %s", targetServer, selectedVolume.DiskType)
}
targetAddress, err := util.ResolveServerAddress(eligibleDisk.NodeID, activeTopology)
if err != nil {
return nil, fmt.Errorf("failed to resolve address for target server %s: %v", eligibleDisk.NodeID, err)
}
return &topology.DestinationPlan{
TargetNode: eligibleDisk.NodeID,
TargetAddress: targetAddress,
TargetDisk: eligibleDisk.DiskID,
TargetRack: eligibleDisk.Rack,
TargetDC: eligibleDisk.DataCenter,
ExpectedSize: selectedVolume.Size,
}, nil
}
// planBalanceDestination plans the destination for a balance operation using
// score-based selection. Used as a fallback when the preferred target cannot
// be resolved, and for single-move scenarios outside the detection loop.
// rp may be nil when the volume has no replication constraint. When replica
// locations are known, candidates that would violate placement are filtered
// out before scoring.
func planBalanceDestination(activeTopology *topology.ActiveTopology, selectedVolume *types.VolumeHealthMetrics, rp *super_block.ReplicaPlacement, replicas []types.ReplicaLocation, allowedServers map[string]int) (*topology.DestinationPlan, error) {
// Get source node information from topology
var sourceRack, sourceDC string
// Extract rack and DC from topology info
topologyInfo := activeTopology.GetTopologyInfo()
if topologyInfo != nil {
for _, dc := range topologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, dataNodeInfo := range rack.DataNodeInfos {
if dataNodeInfo.Id == selectedVolume.Server {
sourceDC = dc.Id
sourceRack = rack.Id
break
}
}
if sourceRack != "" {
break
}
}
if sourceDC != "" {
break
}
}
}
// Get available disks, excluding the source node
availableDisks := activeTopology.GetAvailableDisks(topology.TaskTypeBalance, selectedVolume.Server)
if len(availableDisks) == 0 {
return nil, fmt.Errorf("no available disks for balance operation")
}
// Sort available disks by NodeID then DiskID for deterministic tie-breaking
sort.Slice(availableDisks, func(i, j int) bool {
if availableDisks[i].NodeID != availableDisks[j].NodeID {
return availableDisks[i].NodeID < availableDisks[j].NodeID
}
return availableDisks[i].DiskID < availableDisks[j].DiskID
})
// Find the best destination disk based on balance criteria
var bestDisk *topology.DiskInfo
bestScore := math.Inf(-1)
for _, disk := range availableDisks {
// Ensure disk type matches
if disk.DiskType != selectedVolume.DiskType {
continue
}
if !isAllowedBalanceTarget(disk.NodeID, allowedServers) {
continue
}
if rp != nil && len(replicas) > 0 {
target := types.ReplicaLocation{
DataCenter: disk.DataCenter,
Rack: disk.Rack,
NodeID: disk.NodeID,
Host: hostFromAddress(disk.Address, disk.NodeID),
}
if !IsGoodMove(rp, replicas, selectedVolume.Server, target) {
continue
}
}
score := calculateBalanceScore(disk, sourceRack, sourceDC, selectedVolume.Size, rp)
if score > bestScore {
bestScore = score
bestDisk = disk
}
}
if bestDisk == nil {
return nil, fmt.Errorf("no suitable destination found for balance operation")
}
// Get the target server address
targetAddress, err := util.ResolveServerAddress(bestDisk.NodeID, activeTopology)
if err != nil {
return nil, fmt.Errorf("failed to resolve address for target server %s: %v", bestDisk.NodeID, err)
}
return &topology.DestinationPlan{
TargetNode: bestDisk.NodeID,
TargetAddress: targetAddress,
TargetDisk: bestDisk.DiskID,
TargetRack: bestDisk.Rack,
TargetDC: bestDisk.DataCenter,
ExpectedSize: selectedVolume.Size,
PlacementScore: bestScore,
}, nil
}
// calculateBalanceScore calculates placement score for balance operations.
// LoadCount reflects pending+assigned tasks on the disk, so we factor it into
// the utilization estimate to avoid stacking multiple moves onto the same target.
// rp may be nil when the volume has no replication constraint; in that case the
// scorer defaults to preferring cross-rack/DC distribution.
func calculateBalanceScore(disk *topology.DiskInfo, sourceRack, sourceDC string, volumeSize uint64, rp *super_block.ReplicaPlacement) float64 {
if disk.DiskInfo == nil {
return 0.0
}
score := 0.0
// Prefer disks with lower effective volume count (current + pending moves).
// LoadCount is included so that disks already targeted by planned moves
// appear more utilized, naturally spreading work across targets.
if disk.DiskInfo.MaxVolumeCount > 0 {
effectiveVolumeCount := float64(disk.DiskInfo.VolumeCount) + float64(disk.LoadCount)
utilization := effectiveVolumeCount / float64(disk.DiskInfo.MaxVolumeCount)
score += (1.0 - utilization) * 50.0 // Up to 50 points for low utilization
}
// Rack scoring: respect the replication policy.
// If replicas must stay on the same rack (SameRackCount > 0 with no
// cross-rack requirement), prefer same-rack destinations. Otherwise
// prefer different racks for better distribution.
sameRack := disk.Rack == sourceRack
if rp != nil && rp.DiffRackCount == 0 && rp.SameRackCount > 0 {
if sameRack {
score += 30.0
}
} else {
if !sameRack {
score += 30.0
}
}
// DC scoring: same idea. If the policy requires all copies in one DC,
// prefer same-DC destinations. Otherwise prefer different DCs.
sameDC := disk.DataCenter == sourceDC
if rp != nil && rp.DiffDataCenterCount == 0 && (rp.SameRackCount > 0 || rp.DiffRackCount > 0) {
if sameDC {
score += 20.0
}
} else {
if !sameDC {
score += 20.0
}
}
return score
}
func isAllowedBalanceTarget(nodeID string, allowedServers map[string]int) bool {
if len(allowedServers) == 0 {
return true
}
_, ok := allowedServers[nodeID]
return ok
}
func isValidBalanceDestination(plan *topology.DestinationPlan, allowedServers map[string]int, rp *super_block.ReplicaPlacement, replicas []types.ReplicaLocation, sourceNodeID string) bool {
if plan == nil {
return false
}
if !isAllowedBalanceTarget(plan.TargetNode, allowedServers) {
return false
}
if rp == nil || len(replicas) == 0 {
return true
}
target := types.ReplicaLocation{
DataCenter: plan.TargetDC,
Rack: plan.TargetRack,
NodeID: plan.TargetNode,
Host: hostFromAddress(plan.TargetAddress, plan.TargetNode),
}
return IsGoodMove(rp, replicas, sourceNodeID, target)
}
// parseCSVSet splits a comma-separated string into a set of trimmed, non-empty values.

View File

@@ -0,0 +1,513 @@
package balance
import (
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
const (
fullnessThreshold = 1.01
stateActive = "ACTIVE"
stateFull = "FULL"
)
// filterByState filters metrics by volume state for testing.
func filterByState(metrics []*types.VolumeHealthMetrics, state string) []*types.VolumeHealthMetrics {
if state != stateActive && state != stateFull {
return metrics
}
var out []*types.VolumeHealthMetrics
for _, m := range metrics {
if m == nil {
continue
}
if state == stateActive && m.FullnessRatio < fullnessThreshold {
out = append(out, m)
}
if state == stateFull && m.FullnessRatio >= fullnessThreshold {
out = append(out, m)
}
}
return out
}
// Integration tests that exercise multiple features together:
// DC/rack/node filters, volume state filtering, replica placement validation,
// and collection scoping all interacting within a single detection run.
// makeVolumesWithOptions generates metrics with additional options.
type volumeOption func(m *types.VolumeHealthMetrics)
func withFullness(ratio float64) volumeOption {
return func(m *types.VolumeHealthMetrics) { m.FullnessRatio = ratio }
}
func withReplicas(rp int) volumeOption {
return func(m *types.VolumeHealthMetrics) { m.ExpectedReplicas = rp }
}
func makeVolumesWith(server, diskType, dc, rack, collection string, volumeIDBase uint32, n int, opts ...volumeOption) []*types.VolumeHealthMetrics {
vols := makeVolumes(server, diskType, dc, rack, collection, volumeIDBase, n)
for _, v := range vols {
for _, opt := range opts {
opt(v)
}
}
return vols
}
// buildReplicaMap builds a VolumeReplicaMap from metrics (each metric is one replica location).
func buildReplicaMap(metrics []*types.VolumeHealthMetrics) map[uint32][]types.ReplicaLocation {
m := make(map[uint32][]types.ReplicaLocation)
for _, metric := range metrics {
m[metric.VolumeID] = append(m[metric.VolumeID], types.ReplicaLocation{
DataCenter: metric.DataCenter,
Rack: metric.Rack,
NodeID: metric.Server,
})
}
return m
}
// TestIntegration_DCFilterWithVolumeState tests that DC filtering and volume
// state filtering compose correctly: only ACTIVE volumes in the specified DC
// participate in balancing.
func TestIntegration_DCFilterWithVolumeState(t *testing.T) {
servers := []serverSpec{
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack1"},
{id: "node-c", diskType: "hdd", diskID: 3, dc: "dc2", rack: "rack1"},
}
var allMetrics []*types.VolumeHealthMetrics
// dc1: node-a has 40 active volumes, node-b has 10 active volumes
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 1, 40, withFullness(0.5))...)
allMetrics = append(allMetrics, makeVolumesWith("node-b", "hdd", "dc1", "rack1", "c1", 100, 10, withFullness(0.5))...)
// dc1: node-a also has 20 FULL volumes that should be excluded
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 200, 20, withFullness(1.5))...)
// dc2: node-c has 50 active volumes (should be excluded by DC filter)
allMetrics = append(allMetrics, makeVolumesWith("node-c", "hdd", "dc2", "rack1", "c1", 300, 50, withFullness(0.5))...)
// Apply volume state filter (ACTIVE only) first
activeMetrics := filterByState(allMetrics, stateActive)
// Then apply DC filter
dcMetrics := make([]*types.VolumeHealthMetrics, 0)
for _, m := range activeMetrics {
if m.DataCenter == "dc1" {
dcMetrics = append(dcMetrics, m)
}
}
at := buildTopology(servers, allMetrics)
clusterInfo := &types.ClusterInfo{ActiveTopology: at}
conf := defaultConf()
conf.DataCenterFilter = "dc1"
tasks, _, err := Detection(dcMetrics, clusterInfo, conf, 100)
if err != nil {
t.Fatalf("Detection failed: %v", err)
}
if len(tasks) == 0 {
t.Fatal("Expected balance tasks for 40/10 active-only imbalance in dc1, got 0")
}
for _, task := range tasks {
if task.Server == "node-c" {
t.Error("node-c (dc2) should not be a source")
}
if task.TypedParams != nil {
for _, tgt := range task.TypedParams.Targets {
if strings.Contains(tgt.Node, "node-c") {
t.Error("node-c (dc2) should not be a target")
}
}
}
}
// Verify convergence uses only the 50 active dc1 volumes (40+10),
// not the 20 full volumes
effective := computeEffectiveCounts(servers[:2], dcMetrics, tasks)
total := 0
maxC, minC := 0, len(dcMetrics)
for _, c := range effective {
total += c
if c > maxC {
maxC = c
}
if c < minC {
minC = c
}
}
if total != 50 {
t.Errorf("Expected 50 total active volumes in dc1, got %d", total)
}
t.Logf("DC+state filter: %d tasks, effective=%v", len(tasks), effective)
}
// TestIntegration_NodeFilterWithCollections tests that node filtering works
// correctly when volumes span multiple collections.
func TestIntegration_NodeFilterWithCollections(t *testing.T) {
servers := []serverSpec{
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack1"},
{id: "node-c", diskType: "hdd", diskID: 3, dc: "dc1", rack: "rack1"},
}
var allMetrics []*types.VolumeHealthMetrics
// node-a: 30 "photos" + 20 "videos" = 50 total
allMetrics = append(allMetrics, makeVolumes("node-a", "hdd", "dc1", "rack1", "photos", 1, 30)...)
allMetrics = append(allMetrics, makeVolumes("node-a", "hdd", "dc1", "rack1", "videos", 100, 20)...)
// node-b: 5 "photos" + 5 "videos" = 10 total
allMetrics = append(allMetrics, makeVolumes("node-b", "hdd", "dc1", "rack1", "photos", 200, 5)...)
allMetrics = append(allMetrics, makeVolumes("node-b", "hdd", "dc1", "rack1", "videos", 300, 5)...)
// node-c: 40 volumes (should be excluded by node filter)
allMetrics = append(allMetrics, makeVolumes("node-c", "hdd", "dc1", "rack1", "photos", 400, 40)...)
// Apply node filter
filteredMetrics := make([]*types.VolumeHealthMetrics, 0)
for _, m := range allMetrics {
if m.Server == "node-a" || m.Server == "node-b" {
filteredMetrics = append(filteredMetrics, m)
}
}
at := buildTopology(servers, allMetrics)
clusterInfo := &types.ClusterInfo{ActiveTopology: at}
conf := defaultConf()
conf.NodeFilter = "node-a,node-b"
tasks, _, err := Detection(filteredMetrics, clusterInfo, conf, 100)
if err != nil {
t.Fatalf("Detection failed: %v", err)
}
if len(tasks) == 0 {
t.Fatal("Expected tasks for 50/10 imbalance within node-a,node-b")
}
// All moves should be between node-a and node-b only
for _, task := range tasks {
if task.Server != "node-a" && task.Server != "node-b" {
t.Errorf("Source %s should be node-a or node-b", task.Server)
}
if task.TypedParams != nil {
for _, tgt := range task.TypedParams.Targets {
if !strings.Contains(tgt.Node, "node-a") && !strings.Contains(tgt.Node, "node-b") {
t.Errorf("Target %s should be node-a or node-b", tgt.Node)
}
}
}
}
assertNoDuplicateVolumes(t, tasks)
t.Logf("Node filter with mixed collections: %d tasks", len(tasks))
}
// TestIntegration_ReplicaPlacementWithDCFilter tests that replica placement
// validation prevents moves that would violate replication policy even when
// DC filtering restricts the available servers.
func TestIntegration_ReplicaPlacementWithDCFilter(t *testing.T) {
// Setup: 2 DCs, volumes with rp=100 (1 replica in different DC)
// DC filter restricts to dc1 only.
// Replicas: each volume has one copy in dc1 and one in dc2.
// The balancer should NOT move volumes within dc1 to a node that would
// violate the cross-DC placement requirement.
servers := []serverSpec{
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack1"},
{id: "node-c", diskType: "hdd", diskID: 3, dc: "dc2", rack: "rack1"},
}
// node-a: 30 volumes with rp=100, node-b: 5 volumes with rp=100
// Each volume also has a replica on node-c (dc2)
var allMetrics []*types.VolumeHealthMetrics
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 1, 30, withReplicas(100))...)
allMetrics = append(allMetrics, makeVolumesWith("node-b", "hdd", "dc1", "rack1", "c1", 100, 5, withReplicas(100))...)
// dc2 replicas (not part of filtered metrics, but in replica map)
dc2Replicas := makeVolumesWith("node-c", "hdd", "dc2", "rack1", "c1", 1, 30, withReplicas(100))
// Build replica map: volumes 1-30 have replicas on node-a AND node-c
replicaMap := buildReplicaMap(allMetrics)
for _, r := range dc2Replicas {
replicaMap[r.VolumeID] = append(replicaMap[r.VolumeID], types.ReplicaLocation{
DataCenter: r.DataCenter,
Rack: r.Rack,
NodeID: r.Server,
})
}
// Filter to dc1 only
dc1Metrics := make([]*types.VolumeHealthMetrics, 0)
for _, m := range allMetrics {
if m.DataCenter == "dc1" {
dc1Metrics = append(dc1Metrics, m)
}
}
at := buildTopology(servers, allMetrics)
clusterInfo := &types.ClusterInfo{
ActiveTopology: at,
VolumeReplicaMap: replicaMap,
}
conf := defaultConf()
conf.DataCenterFilter = "dc1"
tasks, _, err := Detection(dc1Metrics, clusterInfo, conf, 100)
if err != nil {
t.Fatalf("Detection failed: %v", err)
}
// With rp=100, moving a volume from node-a to node-b is valid because
// the cross-DC replica on node-c is preserved. The balancer should
// produce moves that keep the dc2 replica intact.
if len(tasks) == 0 {
t.Fatal("Expected tasks: 30/5 imbalance with valid cross-DC replicas")
}
for _, task := range tasks {
// All sources should be from dc1
if task.Server != "node-a" && task.Server != "node-b" {
t.Errorf("Source %s should be in dc1", task.Server)
}
// All targets should be in dc1
if task.TypedParams != nil {
for _, tgt := range task.TypedParams.Targets {
if strings.Contains(tgt.Node, "node-c") {
t.Errorf("Target should not be node-c (dc2) with dc1 filter")
}
}
}
}
assertNoDuplicateVolumes(t, tasks)
t.Logf("Replica placement + DC filter: %d tasks", len(tasks))
}
// TestIntegration_RackFilterWithReplicaPlacement tests that rack filtering
// and replica placement validation (rp=010) work together. With rp=010,
// replicas must be on different racks. When rack filtering restricts to one
// rack, the balancer should still produce valid moves within that rack for
// volumes whose replicas satisfy the cross-rack requirement via nodes outside
// the filter.
func TestIntegration_RackFilterWithReplicaPlacement(t *testing.T) {
servers := []serverSpec{
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack1"},
{id: "node-c", diskType: "hdd", diskID: 3, dc: "dc1", rack: "rack2"},
}
// rack1: node-a has 30 volumes, node-b has 5
// Each volume also has a replica on node-c (rack2), satisfying rp=010
var allMetrics []*types.VolumeHealthMetrics
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 1, 30, withReplicas(10))...)
allMetrics = append(allMetrics, makeVolumesWith("node-b", "hdd", "dc1", "rack1", "c1", 100, 5, withReplicas(10))...)
rack2Replicas := makeVolumesWith("node-c", "hdd", "dc1", "rack2", "c1", 1, 30, withReplicas(10))
replicaMap := buildReplicaMap(allMetrics)
for _, r := range rack2Replicas {
replicaMap[r.VolumeID] = append(replicaMap[r.VolumeID], types.ReplicaLocation{
DataCenter: r.DataCenter,
Rack: r.Rack,
NodeID: r.Server,
})
}
// Filter to rack1 only
rack1Metrics := make([]*types.VolumeHealthMetrics, 0)
for _, m := range allMetrics {
if m.Rack == "rack1" {
rack1Metrics = append(rack1Metrics, m)
}
}
at := buildTopology(servers, allMetrics)
clusterInfo := &types.ClusterInfo{
ActiveTopology: at,
VolumeReplicaMap: replicaMap,
}
conf := defaultConf()
conf.RackFilter = "rack1"
tasks, _, err := Detection(rack1Metrics, clusterInfo, conf, 100)
if err != nil {
t.Fatalf("Detection failed: %v", err)
}
// Moving within rack1 (node-a → node-b) is valid because the cross-rack
// replica on node-c (rack2) is preserved.
if len(tasks) == 0 {
t.Fatal("Expected tasks for 30/5 imbalance within rack1")
}
for _, task := range tasks {
if task.Server == "node-c" {
t.Error("node-c (rack2) should not be a source with rack1 filter")
}
if task.TypedParams != nil {
for _, tgt := range task.TypedParams.Targets {
if strings.Contains(tgt.Node, "node-c") {
t.Error("node-c (rack2) should not be a target with rack1 filter")
}
}
}
}
assertNoDuplicateVolumes(t, tasks)
t.Logf("Rack filter + replica placement: %d tasks", len(tasks))
}
// TestIntegration_AllFactors exercises all filtering dimensions simultaneously:
// DC filter, rack filter, volume state filter, replica placement validation,
// and mixed collections.
func TestIntegration_AllFactors(t *testing.T) {
servers := []serverSpec{
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack1"},
{id: "node-c", diskType: "hdd", diskID: 3, dc: "dc1", rack: "rack2"}, // excluded by rack filter
{id: "node-d", diskType: "hdd", diskID: 4, dc: "dc2", rack: "rack1"}, // excluded by DC filter
}
var allMetrics []*types.VolumeHealthMetrics
// node-a: 25 active "photos" + 10 full "photos" (full excluded by state filter)
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "photos", 1, 25, withFullness(0.5), withReplicas(100))...)
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "photos", 200, 10, withFullness(1.5), withReplicas(100))...)
// node-b: 5 active "photos"
allMetrics = append(allMetrics, makeVolumesWith("node-b", "hdd", "dc1", "rack1", "photos", 100, 5, withFullness(0.5), withReplicas(100))...)
// node-c: 20 volumes in dc1/rack2 (excluded by rack filter)
allMetrics = append(allMetrics, makeVolumesWith("node-c", "hdd", "dc1", "rack2", "photos", 300, 20, withFullness(0.5))...)
// node-d: 30 volumes in dc2 (excluded by DC filter, but provides cross-DC replicas)
dc2Replicas := makeVolumesWith("node-d", "hdd", "dc2", "rack1", "photos", 1, 25, withFullness(0.5), withReplicas(100))
// Build replica map: volumes 1-25 have cross-DC replicas on node-d
replicaMap := buildReplicaMap(allMetrics)
for _, r := range dc2Replicas {
replicaMap[r.VolumeID] = append(replicaMap[r.VolumeID], types.ReplicaLocation{
DataCenter: r.DataCenter,
Rack: r.Rack,
NodeID: r.Server,
})
}
// Apply all filters: ACTIVE state, dc1, rack1
filtered := filterByState(allMetrics, stateActive)
var finalMetrics []*types.VolumeHealthMetrics
for _, m := range filtered {
if m.DataCenter == "dc1" && m.Rack == "rack1" {
finalMetrics = append(finalMetrics, m)
}
}
at := buildTopology(servers, allMetrics)
clusterInfo := &types.ClusterInfo{
ActiveTopology: at,
VolumeReplicaMap: replicaMap,
}
conf := defaultConf()
conf.DataCenterFilter = "dc1"
conf.RackFilter = "rack1"
tasks, _, err := Detection(finalMetrics, clusterInfo, conf, 100)
if err != nil {
t.Fatalf("Detection failed: %v", err)
}
if len(tasks) == 0 {
t.Fatal("Expected tasks for 25/5 active imbalance in dc1/rack1")
}
// Verify all moves stay within dc1/rack1 scope
for i, task := range tasks {
if task.Server != "node-a" && task.Server != "node-b" {
t.Errorf("Task %d: source %s should be node-a or node-b", i, task.Server)
}
if task.TypedParams != nil {
for _, tgt := range task.TypedParams.Targets {
node := tgt.Node
if !strings.Contains(node, "node-a") && !strings.Contains(node, "node-b") {
t.Errorf("Task %d: target %s should be node-a or node-b", i, node)
}
}
}
}
assertNoDuplicateVolumes(t, tasks)
// Verify convergence
effective := computeEffectiveCounts(servers[:2], finalMetrics, tasks)
total := 0
maxC, minC := 0, len(finalMetrics)
for _, c := range effective {
total += c
if c > maxC {
maxC = c
}
if c < minC {
minC = c
}
}
// Should have balanced the 30 active dc1/rack1 volumes (25+5)
if total != 30 {
t.Errorf("Expected 30 total filtered volumes, got %d", total)
}
avg := float64(total) / float64(len(effective))
imbalance := float64(maxC-minC) / avg
if imbalance > conf.ImbalanceThreshold {
t.Errorf("Still imbalanced: effective=%v, imbalance=%.1f%% (threshold=%.1f%%)",
effective, imbalance*100, conf.ImbalanceThreshold*100)
}
t.Logf("All factors combined: %d tasks, effective=%v", len(tasks), effective)
}
// TestIntegration_FullVolumesOnlyBalancing verifies that with FULL state filter,
// only full volumes participate in balancing.
func TestIntegration_FullVolumesOnlyBalancing(t *testing.T) {
servers := []serverSpec{
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack1"},
}
var allMetrics []*types.VolumeHealthMetrics
// node-a: 10 active + 30 full
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 1, 10, withFullness(0.5))...)
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 100, 30, withFullness(1.5))...)
// node-b: 10 active + 5 full
allMetrics = append(allMetrics, makeVolumesWith("node-b", "hdd", "dc1", "rack1", "c1", 200, 10, withFullness(0.5))...)
allMetrics = append(allMetrics, makeVolumesWith("node-b", "hdd", "dc1", "rack1", "c1", 300, 5, withFullness(1.5))...)
// Filter to FULL only
fullMetrics := filterByState(allMetrics, stateFull)
at := buildTopology(servers, allMetrics)
clusterInfo := &types.ClusterInfo{ActiveTopology: at}
tasks, _, err := Detection(fullMetrics, clusterInfo, defaultConf(), 100)
if err != nil {
t.Fatalf("Detection failed: %v", err)
}
if len(tasks) == 0 {
t.Fatal("Expected tasks for 30/5 full volume imbalance")
}
// Verify only full volumes are moved (IDs 100-129 from node-a, 300-304 from node-b)
for _, task := range tasks {
vid := task.VolumeID
// Active volumes are IDs 1-10 (node-a) and 200-209 (node-b)
if (vid >= 1 && vid <= 10) || (vid >= 200 && vid <= 209) {
t.Errorf("Task moved active volume %d, should only move full volumes", vid)
}
}
assertNoDuplicateVolumes(t, tasks)
t.Logf("Full-only balancing: %d tasks", len(tasks))
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,158 @@
package balance
import (
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// TypedTask implements balance operation with typed protobuf parameters
type TypedTask struct {
*base.BaseTypedTask
// Task state from protobuf
sourceServer string
destNode string
volumeID uint32
collection string
estimatedSize uint64
forceMove bool
timeoutSeconds int32
}
// NewTypedTask creates a new typed balance task
func NewTypedTask() types.TypedTaskInterface {
task := &TypedTask{
BaseTypedTask: base.NewBaseTypedTask(types.TaskTypeBalance),
}
return task
}
// ValidateTyped validates the typed parameters for balance task
func (t *TypedTask) ValidateTyped(params *worker_pb.TaskParams) error {
// Basic validation from base class
if err := t.BaseTypedTask.ValidateTyped(params); err != nil {
return err
}
// Check that we have balance-specific parameters
balanceParams := params.GetBalanceParams()
if balanceParams == nil {
return fmt.Errorf("balance_params is required for balance task")
}
// Validate sources and targets
if len(params.Sources) == 0 {
return fmt.Errorf("at least one source is required for balance task")
}
if len(params.Targets) == 0 {
return fmt.Errorf("at least one target is required for balance task")
}
// Validate that source and target have volume IDs
if params.Sources[0].VolumeId == 0 {
return fmt.Errorf("source volume_id is required for balance task")
}
if params.Targets[0].VolumeId == 0 {
return fmt.Errorf("target volume_id is required for balance task")
}
// Validate timeout
if balanceParams.TimeoutSeconds <= 0 {
return fmt.Errorf("timeout_seconds must be greater than 0")
}
return nil
}
// EstimateTimeTyped estimates the time needed for balance operation based on protobuf parameters
func (t *TypedTask) EstimateTimeTyped(params *worker_pb.TaskParams) time.Duration {
balanceParams := params.GetBalanceParams()
if balanceParams != nil {
// Use the timeout from parameters if specified
if balanceParams.TimeoutSeconds > 0 {
return time.Duration(balanceParams.TimeoutSeconds) * time.Second
}
}
// Estimate based on volume size from sources (1 minute per GB)
if len(params.Sources) > 0 {
source := params.Sources[0]
if source.EstimatedSize > 0 {
gbSize := source.EstimatedSize / (1024 * 1024 * 1024)
return time.Duration(gbSize) * time.Minute
}
}
// Default estimation
return 10 * time.Minute
}
// ExecuteTyped implements the balance operation with typed parameters
func (t *TypedTask) ExecuteTyped(params *worker_pb.TaskParams) error {
// Extract basic parameters
t.volumeID = params.VolumeId
t.collection = params.Collection
// Ensure sources and targets are present (should be guaranteed by validation)
if len(params.Sources) == 0 {
return fmt.Errorf("at least one source is required for balance task (ExecuteTyped)")
}
if len(params.Targets) == 0 {
return fmt.Errorf("at least one target is required for balance task (ExecuteTyped)")
}
// Extract source and target information
t.sourceServer = params.Sources[0].Node
t.estimatedSize = params.Sources[0].EstimatedSize
t.destNode = params.Targets[0].Node
// Extract balance-specific parameters
balanceParams := params.GetBalanceParams()
if balanceParams != nil {
t.forceMove = balanceParams.ForceMove
t.timeoutSeconds = balanceParams.TimeoutSeconds
}
glog.Infof("Starting typed balance task for volume %d: %s -> %s (collection: %s, size: %d bytes)",
t.volumeID, t.sourceServer, t.destNode, t.collection, t.estimatedSize)
// Simulate balance operation with progress updates
steps := []struct {
name string
duration time.Duration
progress float64
}{
{"Analyzing cluster state", 2 * time.Second, 15},
{"Verifying destination capacity", 1 * time.Second, 25},
{"Starting volume migration", 1 * time.Second, 35},
{"Moving volume data", 6 * time.Second, 75},
{"Updating cluster metadata", 2 * time.Second, 95},
{"Verifying balance completion", 1 * time.Second, 100},
}
for _, step := range steps {
if t.IsCancelled() {
return fmt.Errorf("balance task cancelled during: %s", step.name)
}
glog.V(1).Infof("Balance task step: %s", step.name)
t.SetProgress(step.progress)
// Simulate work
time.Sleep(step.duration)
}
glog.Infof("Typed balance task completed successfully for volume %d: %s -> %s",
t.volumeID, t.sourceServer, t.destNode)
return nil
}
// Register the typed task in the global registry
func init() {
types.RegisterGlobalTypedTask(types.TaskTypeBalance, NewTypedTask)
glog.V(1).Infof("Registered typed balance task")
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,828 @@
package balance
import (
"context"
"fmt"
"strings"
"sync"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
"google.golang.org/protobuf/proto"
)
func TestDecodeVolumeBalanceTaskParamsFromPayload(t *testing.T) {
expected := &worker_pb.TaskParams{
TaskId: "task-1",
VolumeId: 42,
Collection: "photos",
Sources: []*worker_pb.TaskSource{
{
Node: "10.0.0.1:8080",
VolumeId: 42,
},
},
Targets: []*worker_pb.TaskTarget{
{
Node: "10.0.0.2:8080",
VolumeId: 42,
},
},
TaskParams: &worker_pb.TaskParams_BalanceParams{
BalanceParams: &worker_pb.BalanceTaskParams{
ForceMove: true,
TimeoutSeconds: 1200,
},
},
}
payload, err := proto.Marshal(expected)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
job := &plugin_pb.JobSpec{
JobId: "job-from-admin",
Parameters: map[string]*plugin_pb.ConfigValue{
"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
},
}
actual, err := decodeVolumeBalanceTaskParams(job)
if err != nil {
t.Fatalf("decodeVolumeBalanceTaskParams() err = %v", err)
}
if !proto.Equal(expected, actual) {
t.Fatalf("decoded params mismatch\nexpected: %+v\nactual: %+v", expected, actual)
}
}
func TestDecodeVolumeBalanceTaskParamsFallback(t *testing.T) {
job := &plugin_pb.JobSpec{
JobId: "job-2",
Parameters: map[string]*plugin_pb.ConfigValue{
"volume_id": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7}},
"source_server": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "127.0.0.1:8080"}},
"target_server": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "127.0.0.2:8080"}},
"collection": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "videos"}},
},
}
params, err := decodeVolumeBalanceTaskParams(job)
if err != nil {
t.Fatalf("decodeVolumeBalanceTaskParams() err = %v", err)
}
if params.TaskId != "job-2" || params.VolumeId != 7 || params.Collection != "videos" {
t.Fatalf("unexpected basic params: %+v", params)
}
if len(params.Sources) != 1 || params.Sources[0].Node != "127.0.0.1:8080" {
t.Fatalf("unexpected sources: %+v", params.Sources)
}
if len(params.Targets) != 1 || params.Targets[0].Node != "127.0.0.2:8080" {
t.Fatalf("unexpected targets: %+v", params.Targets)
}
if params.GetBalanceParams() == nil {
t.Fatalf("expected fallback balance params")
}
}
func TestDeriveBalanceWorkerConfig(t *testing.T) {
values := map[string]*plugin_pb.ConfigValue{
"imbalance_threshold": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.45},
},
"min_server_count": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 5},
},
}
cfg := deriveBalanceWorkerConfig(values)
if cfg.TaskConfig.ImbalanceThreshold != 0.45 {
t.Fatalf("expected imbalance_threshold 0.45, got %v", cfg.TaskConfig.ImbalanceThreshold)
}
if cfg.TaskConfig.MinServerCount != 5 {
t.Fatalf("expected min_server_count 5, got %d", cfg.TaskConfig.MinServerCount)
}
// Defaults for batch config when not specified
if cfg.MaxConcurrentMoves != defaultMaxConcurrentMoves {
t.Fatalf("expected default max_concurrent_moves %d, got %d", defaultMaxConcurrentMoves, cfg.MaxConcurrentMoves)
}
if cfg.BatchSize != 20 {
t.Fatalf("expected default batch_size 20, got %d", cfg.BatchSize)
}
}
func TestDeriveBalanceWorkerConfigBatchFields(t *testing.T) {
values := map[string]*plugin_pb.ConfigValue{
"max_concurrent_moves": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 10},
},
"batch_size": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 50},
},
}
cfg := deriveBalanceWorkerConfig(values)
if cfg.MaxConcurrentMoves != 10 {
t.Fatalf("expected max_concurrent_moves 10, got %d", cfg.MaxConcurrentMoves)
}
if cfg.BatchSize != 50 {
t.Fatalf("expected batch_size 50, got %d", cfg.BatchSize)
}
}
func TestDeriveBalanceWorkerConfigBatchClamping(t *testing.T) {
values := map[string]*plugin_pb.ConfigValue{
"max_concurrent_moves": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 999},
},
"batch_size": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0},
},
}
cfg := deriveBalanceWorkerConfig(values)
if cfg.MaxConcurrentMoves != 50 {
t.Fatalf("expected max_concurrent_moves clamped to 50, got %d", cfg.MaxConcurrentMoves)
}
if cfg.BatchSize != 1 {
t.Fatalf("expected batch_size clamped to 1, got %d", cfg.BatchSize)
}
}
func makeDetectionResult(volumeID uint32, source, target, collection string) *workertypes.TaskDetectionResult {
return &workertypes.TaskDetectionResult{
TaskID: fmt.Sprintf("balance-%d", volumeID),
TaskType: workertypes.TaskTypeBalance,
VolumeID: volumeID,
Server: source,
Collection: collection,
Priority: workertypes.TaskPriorityNormal,
Reason: "imbalanced",
TypedParams: &worker_pb.TaskParams{
VolumeId: volumeID,
Collection: collection,
VolumeSize: 1024,
Sources: []*worker_pb.TaskSource{
{Node: source, VolumeId: volumeID},
},
Targets: []*worker_pb.TaskTarget{
{Node: target, VolumeId: volumeID},
},
TaskParams: &worker_pb.TaskParams_BalanceParams{
BalanceParams: &worker_pb.BalanceTaskParams{TimeoutSeconds: 600},
},
},
}
}
func TestBuildBatchVolumeBalanceProposals_SingleBatch(t *testing.T) {
results := []*workertypes.TaskDetectionResult{
makeDetectionResult(1, "s1:8080", "t1:8080", "c1"),
makeDetectionResult(2, "s2:8080", "t2:8080", "c1"),
makeDetectionResult(3, "s1:8080", "t2:8080", "c1"),
}
proposals := buildBatchVolumeBalanceProposals(results, 10, 5)
if len(proposals) != 1 {
t.Fatalf("expected 1 batch proposal, got %d", len(proposals))
}
p := proposals[0]
if p.Labels["batch"] != "true" {
t.Fatalf("expected batch label")
}
if p.Labels["batch_size"] != "3" {
t.Fatalf("expected batch_size label '3', got %q", p.Labels["batch_size"])
}
// Decode and verify moves
payload := p.Parameters["task_params_pb"].GetBytesValue()
if len(payload) == 0 {
t.Fatalf("expected task_params_pb payload")
}
decoded := &worker_pb.TaskParams{}
if err := proto.Unmarshal(payload, decoded); err != nil {
t.Fatalf("unmarshal: %v", err)
}
moves := decoded.GetBalanceParams().GetMoves()
if len(moves) != 3 {
t.Fatalf("expected 3 moves, got %d", len(moves))
}
if moves[0].VolumeId != 1 || moves[1].VolumeId != 2 || moves[2].VolumeId != 3 {
t.Fatalf("unexpected volume IDs: %v", moves)
}
if decoded.GetBalanceParams().MaxConcurrentMoves != 5 {
t.Fatalf("expected MaxConcurrentMoves 5, got %d", decoded.GetBalanceParams().MaxConcurrentMoves)
}
}
func TestBuildBatchVolumeBalanceProposals_MultipleBatches(t *testing.T) {
results := make([]*workertypes.TaskDetectionResult, 5)
for i := range results {
results[i] = makeDetectionResult(uint32(i+1), "s1:8080", "t1:8080", "c1")
}
proposals := buildBatchVolumeBalanceProposals(results, 2, 3)
// 5 results / batch_size 2 = 3 proposals (2, 2, 1)
if len(proposals) != 3 {
t.Fatalf("expected 3 proposals, got %d", len(proposals))
}
// First two should be batch proposals
if proposals[0].Labels["batch"] != "true" {
t.Fatalf("first proposal should be batch")
}
if proposals[1].Labels["batch"] != "true" {
t.Fatalf("second proposal should be batch")
}
// Last one has only 1 result, should fall back to single-move proposal
if proposals[2].Labels["batch"] == "true" {
t.Fatalf("last proposal with 1 result should be single-move, not batch")
}
}
func TestBuildBatchVolumeBalanceProposals_BatchSizeOne(t *testing.T) {
results := []*workertypes.TaskDetectionResult{
makeDetectionResult(1, "s1:8080", "t1:8080", "c1"),
makeDetectionResult(2, "s2:8080", "t2:8080", "c1"),
}
// batch_size=1 should not be called (Detect guards this), but test the function directly
proposals := buildBatchVolumeBalanceProposals(results, 1, 5)
// Each result becomes its own single-move proposal
if len(proposals) != 2 {
t.Fatalf("expected 2 proposals, got %d", len(proposals))
}
}
func TestVolumeBalanceDescriptorHasBatchFields(t *testing.T) {
descriptor := NewVolumeBalanceHandler(nil).Descriptor()
if !workerConfigFormHasField(descriptor.WorkerConfigForm, "max_concurrent_moves") {
t.Fatalf("expected max_concurrent_moves in worker config form")
}
if !workerConfigFormHasField(descriptor.WorkerConfigForm, "batch_size") {
t.Fatalf("expected batch_size in worker config form")
}
}
func TestBuildVolumeBalanceProposal(t *testing.T) {
params := &worker_pb.TaskParams{
TaskId: "balance-task-1",
VolumeId: 55,
Collection: "images",
Sources: []*worker_pb.TaskSource{
{
Node: "source-a:8080",
VolumeId: 55,
},
},
Targets: []*worker_pb.TaskTarget{
{
Node: "target-b:8080",
VolumeId: 55,
},
},
TaskParams: &worker_pb.TaskParams_BalanceParams{
BalanceParams: &worker_pb.BalanceTaskParams{
TimeoutSeconds: 600,
},
},
}
result := &workertypes.TaskDetectionResult{
TaskID: "balance-task-1",
TaskType: workertypes.TaskTypeBalance,
VolumeID: 55,
Server: "source-a",
Collection: "images",
Priority: workertypes.TaskPriorityHigh,
Reason: "imbalanced load",
TypedParams: params,
}
proposal, err := buildVolumeBalanceProposal(result)
if err != nil {
t.Fatalf("buildVolumeBalanceProposal() err = %v", err)
}
if proposal.JobType != "volume_balance" {
t.Fatalf("unexpected job type %q", proposal.JobType)
}
if proposal.DedupeKey == "" {
t.Fatalf("expected dedupe key")
}
if proposal.Parameters["task_params_pb"] == nil {
t.Fatalf("expected serialized task params")
}
if proposal.Labels["source_node"] != "source-a:8080" {
t.Fatalf("unexpected source label %q", proposal.Labels["source_node"])
}
if proposal.Labels["target_node"] != "target-b:8080" {
t.Fatalf("unexpected target label %q", proposal.Labels["target_node"])
}
}
func TestVolumeBalanceHandlerRejectsUnsupportedJobType(t *testing.T) {
handler := NewVolumeBalanceHandler(nil)
err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
JobType: "vacuum",
}, noopDetectionSender{})
if err == nil {
t.Fatalf("expected detect job type mismatch error")
}
err = handler.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
Job: &plugin_pb.JobSpec{JobId: "job-1", JobType: "vacuum"},
}, noopExecutionSender{})
if err == nil {
t.Fatalf("expected execute job type mismatch error")
}
}
func TestEmitVolumeBalanceDetectionDecisionTraceNoTasks(t *testing.T) {
sender := &recordingDetectionSender{}
config := NewDefaultConfig()
config.ImbalanceThreshold = 0.2
config.MinServerCount = 2
metrics := []*workertypes.VolumeHealthMetrics{
{VolumeID: 1, Server: "server-a", DiskType: "hdd"},
{VolumeID: 2, Server: "server-a", DiskType: "hdd"},
{VolumeID: 3, Server: "server-b", DiskType: "hdd"},
{VolumeID: 4, Server: "server-b", DiskType: "hdd"},
}
if err := emitVolumeBalanceDetectionDecisionTrace(sender, metrics, nil, config, nil); err != nil {
t.Fatalf("emitVolumeBalanceDetectionDecisionTrace error: %v", err)
}
if len(sender.events) < 2 {
t.Fatalf("expected at least 2 detection events, got %d", len(sender.events))
}
if sender.events[0].Source != plugin_pb.ActivitySource_ACTIVITY_SOURCE_DETECTOR {
t.Fatalf("expected detector source, got %v", sender.events[0].Source)
}
if !strings.Contains(sender.events[0].Message, "BALANCE: No tasks created for 4 volumes") {
t.Fatalf("unexpected summary message: %q", sender.events[0].Message)
}
foundDiskTypeDecision := false
for _, event := range sender.events {
if strings.Contains(event.Message, "BALANCE [hdd]: No tasks created - cluster well balanced") {
foundDiskTypeDecision = true
break
}
}
if !foundDiskTypeDecision {
t.Fatalf("expected per-disk-type decision message")
}
}
func TestVolumeBalanceDescriptorOmitsExecutionTuningFields(t *testing.T) {
descriptor := NewVolumeBalanceHandler(nil).Descriptor()
if descriptor == nil || descriptor.WorkerConfigForm == nil {
t.Fatalf("expected worker config form in descriptor")
}
if workerConfigFormHasField(descriptor.WorkerConfigForm, "timeout_seconds") {
t.Fatalf("unexpected timeout_seconds in volume balance worker config form")
}
if workerConfigFormHasField(descriptor.WorkerConfigForm, "force_move") {
t.Fatalf("unexpected force_move in volume balance worker config form")
}
}
type recordingExecutionSender struct {
mu sync.Mutex
progress []*plugin_pb.JobProgressUpdate
completed *plugin_pb.JobCompleted
}
func (r *recordingExecutionSender) SendProgress(p *plugin_pb.JobProgressUpdate) error {
r.mu.Lock()
defer r.mu.Unlock()
r.progress = append(r.progress, proto.Clone(p).(*plugin_pb.JobProgressUpdate))
return nil
}
func (r *recordingExecutionSender) SendCompleted(c *plugin_pb.JobCompleted) error {
r.mu.Lock()
defer r.mu.Unlock()
r.completed = proto.Clone(c).(*plugin_pb.JobCompleted)
return nil
}
func TestBuildMoveTaskParams(t *testing.T) {
move := &worker_pb.BalanceMoveSpec{
VolumeId: 42,
SourceNode: "10.0.0.1:8080",
TargetNode: "10.0.0.2:8080",
Collection: "photos",
VolumeSize: 1024 * 1024,
}
outerParams := &worker_pb.BalanceTaskParams{
ForceMove: true,
TimeoutSeconds: 300,
}
params := buildMoveTaskParams(move, outerParams)
if params.VolumeId != 42 {
t.Fatalf("expected volume_id 42, got %d", params.VolumeId)
}
if params.Collection != "photos" {
t.Fatalf("expected collection photos, got %s", params.Collection)
}
if params.VolumeSize != 1024*1024 {
t.Fatalf("expected volume_size %d, got %d", 1024*1024, params.VolumeSize)
}
if len(params.Sources) != 1 || params.Sources[0].Node != "10.0.0.1:8080" {
t.Fatalf("unexpected sources: %+v", params.Sources)
}
if len(params.Targets) != 1 || params.Targets[0].Node != "10.0.0.2:8080" {
t.Fatalf("unexpected targets: %+v", params.Targets)
}
bp := params.GetBalanceParams()
if bp == nil {
t.Fatalf("expected balance params")
}
if bp.TimeoutSeconds != 300 {
t.Fatalf("expected timeout 300, got %d", bp.TimeoutSeconds)
}
if !bp.ForceMove {
t.Fatalf("expected force_move to be propagated from outer params")
}
}
func TestBuildMoveTaskParamsDefaultTimeout(t *testing.T) {
move := &worker_pb.BalanceMoveSpec{
VolumeId: 1,
SourceNode: "a:8080",
TargetNode: "b:8080",
}
params := buildMoveTaskParams(move, nil)
if params.GetBalanceParams().TimeoutSeconds != defaultBalanceTimeoutSeconds {
t.Fatalf("expected default timeout %d, got %d", defaultBalanceTimeoutSeconds, params.GetBalanceParams().TimeoutSeconds)
}
if params.GetBalanceParams().ForceMove {
t.Fatalf("expected force_move to default to false with nil outer params")
}
}
func TestExecuteDispatchesBatchPath(t *testing.T) {
// Build a job with batch moves in BalanceTaskParams
bp := &worker_pb.BalanceTaskParams{
TimeoutSeconds: 60,
MaxConcurrentMoves: 2,
Moves: []*worker_pb.BalanceMoveSpec{
{VolumeId: 1, SourceNode: "s1:8080", TargetNode: "t1:8080", Collection: "c1"},
{VolumeId: 2, SourceNode: "s2:8080", TargetNode: "t2:8080", Collection: "c1"},
},
}
taskParams := &worker_pb.TaskParams{
TaskId: "batch-1",
TaskParams: &worker_pb.TaskParams_BalanceParams{
BalanceParams: bp,
},
}
payload, err := proto.Marshal(taskParams)
if err != nil {
t.Fatalf("marshal: %v", err)
}
job := &plugin_pb.JobSpec{
JobId: "batch-job-1",
JobType: "volume_balance",
Parameters: map[string]*plugin_pb.ConfigValue{
"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
},
}
handler := NewVolumeBalanceHandler(nil)
sender := &recordingExecutionSender{}
// Execute will enter the batch path. It will fail because there's no real gRPC server,
// but we verify it sends the assigned progress and eventually a completion.
err = handler.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
Job: job,
}, sender)
if err != nil {
t.Fatalf("handler.Execute: %v", err)
}
// Expect an error since no real volume servers exist
// But verify the batch path was taken by checking the assigned message
sender.mu.Lock()
defer sender.mu.Unlock()
if len(sender.progress) == 0 {
t.Fatalf("expected progress messages from batch path")
}
// First progress should be "assigned" with batch info
first := sender.progress[0]
if first.Stage != "assigned" {
t.Fatalf("expected first stage 'assigned', got %q", first.Stage)
}
if !strings.Contains(first.Message, "batch") || !strings.Contains(first.Message, "2 moves") {
t.Fatalf("expected batch assigned message, got %q", first.Message)
}
// Should have a completion with failure details (since no servers)
if sender.completed == nil {
t.Fatalf("expected completion message")
}
if sender.completed.Success {
t.Fatalf("expected failure since no real gRPC servers")
}
// Should report 0 succeeded, 2 failed
if v, ok := sender.completed.Result.OutputValues["failed"]; !ok || v.GetInt64Value() != 2 {
t.Fatalf("expected 2 failed moves, got %+v", sender.completed.Result.OutputValues)
}
}
func TestExecuteSingleMovePathUnchanged(t *testing.T) {
// Build a single-move job (no batch moves)
taskParams := &worker_pb.TaskParams{
TaskId: "single-1",
VolumeId: 99,
Collection: "videos",
Sources: []*worker_pb.TaskSource{
{Node: "src:8080", VolumeId: 99},
},
Targets: []*worker_pb.TaskTarget{
{Node: "dst:8080", VolumeId: 99},
},
TaskParams: &worker_pb.TaskParams_BalanceParams{
BalanceParams: &worker_pb.BalanceTaskParams{
TimeoutSeconds: 60,
},
},
}
payload, err := proto.Marshal(taskParams)
if err != nil {
t.Fatalf("marshal: %v", err)
}
job := &plugin_pb.JobSpec{
JobId: "single-job-1",
JobType: "volume_balance",
Parameters: map[string]*plugin_pb.ConfigValue{
"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
},
}
handler := NewVolumeBalanceHandler(nil)
sender := &recordingExecutionSender{}
// Execute single-move path. Will fail on gRPC but verify it takes the single-move path.
_ = handler.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
Job: job,
}, sender)
sender.mu.Lock()
defer sender.mu.Unlock()
if len(sender.progress) == 0 {
t.Fatalf("expected progress messages")
}
// Single-move path sends "volume balance job accepted" not "batch volume balance"
first := sender.progress[0]
if first.Stage != "assigned" {
t.Fatalf("expected first stage 'assigned', got %q", first.Stage)
}
if strings.Contains(first.Message, "batch") {
t.Fatalf("single-move path should not mention batch, got %q", first.Message)
}
}
func TestFilterMetricsByLocation(t *testing.T) {
metrics := []*workertypes.VolumeHealthMetrics{
{VolumeID: 1, Server: "node-a", DataCenter: "dc1", Rack: "rack1"},
{VolumeID: 2, Server: "node-b", DataCenter: "dc1", Rack: "rack2"},
{VolumeID: 3, Server: "node-c", DataCenter: "dc2", Rack: "rack1"},
{VolumeID: 4, Server: "node-d", DataCenter: "dc2", Rack: "rack3"},
}
// Filter by DC
filtered := pluginworker.FilterMetricsByLocation(metrics, "dc1", "", "")
if len(filtered) != 2 {
t.Fatalf("DC filter: expected 2, got %d", len(filtered))
}
// Filter by rack
filtered = pluginworker.FilterMetricsByLocation(metrics, "", "rack1,rack2", "")
if len(filtered) != 3 {
t.Fatalf("rack filter: expected 3, got %d", len(filtered))
}
// Filter by node
filtered = pluginworker.FilterMetricsByLocation(metrics, "", "", "node-a,node-c")
if len(filtered) != 2 {
t.Fatalf("node filter: expected 2, got %d", len(filtered))
}
// Combined DC + rack
filtered = pluginworker.FilterMetricsByLocation(metrics, "dc2", "rack3", "")
if len(filtered) != 1 {
t.Fatalf("DC+rack filter: expected 1, got %d", len(filtered))
}
// Empty filters pass all
filtered = pluginworker.FilterMetricsByLocation(metrics, "", "", "")
if len(filtered) != 4 {
t.Fatalf("no filter: expected 4, got %d", len(filtered))
}
}
func TestFilterMetricsByVolumeState(t *testing.T) {
metrics := []*workertypes.VolumeHealthMetrics{
{VolumeID: 1, FullnessRatio: 0.5}, // active
{VolumeID: 2, FullnessRatio: 1.0}, // active (below 1.01)
{VolumeID: 3, FullnessRatio: 1.009}, // active (below 1.01)
{VolumeID: 4, FullnessRatio: 1.01}, // full (exactly at threshold)
{VolumeID: 5, FullnessRatio: 1.5}, // full
{VolumeID: 6, FullnessRatio: 2.0}, // full
}
tests := []struct {
name string
state string
expectedIDs []uint32
}{
{
name: "ALL returns everything",
state: "ALL",
expectedIDs: []uint32{1, 2, 3, 4, 5, 6},
},
{
name: "empty string returns everything",
state: "",
expectedIDs: []uint32{1, 2, 3, 4, 5, 6},
},
{
name: "ACTIVE keeps FullnessRatio below 1.01",
state: "ACTIVE",
expectedIDs: []uint32{1, 2, 3},
},
{
name: "FULL keeps FullnessRatio at or above 1.01",
state: "FULL",
expectedIDs: []uint32{4, 5, 6},
},
{
name: "unknown value returns everything",
state: "INVALID",
expectedIDs: []uint32{1, 2, 3, 4, 5, 6},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := pluginworker.FilterMetricsByVolumeState(metrics, pluginworker.VolumeState(tt.state))
if len(result) != len(tt.expectedIDs) {
t.Fatalf("expected %d metrics, got %d", len(tt.expectedIDs), len(result))
}
for i, m := range result {
if m.VolumeID != tt.expectedIDs[i] {
t.Errorf("result[%d].VolumeID = %d, want %d", i, m.VolumeID, tt.expectedIDs[i])
}
}
})
}
}
func TestFilterMetricsByVolumeState_NilElement(t *testing.T) {
metrics := []*workertypes.VolumeHealthMetrics{
nil,
{VolumeID: 1, FullnessRatio: 0.5},
nil,
{VolumeID: 2, FullnessRatio: 1.5},
}
result := pluginworker.FilterMetricsByVolumeState(metrics, pluginworker.VolumeStateActive)
if len(result) != 1 || result[0].VolumeID != 1 {
t.Fatalf("expected [vol 1] for ACTIVE with nil elements, got %d results", len(result))
}
result = pluginworker.FilterMetricsByVolumeState(metrics, pluginworker.VolumeStateFull)
if len(result) != 1 || result[0].VolumeID != 2 {
t.Fatalf("expected [vol 2] for FULL with nil elements, got %d results", len(result))
}
}
func TestFilterMetricsByVolumeState_EmptyInput(t *testing.T) {
result := pluginworker.FilterMetricsByVolumeState(nil, pluginworker.VolumeStateActive)
if len(result) != 0 {
t.Fatalf("expected 0 metrics for nil input, got %d", len(result))
}
result = pluginworker.FilterMetricsByVolumeState([]*workertypes.VolumeHealthMetrics{}, pluginworker.VolumeStateFull)
if len(result) != 0 {
t.Fatalf("expected 0 metrics for empty input, got %d", len(result))
}
}
func TestVolumeBalanceDescriptorHasVolumeStateField(t *testing.T) {
descriptor := NewVolumeBalanceHandler(nil).Descriptor()
if descriptor == nil || descriptor.AdminConfigForm == nil {
t.Fatalf("expected admin config form in descriptor")
}
found := false
for _, section := range descriptor.AdminConfigForm.Sections {
for _, field := range section.Fields {
if field.Name == "volume_state" {
found = true
break
}
}
}
if !found {
t.Fatalf("expected volume_state field in admin config form")
}
defaultVal, ok := descriptor.AdminConfigForm.DefaultValues["volume_state"]
if !ok {
t.Fatalf("expected volume_state default value")
}
if defaultVal.GetStringValue() != "ALL" {
t.Fatalf("expected volume_state default 'ALL', got %q", defaultVal.GetStringValue())
}
}
func workerConfigFormHasField(form *plugin_pb.ConfigForm, fieldName string) bool {
if form == nil {
return false
}
for _, section := range form.Sections {
if section == nil {
continue
}
for _, field := range section.Fields {
if field != nil && field.Name == fieldName {
return true
}
}
}
return false
}
type noopDetectionSender struct{}
func (noopDetectionSender) SendProposals(*plugin_pb.DetectionProposals) error { return nil }
func (noopDetectionSender) SendComplete(*plugin_pb.DetectionComplete) error { return nil }
func (noopDetectionSender) SendActivity(*plugin_pb.ActivityEvent) error { return nil }
type noopExecutionSender struct{}
func (noopExecutionSender) SendProgress(*plugin_pb.JobProgressUpdate) error { return nil }
func (noopExecutionSender) SendCompleted(*plugin_pb.JobCompleted) error { return nil }
type recordingDetectionSender struct {
proposals *plugin_pb.DetectionProposals
complete *plugin_pb.DetectionComplete
events []*plugin_pb.ActivityEvent
}
func (r *recordingDetectionSender) SendProposals(proposals *plugin_pb.DetectionProposals) error {
r.proposals = proposals
return nil
}
func (r *recordingDetectionSender) SendComplete(complete *plugin_pb.DetectionComplete) error {
r.complete = complete
return nil
}
func (r *recordingDetectionSender) SendActivity(event *plugin_pb.ActivityEvent) error {
if event != nil {
r.events = append(r.events, event)
}
return nil
}
func TestCheckMovePreconditions(t *testing.T) {
tests := []struct {
name string
locations []string
sourceNode string
targetNode string
wantErr string
}{
{"valid move", []string{"10.0.0.1:8080", "10.0.0.3:8080"}, "10.0.0.1:8080", "10.0.0.2:8080", ""},
{"volume left the source", []string{"10.0.0.3:8080"}, "10.0.0.1:8080", "10.0.0.2:8080", "no longer on source"},
{"volume gone entirely", nil, "10.0.0.1:8080", "10.0.0.2:8080", "no longer on source"},
{"target already has a replica", []string{"10.0.0.1:8080", "10.0.0.2:8080"}, "10.0.0.1:8080", "10.0.0.2:8080", "already has a replica on target"},
{"proposal nodes carry grpc suffix", []string{"10.0.0.1:8080", "10.0.0.3:8080"}, "10.0.0.1:8080.18080", "10.0.0.2:8080.18080", ""},
{"grpc-suffixed target already has a replica", []string{"10.0.0.1:8080", "10.0.0.2:8080"}, "10.0.0.1:8080.18080", "10.0.0.2:8080.18080", "already has a replica on target"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := checkMovePreconditions(tt.locations, 5, tt.sourceNode, tt.targetNode)
if tt.wantErr == "" {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
return
}
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
t.Fatalf("expected error containing %q, got %v", tt.wantErr, err)
}
})
}
}

View File

@@ -0,0 +1,95 @@
package balance
import (
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// Global variable to hold the task definition for configuration updates
var globalTaskDef *base.TaskDefinition
// Auto-register this task when the package is imported
func init() {
RegisterBalanceTask()
// Register config updater
tasks.AutoRegisterConfigUpdater(types.TaskTypeBalance, UpdateConfigFromPersistence)
}
// RegisterBalanceTask registers the balance task with the new architecture
func RegisterBalanceTask() {
// Create configuration instance
config := NewDefaultConfig()
// Create shared gRPC dial option using TLS configuration
dialOpt := security.LoadClientTLS(util.GetViper(), "grpc.worker")
// Create complete task definition
taskDef := &base.TaskDefinition{
Type: types.TaskTypeBalance,
Name: "balance",
DisplayName: "Volume Balance",
Description: "Balances volume distribution across servers",
Icon: "fas fa-balance-scale text-warning",
Capabilities: []string{"balance", "distribution"},
Config: config,
ConfigSpec: GetConfigSpec(),
CreateTask: func(params *worker_pb.TaskParams) (types.Task, error) {
if params == nil {
return nil, fmt.Errorf("task parameters are required")
}
if len(params.Sources) == 0 {
return nil, fmt.Errorf("at least one source is required for balance task")
}
return NewBalanceTask(
fmt.Sprintf("balance-%d", params.VolumeId),
params.Sources[0].Node, // Use first source node
params.VolumeId,
params.Collection,
dialOpt,
), nil
},
DetectionFunc: func(metrics []*types.VolumeHealthMetrics, info *types.ClusterInfo, config base.TaskConfig) ([]*types.TaskDetectionResult, error) {
results, _, err := Detection(metrics, info, config, 0)
return results, err
},
ScanInterval: 30 * time.Minute,
SchedulingFunc: Scheduling,
MaxConcurrent: 1,
RepeatInterval: 2 * time.Hour,
}
// Store task definition globally for configuration updates
globalTaskDef = taskDef
// Register everything with a single function call!
base.RegisterTask(taskDef)
}
// UpdateConfigFromPersistence updates the balance configuration from persistence
func UpdateConfigFromPersistence(configPersistence interface{}) error {
if globalTaskDef == nil {
return fmt.Errorf("balance task not registered")
}
// Load configuration from persistence
newConfig := LoadConfigFromPersistence(configPersistence)
if newConfig == nil {
return fmt.Errorf("failed to load configuration from persistence")
}
// Update the task definition's config
globalTaskDef.Config = newConfig
glog.V(1).Infof("Updated balance task configuration from persistence")
return nil
}

View File

@@ -0,0 +1,37 @@
package balance
import (
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/topology/balancer"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// hostFromAddress returns the physical machine (host/IP) of a server address,
// falling back to the node id when no address is known.
func hostFromAddress(address, nodeID string) string {
return balancer.HostFromAddress(address, nodeID)
}
// IsGoodMove checks whether moving a volume from sourceNodeID to target would
// satisfy the volume's replica placement policy. It is a thin adapter over the
// shared placement logic in weed/topology/balancer so the shell command and this
// worker cannot drift.
func IsGoodMove(rp *super_block.ReplicaPlacement, existingReplicas []types.ReplicaLocation, sourceNodeID string, target types.ReplicaLocation) bool {
locs := make([]balancer.Location, len(existingReplicas))
for i, r := range existingReplicas {
locs[i] = toBalancerLocation(r)
}
return balancer.IsGoodMove(rp, locs, sourceNodeID, toBalancerLocation(target))
}
// toBalancerLocation copies a worker replica location to the shared placement
// type field-by-field, so a future change to either struct is a compile error
// here rather than a silent mismatch.
func toBalancerLocation(r types.ReplicaLocation) balancer.Location {
return balancer.Location{
DataCenter: r.DataCenter,
Rack: r.Rack,
NodeID: r.NodeID,
Host: r.Host,
}
}

View File

@@ -0,0 +1,256 @@
package balance
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
func rp(t *testing.T, code string) *super_block.ReplicaPlacement {
t.Helper()
r, err := super_block.NewReplicaPlacementFromString(code)
if err != nil {
t.Fatalf("invalid replica placement code %q: %v", code, err)
}
return r
}
func loc(dc, rack, node string) types.ReplicaLocation {
return types.ReplicaLocation{DataCenter: dc, Rack: rack, NodeID: node}
}
func TestIsGoodMove_NoReplication(t *testing.T) {
// 000 = no replication. Any move is fine.
if !IsGoodMove(rp(t, "000"), []types.ReplicaLocation{loc("dc1", "r1", "n1")}, "n1", loc("dc1", "r1", "n2")) {
t.Error("000: any move should be allowed")
}
}
func TestIsGoodMove_MachineAntiAffinity(t *testing.T) {
// rep 001 allows two copies in one rack, so replica placement alone would permit
// moving onto another port of a host that already holds a replica; machine
// anti-affinity must reject that and allow a distinct host.
existing := []types.ReplicaLocation{
{DataCenter: "dc1", Rack: "r1", NodeID: "10.0.0.1:8080", Host: "10.0.0.1"},
{DataCenter: "dc1", Rack: "r1", NodeID: "10.0.0.2:8080", Host: "10.0.0.2"},
}
onSameMachine := types.ReplicaLocation{DataCenter: "dc1", Rack: "r1", NodeID: "10.0.0.1:8081", Host: "10.0.0.1"}
if IsGoodMove(rp(t, "001"), existing, "10.0.0.2:8080", onSameMachine) {
t.Error("move onto a machine already holding a replica should be rejected")
}
onOtherMachine := types.ReplicaLocation{DataCenter: "dc1", Rack: "r1", NodeID: "10.0.0.3:8080", Host: "10.0.0.3"}
if !IsGoodMove(rp(t, "001"), existing, "10.0.0.2:8080", onOtherMachine) {
t.Error("move onto a distinct machine should be allowed")
}
}
func TestIsGoodMove_001_SameRack(t *testing.T) {
// 001 = 1 replica on same rack (2 total on same rack)
existing := []types.ReplicaLocation{
loc("dc1", "r1", "n1"),
loc("dc1", "r1", "n2"),
}
// Move n1 -> n3 on same rack: good
if !IsGoodMove(rp(t, "001"), existing, "n1", loc("dc1", "r1", "n3")) {
t.Error("001: move to same rack should be allowed")
}
// Move n1 -> n3 on different rack: bad (would leave only 1 on r1, need 2)
if IsGoodMove(rp(t, "001"), existing, "n1", loc("dc1", "r2", "n3")) {
t.Error("001: move to different rack should not be allowed when it breaks same-rack count")
}
}
func TestIsGoodMove_010_DiffRack(t *testing.T) {
// 010 = 1 replica on different rack (2 racks total)
existing := []types.ReplicaLocation{
loc("dc1", "r1", "n1"),
loc("dc1", "r2", "n2"),
}
// Move n1 -> n3 on r2: bad (both replicas on same rack)
if IsGoodMove(rp(t, "010"), existing, "n1", loc("dc1", "r2", "n3")) {
t.Error("010: move to same rack as other replica should not be allowed")
}
// Move n1 -> n3 on r3: good (still 2 different racks)
if !IsGoodMove(rp(t, "010"), existing, "n1", loc("dc1", "r3", "n3")) {
t.Error("010: move to different rack should be allowed")
}
}
func TestIsGoodMove_100_DiffDC(t *testing.T) {
// 100 = 1 replica in different DC
existing := []types.ReplicaLocation{
loc("dc1", "r1", "n1"),
loc("dc2", "r1", "n2"),
}
// Move n1 -> n3 in dc2: bad (both in same DC)
if IsGoodMove(rp(t, "100"), existing, "n1", loc("dc2", "r1", "n3")) {
t.Error("100: move to same DC as other replica should not be allowed")
}
// Move n1 -> n3 in dc3: good (different DCs)
if !IsGoodMove(rp(t, "100"), existing, "n1", loc("dc3", "r1", "n3")) {
t.Error("100: move to different DC should be allowed")
}
}
func TestIsGoodMove_SameNode(t *testing.T) {
// Moving to the same node as an existing replica should always be rejected
existing := []types.ReplicaLocation{
loc("dc1", "r1", "n1"),
loc("dc1", "r2", "n2"),
}
if IsGoodMove(rp(t, "010"), existing, "n1", loc("dc1", "r2", "n2")) {
t.Error("should reject move to same node as existing replica")
}
}
func TestIsGoodMove_011_Composite(t *testing.T) {
// 011 = 1 same-rack + 1 different-rack (3 replicas: 2 on same rack, 1 on different)
existing := []types.ReplicaLocation{
loc("dc1", "r1", "n1"),
loc("dc1", "r1", "n2"),
loc("dc1", "r2", "n3"),
}
// Move n1 -> n4 on r1: good (maintains 2 on r1, 1 on r2)
if !IsGoodMove(rp(t, "011"), existing, "n1", loc("dc1", "r1", "n4")) {
t.Error("011: move within same rack should be allowed")
}
// Move n3 -> n4 on r1: bad (would have 3 on r1, 0 on different rack)
if IsGoodMove(rp(t, "011"), existing, "n3", loc("dc1", "r1", "n4")) {
t.Error("011: move that eliminates different-rack replica should not be allowed")
}
}
func TestIsGoodMove_110_Composite(t *testing.T) {
// 110 = 1 different-rack + 1 different-DC (3 replicas across 2 DCs and 2 racks)
existing := []types.ReplicaLocation{
loc("dc1", "r1", "n1"),
loc("dc1", "r2", "n2"),
loc("dc2", "r1", "n3"),
}
// Move n1 -> n4 in dc1/r3: good (dc1 still has r2+r3, dc2 has r1)
if !IsGoodMove(rp(t, "110"), existing, "n1", loc("dc1", "r3", "n4")) {
t.Error("110: move to new rack in same DC should be allowed")
}
// Move n3 -> n4 in dc1/r1: bad (would lose the different-DC replica)
if IsGoodMove(rp(t, "110"), existing, "n3", loc("dc1", "r1", "n4")) {
t.Error("110: move that eliminates different-DC replica should not be allowed")
}
}
func TestIsGoodMove_NilReplicaPlacement(t *testing.T) {
if !IsGoodMove(nil, []types.ReplicaLocation{loc("dc1", "r1", "n1")}, "n1", loc("dc1", "r1", "n2")) {
t.Error("nil replica placement should allow any move")
}
}
func TestCalculateBalanceScore_ReplicationAware(t *testing.T) {
disk := func(dc, rack string) *topology.DiskInfo {
return &topology.DiskInfo{
DataCenter: dc,
Rack: rack,
DiskInfo: &master_pb.DiskInfo{MaxVolumeCount: 100, VolumeCount: 50},
}
}
// 001: same-rack replication — should prefer same rack and same DC
rp001 := rp(t, "001")
sameRack := calculateBalanceScore(disk("dc1", "r1"), "r1", "dc1", 0, rp001)
diffRack := calculateBalanceScore(disk("dc1", "r2"), "r1", "dc1", 0, rp001)
diffDC := calculateBalanceScore(disk("dc2", "r2"), "r1", "dc1", 0, rp001)
if sameRack <= diffRack {
t.Errorf("001: same-rack score (%v) should exceed different-rack score (%v)", sameRack, diffRack)
}
if sameRack <= diffDC {
t.Errorf("001: same-rack score (%v) should exceed different-DC score (%v)", sameRack, diffDC)
}
// 010: different-rack replication — should prefer different rack, same DC
rp010 := rp(t, "010")
sameRack = calculateBalanceScore(disk("dc1", "r1"), "r1", "dc1", 0, rp010)
diffRack = calculateBalanceScore(disk("dc1", "r2"), "r1", "dc1", 0, rp010)
if diffRack <= sameRack {
t.Errorf("010: different-rack score (%v) should exceed same-rack score (%v)", diffRack, sameRack)
}
// 100: different-DC replication — should prefer different DC
rp100 := rp(t, "100")
sameDC := calculateBalanceScore(disk("dc1", "r2"), "r1", "dc1", 0, rp100)
diffDCScore := calculateBalanceScore(disk("dc2", "r2"), "r1", "dc1", 0, rp100)
if diffDCScore <= sameDC {
t.Errorf("100: different-DC score (%v) should exceed same-DC score (%v)", diffDCScore, sameDC)
}
// nil rp: should prefer cross-rack/DC (default behavior)
sameRack = calculateBalanceScore(disk("dc1", "r1"), "r1", "dc1", 0, nil)
diffRack = calculateBalanceScore(disk("dc1", "r2"), "r1", "dc1", 0, nil)
if diffRack <= sameRack {
t.Errorf("nil rp: different-rack score (%v) should exceed same-rack score (%v)", diffRack, sameRack)
}
}
func TestPlanBalanceDestination_ChoosesBestValidCompositeDestination(t *testing.T) {
servers := []serverSpec{
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack3"},
{id: "node-c", diskType: "hdd", diskID: 3, dc: "dc1", rack: "rack1"},
}
volumes := makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 1, 1, withReplicas(11))
replicas := []types.ReplicaLocation{
loc("dc1", "rack1", "node-a"),
loc("dc1", "rack1", "node-d"),
loc("dc1", "rack2", "node-e"),
}
plan, err := planBalanceDestination(buildTopology(servers, volumes), volumes[0], rp(t, "011"), replicas, map[string]int{
"node-b": 0,
"node-c": 0,
})
if err != nil {
t.Fatalf("planBalanceDestination failed: %v", err)
}
if plan.TargetNode != "node-c" {
t.Fatalf("expected valid same-rack destination node-c, got %s", plan.TargetNode)
}
if plan.TargetRack != "rack1" {
t.Fatalf("expected rack1 destination, got %s", plan.TargetRack)
}
}
func TestCreateBalanceTask_FallbackSelectsValidCompositeDestination(t *testing.T) {
servers := []serverSpec{
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack3"},
{id: "node-c", diskType: "hdd", diskID: 3, dc: "dc1", rack: "rack1"},
}
volumes := makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 1, 1, withReplicas(11))
clusterInfo := &types.ClusterInfo{
ActiveTopology: buildTopology(servers, volumes),
VolumeReplicaMap: map[uint32][]types.ReplicaLocation{
1: {
loc("dc1", "rack1", "node-a"),
loc("dc1", "rack1", "node-d"),
loc("dc1", "rack2", "node-e"),
},
},
}
task, destination := createBalanceTask("hdd", volumes[0], clusterInfo, "node-b", map[string]int{
"node-b": 0,
"node-c": 0,
}, 0)
if task == nil {
t.Fatal("expected a balance task")
}
if destination != "node-c" {
t.Fatalf("expected fallback destination node-c, got %s", destination)
}
if len(task.TypedParams.Targets) != 1 {
t.Fatalf("expected 1 target, got %d", len(task.TypedParams.Targets))
}
if got := task.TypedParams.Targets[0].Node; got != "node-c:8080" {
t.Fatalf("expected target node-c:8080, got %s", got)
}
}

View File

@@ -0,0 +1,37 @@
package balance
import (
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// Scheduling implements the scheduling logic for balance tasks
func Scheduling(task *types.TaskInput, runningTasks []*types.TaskInput, availableWorkers []*types.WorkerData, config base.TaskConfig) bool {
balanceConfig := config.(*Config)
// Count running balance tasks
runningBalanceCount := 0
for _, runningTask := range runningTasks {
if runningTask.Type == types.TaskTypeBalance {
runningBalanceCount++
}
}
// Check concurrency limit
if runningBalanceCount >= balanceConfig.MaxConcurrent {
return false
}
// Check if we have available workers
availableWorkerCount := 0
for _, worker := range availableWorkers {
for _, capability := range worker.Capabilities {
if capability == types.TaskTypeBalance {
availableWorkerCount++
break
}
}
}
return availableWorkerCount > 0
}