snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
135
weed/worker/tasks/balance/balance_task.go
Normal file
135
weed/worker/tasks/balance/balance_task.go
Normal file
@@ -0,0 +1,135 @@
|
||||
package balance
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation/volume_move"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types/base"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
// BalanceTask implements the Task interface
|
||||
type BalanceTask struct {
|
||||
*base.BaseTask
|
||||
server string
|
||||
volumeID uint32
|
||||
collection string
|
||||
progress float64
|
||||
grpcDialOption grpc.DialOption
|
||||
}
|
||||
|
||||
// NewBalanceTask creates a new balance task instance
|
||||
func NewBalanceTask(id string, server string, volumeID uint32, collection string, grpcDialOption grpc.DialOption) *BalanceTask {
|
||||
return &BalanceTask{
|
||||
BaseTask: base.NewBaseTask(id, types.TaskTypeBalance),
|
||||
server: server,
|
||||
volumeID: volumeID,
|
||||
collection: collection,
|
||||
grpcDialOption: grpcDialOption,
|
||||
}
|
||||
}
|
||||
|
||||
// Execute implements the Task interface
|
||||
func (t *BalanceTask) Execute(ctx context.Context, params *worker_pb.TaskParams) error {
|
||||
if params == nil {
|
||||
return fmt.Errorf("task parameters are required")
|
||||
}
|
||||
|
||||
balanceParams := params.GetBalanceParams()
|
||||
if balanceParams == nil {
|
||||
return fmt.Errorf("balance parameters are required")
|
||||
}
|
||||
|
||||
// Get source and destination from unified arrays
|
||||
if len(params.Sources) == 0 {
|
||||
return fmt.Errorf("source is required for balance task")
|
||||
}
|
||||
if len(params.Targets) == 0 {
|
||||
return fmt.Errorf("target is required for balance task")
|
||||
}
|
||||
|
||||
sourceNode := params.Sources[0].Node
|
||||
destNode := params.Targets[0].Node
|
||||
|
||||
if sourceNode == "" {
|
||||
return fmt.Errorf("source node is required for balance task")
|
||||
}
|
||||
if destNode == "" {
|
||||
return fmt.Errorf("destination node is required for balance task")
|
||||
}
|
||||
|
||||
t.GetLogger().WithFields(map[string]interface{}{
|
||||
"volume_id": t.volumeID,
|
||||
"source": sourceNode,
|
||||
"destination": destNode,
|
||||
"collection": t.collection,
|
||||
}).Info("Starting balance task - moving volume")
|
||||
|
||||
// The move sequence — freeze the source, copy, tail, verify the target
|
||||
// matches the source before the destructive delete — is shared with the
|
||||
// shell's volume.move/volume.balance commands.
|
||||
mover := volume_move.NewMover(t.grpcDialOption)
|
||||
err := mover.LiveMoveVolume(ctx, needle.VolumeId(t.volumeID), pb.ServerAddress(sourceNode), pb.ServerAddress(destNode), volume_move.VolumeMoveOptions{
|
||||
IdleTimeout: 60 * time.Second,
|
||||
IoBytePerSecond: balanceParams.IoBytePerSecond,
|
||||
Progress: func(percent float64, stage string) {
|
||||
t.ReportProgress(percent)
|
||||
t.GetLogger().Info("move stage: %s", stage)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("move volume %d from %s to %s: %w", t.volumeID, sourceNode, destNode, err)
|
||||
}
|
||||
|
||||
glog.Infof("Balance task completed successfully: volume %d moved from %s to %s",
|
||||
t.volumeID, sourceNode, destNode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate implements the UnifiedTask interface
|
||||
func (t *BalanceTask) Validate(params *worker_pb.TaskParams) error {
|
||||
if params == nil {
|
||||
return fmt.Errorf("task parameters are required")
|
||||
}
|
||||
|
||||
balanceParams := params.GetBalanceParams()
|
||||
if balanceParams == nil {
|
||||
return fmt.Errorf("balance parameters are required")
|
||||
}
|
||||
|
||||
if params.VolumeId != t.volumeID {
|
||||
return fmt.Errorf("volume ID mismatch: expected %d, got %d", t.volumeID, params.VolumeId)
|
||||
}
|
||||
|
||||
// Validate that at least one source matches our server
|
||||
found := false
|
||||
for _, source := range params.Sources {
|
||||
if source.Node == t.server {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("no source matches expected server %s", t.server)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// EstimateTime implements the UnifiedTask interface
|
||||
func (t *BalanceTask) EstimateTime(params *worker_pb.TaskParams) time.Duration {
|
||||
// Basic estimate based on simulated steps
|
||||
return 14 * time.Second // Sum of all step durations
|
||||
}
|
||||
|
||||
// GetProgress returns current progress
|
||||
func (t *BalanceTask) GetProgress() float64 {
|
||||
return t.progress
|
||||
}
|
||||
204
weed/worker/tasks/balance/config.go
Normal file
204
weed/worker/tasks/balance/config.go
Normal file
@@ -0,0 +1,204 @@
|
||||
package balance
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/config"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
)
|
||||
|
||||
// Config extends BaseConfig with balance-specific settings
|
||||
type Config struct {
|
||||
base.BaseConfig
|
||||
ImbalanceThreshold float64 `json:"imbalance_threshold"`
|
||||
MinServerCount int `json:"min_server_count"`
|
||||
IoBytePerSecond int64 `json:"io_byte_per_second"`
|
||||
DataCenterFilter string `json:"-"` // per-detection-run, not persisted
|
||||
RackFilter string `json:"-"` // per-detection-run, not persisted
|
||||
NodeFilter string `json:"-"` // per-detection-run, not persisted
|
||||
}
|
||||
|
||||
// NewDefaultConfig creates a new default balance configuration
|
||||
func NewDefaultConfig() *Config {
|
||||
return &Config{
|
||||
BaseConfig: base.BaseConfig{
|
||||
Enabled: true,
|
||||
ScanIntervalSeconds: 30 * 60, // 30 minutes
|
||||
MaxConcurrent: 1,
|
||||
},
|
||||
ImbalanceThreshold: 0.2, // 20%
|
||||
MinServerCount: 2,
|
||||
}
|
||||
}
|
||||
|
||||
// GetConfigSpec returns the configuration schema for balance tasks
|
||||
func GetConfigSpec() base.ConfigSpec {
|
||||
return base.ConfigSpec{
|
||||
Fields: []*config.Field{
|
||||
{
|
||||
Name: "enabled",
|
||||
JSONName: "enabled",
|
||||
Type: config.FieldTypeBool,
|
||||
DefaultValue: true,
|
||||
Required: false,
|
||||
DisplayName: "Enable Balance Tasks",
|
||||
Description: "Whether balance tasks should be automatically created",
|
||||
HelpText: "Toggle this to enable or disable automatic balance task generation",
|
||||
InputType: "checkbox",
|
||||
CSSClasses: "form-check-input",
|
||||
},
|
||||
{
|
||||
Name: "scan_interval_seconds",
|
||||
JSONName: "scan_interval_seconds",
|
||||
Type: config.FieldTypeInterval,
|
||||
DefaultValue: 30 * 60,
|
||||
MinValue: 5 * 60,
|
||||
MaxValue: 2 * 60 * 60,
|
||||
Required: true,
|
||||
DisplayName: "Scan Interval",
|
||||
Description: "How often to scan for volume distribution imbalances",
|
||||
HelpText: "The system will check for volume distribution imbalances at this interval",
|
||||
Placeholder: "30",
|
||||
Unit: config.UnitMinutes,
|
||||
InputType: "interval",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "max_concurrent",
|
||||
JSONName: "max_concurrent",
|
||||
Type: config.FieldTypeInt,
|
||||
DefaultValue: 1,
|
||||
MinValue: 1,
|
||||
MaxValue: 3,
|
||||
Required: true,
|
||||
DisplayName: "Max Concurrent Tasks",
|
||||
Description: "Maximum number of balance tasks that can run simultaneously",
|
||||
HelpText: "Limits the number of balance operations running at the same time",
|
||||
Placeholder: "1 (default)",
|
||||
Unit: config.UnitCount,
|
||||
InputType: "number",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "imbalance_threshold",
|
||||
JSONName: "imbalance_threshold",
|
||||
Type: config.FieldTypeFloat,
|
||||
DefaultValue: 0.2,
|
||||
MinValue: 0.05,
|
||||
MaxValue: 0.5,
|
||||
Required: true,
|
||||
DisplayName: "Imbalance Threshold",
|
||||
Description: "Minimum imbalance ratio to trigger balancing",
|
||||
HelpText: "Volume distribution imbalances above this threshold will trigger balancing",
|
||||
Placeholder: "0.20 (20%)",
|
||||
Unit: config.UnitNone,
|
||||
InputType: "number",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "io_byte_per_second",
|
||||
JSONName: "io_byte_per_second",
|
||||
Type: config.FieldTypeInt,
|
||||
DefaultValue: 0,
|
||||
MinValue: 0,
|
||||
Required: false,
|
||||
DisplayName: "Move IO Limit (bytes/sec)",
|
||||
Description: "Limit each volume move's copy rate",
|
||||
HelpText: "0 falls back to each volume server's own maintenance rate (-maintenanceBytePerSecond)",
|
||||
Placeholder: "0 (server maintenance rate)",
|
||||
Unit: config.UnitNone,
|
||||
InputType: "number",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "min_server_count",
|
||||
JSONName: "min_server_count",
|
||||
Type: config.FieldTypeInt,
|
||||
DefaultValue: 2,
|
||||
MinValue: 2,
|
||||
MaxValue: 10,
|
||||
Required: true,
|
||||
DisplayName: "Minimum Server Count",
|
||||
Description: "Minimum number of servers required for balancing",
|
||||
HelpText: "Balancing will only occur if there are at least this many servers",
|
||||
Placeholder: "2 (default)",
|
||||
Unit: config.UnitCount,
|
||||
InputType: "number",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ToTaskPolicy converts configuration to a TaskPolicy protobuf message
|
||||
func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
|
||||
return &worker_pb.TaskPolicy{
|
||||
Enabled: c.Enabled,
|
||||
MaxConcurrent: int32(c.MaxConcurrent),
|
||||
RepeatIntervalSeconds: int32(c.ScanIntervalSeconds),
|
||||
CheckIntervalSeconds: int32(c.ScanIntervalSeconds),
|
||||
TaskConfig: &worker_pb.TaskPolicy_BalanceConfig{
|
||||
BalanceConfig: &worker_pb.BalanceTaskConfig{
|
||||
ImbalanceThreshold: float64(c.ImbalanceThreshold),
|
||||
MinServerCount: int32(c.MinServerCount),
|
||||
IoBytePerSecond: c.IoBytePerSecond,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// FromTaskPolicy loads configuration from a TaskPolicy protobuf message
|
||||
func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
|
||||
if policy == nil {
|
||||
return fmt.Errorf("policy is nil")
|
||||
}
|
||||
|
||||
// Set general TaskPolicy fields
|
||||
c.Enabled = policy.Enabled
|
||||
c.MaxConcurrent = int(policy.MaxConcurrent)
|
||||
c.ScanIntervalSeconds = int(policy.RepeatIntervalSeconds) // Direct seconds-to-seconds mapping
|
||||
|
||||
// Set balance-specific fields from the task config
|
||||
if balanceConfig := policy.GetBalanceConfig(); balanceConfig != nil {
|
||||
c.ImbalanceThreshold = float64(balanceConfig.ImbalanceThreshold)
|
||||
c.MinServerCount = int(balanceConfig.MinServerCount)
|
||||
c.IoBytePerSecond = balanceConfig.IoBytePerSecond
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadConfigFromPersistence loads configuration from the persistence layer if available
|
||||
func LoadConfigFromPersistence(configPersistence interface{}) *Config {
|
||||
config := NewDefaultConfig()
|
||||
|
||||
// Try to load from persistence if available
|
||||
if persistence, ok := configPersistence.(interface {
|
||||
LoadBalanceTaskPolicy() (*worker_pb.TaskPolicy, error)
|
||||
}); ok {
|
||||
policy, err := persistence.LoadBalanceTaskPolicy()
|
||||
switch {
|
||||
case err != nil:
|
||||
glog.Warningf("Could not read the persisted balance configuration, falling back to defaults: %v", err)
|
||||
case policy == nil:
|
||||
glog.V(1).Infof("No balance configuration persisted yet, using defaults")
|
||||
default:
|
||||
if err := config.FromTaskPolicy(policy); err != nil {
|
||||
glog.Warningf("Could not apply the persisted balance configuration, falling back to defaults: %v", err)
|
||||
} else {
|
||||
glog.V(1).Infof("Loaded balance configuration from persistence")
|
||||
return config
|
||||
}
|
||||
}
|
||||
} else if configPersistence != nil {
|
||||
// A store was handed in but does not expose the accessor, so the persisted
|
||||
// settings are silently ignored - always a wiring bug, never a normal state.
|
||||
glog.Warningf("%T cannot provide the persisted balance configuration: it has no LoadBalanceTaskPolicy() method, "+
|
||||
"so the compiled-in defaults are used and any saved balance settings are ignored", configPersistence)
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Using default balance configuration")
|
||||
return config
|
||||
}
|
||||
839
weed/worker/tasks/balance/detection.go
Normal file
839
weed/worker/tasks/balance/detection.go
Normal file
@@ -0,0 +1,839 @@
|
||||
package balance
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/topology/balancer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// Detection implements the detection logic for balance tasks.
|
||||
// maxResults limits how many balance operations are returned per invocation.
|
||||
// A non-positive maxResults means no explicit limit (uses a large default).
|
||||
// The returned truncated flag is true when detection stopped because it hit
|
||||
// maxResults rather than running out of work.
|
||||
func Detection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo, config base.TaskConfig, maxResults int) ([]*types.TaskDetectionResult, bool, error) {
|
||||
if !config.IsEnabled() {
|
||||
return nil, false, nil
|
||||
}
|
||||
if clusterInfo == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
balanceConfig := config.(*Config)
|
||||
|
||||
if maxResults <= 0 {
|
||||
maxResults = math.MaxInt32
|
||||
}
|
||||
|
||||
// Group volumes by disk type to ensure we compare apples to apples.
|
||||
// Remote-tiered volumes are skipped: a "move" copies .idx/.vif then
|
||||
// Volume.Destroy on the source deletes the remote object the destination
|
||||
// now points at. Mirrors shell/command_volume_balance.go.
|
||||
volumesByDiskType := make(map[string][]*types.VolumeHealthMetrics)
|
||||
for _, metric := range metrics {
|
||||
if metric.HasRemoteCopy {
|
||||
continue
|
||||
}
|
||||
volumesByDiskType[metric.DiskType] = append(volumesByDiskType[metric.DiskType], metric)
|
||||
}
|
||||
|
||||
// Sort disk types for deterministic iteration order when maxResults
|
||||
// spans multiple disk types.
|
||||
diskTypes := make([]string, 0, len(volumesByDiskType))
|
||||
for dt := range volumesByDiskType {
|
||||
diskTypes = append(diskTypes, dt)
|
||||
}
|
||||
sort.Strings(diskTypes)
|
||||
|
||||
var allParams []*types.TaskDetectionResult
|
||||
truncated := false
|
||||
|
||||
for _, diskType := range diskTypes {
|
||||
remaining := maxResults - len(allParams)
|
||||
if remaining <= 0 {
|
||||
truncated = true
|
||||
break
|
||||
}
|
||||
tasks, diskTruncated := detectForDiskType(diskType, volumesByDiskType[diskType], balanceConfig, clusterInfo, remaining)
|
||||
allParams = append(allParams, tasks...)
|
||||
if diskTruncated {
|
||||
truncated = true
|
||||
}
|
||||
}
|
||||
|
||||
return allParams, truncated, nil
|
||||
}
|
||||
|
||||
// detectForDiskType performs balance detection for a specific disk type,
|
||||
// returning up to maxResults balance tasks and whether it was truncated by the limit.
|
||||
func detectForDiskType(diskType string, diskMetrics []*types.VolumeHealthMetrics, balanceConfig *Config, clusterInfo *types.ClusterInfo, maxResults int) ([]*types.TaskDetectionResult, bool) {
|
||||
// Skip if cluster segment is too small
|
||||
minVolumeCount := 2 // More reasonable for small clusters
|
||||
if len(diskMetrics) < minVolumeCount {
|
||||
// Only log at verbose level to avoid spamming for small/empty disk types
|
||||
glog.V(1).Infof("BALANCE [%s]: No tasks created - cluster too small (%d volumes, need ≥%d)", diskType, len(diskMetrics), minVolumeCount)
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Analyze volume distribution across servers.
|
||||
// Seed from ActiveTopology so servers with matching disk type but zero
|
||||
// volumes are included in the count and imbalance calculation.
|
||||
// Also collect MaxVolumeCount per server to compute utilization ratios.
|
||||
serverVolumeCounts := make(map[string]int)
|
||||
serverMaxVolumes := make(map[string]int64)
|
||||
serverDiskTotalBytes := make(map[string]uint64)
|
||||
serverDiskFreeBytes := make(map[string]uint64)
|
||||
if clusterInfo.ActiveTopology != nil {
|
||||
topologyInfo := clusterInfo.ActiveTopology.GetTopologyInfo()
|
||||
if topologyInfo != nil {
|
||||
dcMatchers := wildcard.CompileWildcardMatchers(balanceConfig.DataCenterFilter)
|
||||
rackMatchers := wildcard.CompileWildcardMatchers(balanceConfig.RackFilter)
|
||||
nodeMatchers := wildcard.CompileWildcardMatchers(balanceConfig.NodeFilter)
|
||||
for _, dc := range topologyInfo.DataCenterInfos {
|
||||
if !wildcard.MatchesAnyWildcard(dcMatchers, dc.Id) {
|
||||
continue
|
||||
}
|
||||
for _, rack := range dc.RackInfos {
|
||||
if !wildcard.MatchesAnyWildcard(rackMatchers, rack.Id) {
|
||||
continue
|
||||
}
|
||||
for _, node := range rack.DataNodeInfos {
|
||||
if !wildcard.MatchesAnyWildcard(nodeMatchers, node.Id) {
|
||||
continue
|
||||
}
|
||||
for diskTypeName, diskInfo := range node.DiskInfos {
|
||||
if diskTypeName == diskType {
|
||||
serverVolumeCounts[node.Id] = 0
|
||||
serverMaxVolumes[node.Id] += diskInfo.MaxVolumeCount
|
||||
serverDiskTotalBytes[node.Id] += diskInfo.DiskTotalBytes
|
||||
serverDiskFreeBytes[node.Id] += diskInfo.DiskFreeBytes
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
hasLocationFilter := balanceConfig.DataCenterFilter != "" || balanceConfig.RackFilter != "" || balanceConfig.NodeFilter != ""
|
||||
for _, metric := range diskMetrics {
|
||||
if hasLocationFilter {
|
||||
// Only count metrics for servers that passed filtering.
|
||||
// Without this guard, out-of-scope servers are re-introduced.
|
||||
if _, allowed := serverVolumeCounts[metric.Server]; !allowed {
|
||||
continue
|
||||
}
|
||||
}
|
||||
serverVolumeCounts[metric.Server]++
|
||||
}
|
||||
|
||||
if len(serverVolumeCounts) < balanceConfig.MinServerCount {
|
||||
glog.V(1).Infof("BALANCE [%s]: No tasks created - too few servers (%d servers, need ≥%d)", diskType, len(serverVolumeCounts), balanceConfig.MinServerCount)
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Seed adjustments from existing pending/assigned balance tasks so that
|
||||
// effectiveCounts reflects in-flight moves and prevents over-scheduling.
|
||||
var adjustments map[string]int
|
||||
if clusterInfo.ActiveTopology != nil {
|
||||
adjustments = clusterInfo.ActiveTopology.GetTaskServerAdjustments(topology.TaskTypeBalance)
|
||||
}
|
||||
if adjustments == nil {
|
||||
adjustments = make(map[string]int)
|
||||
}
|
||||
// Servers where we can no longer find eligible volumes or plan destinations
|
||||
exhaustedServers := make(map[string]bool)
|
||||
|
||||
// Sort servers for deterministic iteration and tie-breaking
|
||||
sortedServers := make([]string, 0, len(serverVolumeCounts))
|
||||
for server := range serverVolumeCounts {
|
||||
sortedServers = append(sortedServers, server)
|
||||
}
|
||||
sort.Strings(sortedServers)
|
||||
|
||||
// Pre-index volumes by server with cursors to avoid O(maxResults * volumes) scanning.
|
||||
// Sort each server's volumes by VolumeID for deterministic selection.
|
||||
volumesByServer := make(map[string][]*types.VolumeHealthMetrics, len(serverVolumeCounts))
|
||||
for _, metric := range diskMetrics {
|
||||
volumesByServer[metric.Server] = append(volumesByServer[metric.Server], metric)
|
||||
}
|
||||
for _, vols := range volumesByServer {
|
||||
sort.Slice(vols, func(i, j int) bool {
|
||||
return vols[i].VolumeID < vols[j].VolumeID
|
||||
})
|
||||
}
|
||||
serverCursors := make(map[string]int, len(serverVolumeCounts))
|
||||
|
||||
var results []*types.TaskDetectionResult
|
||||
balanced := false
|
||||
|
||||
// Decide upfront whether all servers have MaxVolumeCount info.
|
||||
// If any server is missing it, fall back to raw counts for ALL servers
|
||||
// to avoid mixing utilization ratios (0.0–1.0) with raw counts.
|
||||
allServersHaveMaxInfo := true
|
||||
for _, server := range sortedServers {
|
||||
if maxVol, ok := serverMaxVolumes[server]; !ok || maxVol <= 0 {
|
||||
allServersHaveMaxInfo = false
|
||||
glog.V(1).Infof("BALANCE [%s]: Server %s is missing MaxVolumeCount info, falling back to raw volume counts for balancing", diskType, server)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
var serverUtilization func(server string, effectiveCount int) float64
|
||||
if allServersHaveMaxInfo {
|
||||
serverUtilization = func(server string, effectiveCount int) float64 {
|
||||
return float64(effectiveCount) / float64(serverMaxVolumes[server])
|
||||
}
|
||||
} else {
|
||||
serverUtilization = func(_ string, effectiveCount int) float64 {
|
||||
return float64(effectiveCount)
|
||||
}
|
||||
}
|
||||
|
||||
// plannedBytes tracks data already routed to each destination earlier in this
|
||||
// detection cycle, so the disk-fullness gate sees a target fill up as moves are
|
||||
// planned instead of only its heartbeat-time free space (the shell equivalent
|
||||
// is adjustAfterMove decrementing DiskFreeBytes).
|
||||
plannedBytes := make(map[string]uint64)
|
||||
// destinationFull excludes servers a projected move would push over, for the cycle.
|
||||
destinationFull := make(map[string]bool)
|
||||
|
||||
// destinationDiskTooFull reports whether landing incomingBytes on server would put
|
||||
// its disk at/over the mark, net of bytes already planned this cycle; unreported
|
||||
// disks are never gated.
|
||||
destinationDiskTooFull := func(server string, incomingBytes uint64) bool {
|
||||
free := serverDiskFreeBytes[server]
|
||||
if planned := plannedBytes[server]; planned < free {
|
||||
free -= planned
|
||||
} else {
|
||||
free = 0
|
||||
}
|
||||
return balancer.DiskTooFullAfter(serverDiskTotalBytes[server], free, incomingBytes, balancer.DefaultMaxDiskUsagePercent)
|
||||
}
|
||||
|
||||
for len(results) < maxResults {
|
||||
// Compute effective volume counts with adjustments from planned moves
|
||||
effectiveCounts := make(map[string]int, len(serverVolumeCounts))
|
||||
for server, count := range serverVolumeCounts {
|
||||
effective := count + adjustments[server]
|
||||
if effective < 0 {
|
||||
effective = 0
|
||||
}
|
||||
effectiveCounts[server] = effective
|
||||
}
|
||||
|
||||
// Find the most and least utilized servers using utilization ratio
|
||||
// (volumes / maxVolumes) so that servers with higher capacity are
|
||||
// expected to hold proportionally more volumes.
|
||||
maxUtilization := -1.0
|
||||
minUtilization := math.Inf(1)
|
||||
maxServer := ""
|
||||
minServer := ""
|
||||
|
||||
for _, server := range sortedServers {
|
||||
count := effectiveCounts[server]
|
||||
util := serverUtilization(server, count)
|
||||
// Min is the emptiest server that can actually receive a volume, so a
|
||||
// physically full server (whose over-set maxVolumeCount makes its slot
|
||||
// utilization look low) is never chosen as the destination.
|
||||
if !destinationFull[server] && !destinationDiskTooFull(server, 0) && util < minUtilization {
|
||||
minUtilization = util
|
||||
minServer = server
|
||||
}
|
||||
// Max is only among non-exhausted servers since we can only move from them
|
||||
if exhaustedServers[server] {
|
||||
continue
|
||||
}
|
||||
if util > maxUtilization {
|
||||
maxUtilization = util
|
||||
maxServer = server
|
||||
}
|
||||
}
|
||||
|
||||
if maxServer == "" {
|
||||
// All servers exhausted
|
||||
glog.V(1).Infof("BALANCE [%s]: All overloaded servers exhausted after %d task(s)", diskType, len(results))
|
||||
break
|
||||
}
|
||||
|
||||
if minServer == "" {
|
||||
// Every candidate destination is at/above the physical disk high-water
|
||||
// mark, so no move can safely improve balance.
|
||||
glog.V(1).Infof("BALANCE [%s]: No eligible destination - all candidates at/above %d%% disk usage after %d task(s)", diskType, balancer.DefaultMaxDiskUsagePercent, len(results))
|
||||
break
|
||||
}
|
||||
|
||||
// Check if utilization imbalance exceeds threshold.
|
||||
// imbalanceRatio is the difference between the most and least utilized
|
||||
// servers, expressed as a fraction of mean utilization.
|
||||
avgUtilization := (maxUtilization + minUtilization) / 2.0
|
||||
var imbalanceRatio float64
|
||||
if avgUtilization > 0 {
|
||||
imbalanceRatio = (maxUtilization - minUtilization) / avgUtilization
|
||||
}
|
||||
if imbalanceRatio <= balanceConfig.ImbalanceThreshold {
|
||||
if len(results) == 0 {
|
||||
glog.Infof("BALANCE [%s]: No tasks created - cluster well balanced. Imbalance=%.1f%% (threshold=%.1f%%). MaxUtil=%.1f%% on %s, MinUtil=%.1f%% on %s",
|
||||
diskType, imbalanceRatio*100, balanceConfig.ImbalanceThreshold*100, maxUtilization*100, maxServer, minUtilization*100, minServer)
|
||||
} else {
|
||||
glog.Infof("BALANCE [%s]: Created %d task(s), cluster now balanced. Imbalance=%.1f%% (threshold=%.1f%%)",
|
||||
diskType, len(results), imbalanceRatio*100, balanceConfig.ImbalanceThreshold*100)
|
||||
}
|
||||
balanced = true
|
||||
break
|
||||
}
|
||||
|
||||
// When the global max and min effective counts differ by at most 1,
|
||||
// no single move can improve balance — it would just swap which server
|
||||
// is min vs max. Stop here to avoid infinite oscillation when the
|
||||
// threshold is unachievable (e.g., 11 vols across 4 servers: best is
|
||||
// 3/3/3/2, imbalance=36%). We scan ALL servers' effective counts so the
|
||||
// check works regardless of whether utilization or raw counts are used.
|
||||
globalMaxCount, globalMinCount := 0, math.MaxInt
|
||||
for _, c := range effectiveCounts {
|
||||
if c > globalMaxCount {
|
||||
globalMaxCount = c
|
||||
}
|
||||
if c < globalMinCount {
|
||||
globalMinCount = c
|
||||
}
|
||||
}
|
||||
if globalMaxCount-globalMinCount <= 1 {
|
||||
if len(results) == 0 {
|
||||
glog.Infof("BALANCE [%s]: No tasks created - cluster as balanced as possible. Imbalance=%.1f%% (threshold=%.1f%%), but max-min diff is %d",
|
||||
diskType, imbalanceRatio*100, balanceConfig.ImbalanceThreshold*100, globalMaxCount-globalMinCount)
|
||||
} else {
|
||||
glog.Infof("BALANCE [%s]: Created %d task(s), cluster as balanced as possible. Imbalance=%.1f%% (threshold=%.1f%%), max-min diff=%d",
|
||||
diskType, len(results), imbalanceRatio*100, balanceConfig.ImbalanceThreshold*100, globalMaxCount-globalMinCount)
|
||||
}
|
||||
balanced = true
|
||||
break
|
||||
}
|
||||
|
||||
// Per-move convergence guard: mirrors weed/shell/command_volume_balance.go
|
||||
// to prevent oscillation and destination overshoot when MaxVolumeCount
|
||||
// values are heterogeneous. Without this guard, the greedy max→min
|
||||
// algorithm can schedule moves that flip which server is most-utilized
|
||||
// (source becomes min, destination becomes max), producing A→B, B→A
|
||||
// oscillation within a single detection cycle.
|
||||
//
|
||||
// Check: after the move, the destination's utilization must not strictly
|
||||
// exceed the source's utilization. If it would, no single move can
|
||||
// improve balance — stop here. This also handles heterogeneous capacity
|
||||
// correctly by comparing post-move utilization ratios rather than raw
|
||||
// counts. The integer discretization is handled automatically: when
|
||||
// counts cannot match the ideal exactly, the check still admits moves
|
||||
// that reduce the max/min gap without flipping it.
|
||||
maxCap := serverMaxVolumes[maxServer]
|
||||
minCap := serverMaxVolumes[minServer]
|
||||
if allServersHaveMaxInfo && maxCap > 0 && minCap > 0 {
|
||||
newSrcUtil := float64(effectiveCounts[maxServer]-1) / float64(maxCap)
|
||||
newDstUtil := float64(effectiveCounts[minServer]+1) / float64(minCap)
|
||||
if newDstUtil > newSrcUtil {
|
||||
if len(results) == 0 {
|
||||
glog.Infof("BALANCE [%s]: No tasks created - no beneficial move available. After move, dest %s util would be %.1f%% vs source %s util %.1f%%",
|
||||
diskType, minServer, newDstUtil*100, maxServer, newSrcUtil*100)
|
||||
} else {
|
||||
glog.Infof("BALANCE [%s]: Created %d task(s), no more beneficial moves available. After move, dest %s util would be %.1f%% vs source %s util %.1f%%",
|
||||
diskType, len(results), minServer, newDstUtil*100, maxServer, newSrcUtil*100)
|
||||
}
|
||||
balanced = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Select a volume from the overloaded server using per-server cursor
|
||||
var selectedVolume *types.VolumeHealthMetrics
|
||||
serverVols := volumesByServer[maxServer]
|
||||
cursor := serverCursors[maxServer]
|
||||
for cursor < len(serverVols) {
|
||||
metric := serverVols[cursor]
|
||||
cursor++
|
||||
// Skip volumes that already have a task in ActiveTopology
|
||||
if clusterInfo.ActiveTopology != nil && clusterInfo.ActiveTopology.HasAnyTask(metric.VolumeID) {
|
||||
continue
|
||||
}
|
||||
selectedVolume = metric
|
||||
break
|
||||
}
|
||||
serverCursors[maxServer] = cursor
|
||||
|
||||
if selectedVolume == nil {
|
||||
glog.V(1).Infof("BALANCE [%s]: No more eligible volumes on overloaded server %s, trying other servers", diskType, maxServer)
|
||||
exhaustedServers[maxServer] = true
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip a destination this specific volume would push over the mark, and
|
||||
// re-pick, rather than overshoot.
|
||||
if destinationDiskTooFull(minServer, uint64(selectedVolume.Size)) {
|
||||
glog.V(1).Infof("BALANCE [%s]: skip destination %s: volume %d (%d bytes) would cross %d%% disk usage",
|
||||
diskType, minServer, selectedVolume.VolumeID, selectedVolume.Size, balancer.DefaultMaxDiskUsagePercent)
|
||||
destinationFull[minServer] = true
|
||||
serverCursors[maxServer]-- // retry this volume against another destination
|
||||
continue
|
||||
}
|
||||
|
||||
// Create task targeting minServer — the greedy algorithm's natural choice.
|
||||
// Using minServer instead of letting planBalanceDestination independently
|
||||
// pick a destination ensures that the detection loop's effective counts
|
||||
// and the destination selection stay in sync. Without this, the topology's
|
||||
// LoadCount-based scoring can diverge from the adjustment-based effective
|
||||
// counts, causing moves to pile onto one server or oscillate (A→B, B→A).
|
||||
//
|
||||
// Constrain the destination (including the score-based fallback inside
|
||||
// createBalanceTask) to servers whose physical disk is not near full;
|
||||
// sources are unaffected, so a full server can still be drained.
|
||||
eligibleTargets := make(map[string]int, len(serverVolumeCounts))
|
||||
for s, c := range serverVolumeCounts {
|
||||
if !destinationFull[s] && !destinationDiskTooFull(s, 0) {
|
||||
eligibleTargets[s] = c
|
||||
}
|
||||
}
|
||||
task, destServerID := createBalanceTask(diskType, selectedVolume, clusterInfo, minServer, eligibleTargets, balanceConfig.IoBytePerSecond)
|
||||
if task == nil {
|
||||
glog.V(1).Infof("BALANCE [%s]: Cannot plan task for volume %d on server %s, trying next volume", diskType, selectedVolume.VolumeID, maxServer)
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, task)
|
||||
|
||||
// Adjust effective counts for the next iteration.
|
||||
adjustments[maxServer]--
|
||||
if destServerID != "" {
|
||||
adjustments[destServerID]++
|
||||
// Charge the moved volume's bytes to the destination so the disk-fullness
|
||||
// gate sees it fill up over the course of this cycle.
|
||||
plannedBytes[destServerID] += uint64(selectedVolume.Size)
|
||||
// If the destination server wasn't in serverVolumeCounts (e.g., a
|
||||
// server with 0 volumes not seeded from topology), add it so
|
||||
// subsequent iterations include it in effective/average/min/max.
|
||||
if _, exists := serverVolumeCounts[destServerID]; !exists {
|
||||
serverVolumeCounts[destServerID] = 0
|
||||
sortedServers = append(sortedServers, destServerID)
|
||||
sort.Strings(sortedServers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Truncated only if we hit maxResults and detection didn't naturally finish
|
||||
truncated := len(results) >= maxResults && !balanced
|
||||
return results, truncated
|
||||
}
|
||||
|
||||
// createBalanceTask creates a single balance task for the selected volume.
|
||||
// targetServer is the server ID chosen by the detection loop's greedy algorithm.
|
||||
// Returns (nil, "") if destination planning fails.
|
||||
// On success, returns the task result and the canonical destination server ID.
|
||||
// allowedServers is the set of servers that passed DC/rack/node filtering in
|
||||
// the detection loop. When non-empty, the fallback destination planner is
|
||||
// checked against this set so that filter scope cannot leak.
|
||||
func createBalanceTask(diskType string, selectedVolume *types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo, targetServer string, allowedServers map[string]int, ioBytePerSecond int64) (*types.TaskDetectionResult, string) {
|
||||
taskID := fmt.Sprintf("balance_vol_%d_%d", selectedVolume.VolumeID, time.Now().UnixNano())
|
||||
|
||||
task := &types.TaskDetectionResult{
|
||||
TaskID: taskID,
|
||||
TaskType: types.TaskTypeBalance,
|
||||
VolumeID: selectedVolume.VolumeID,
|
||||
Server: selectedVolume.Server,
|
||||
Collection: selectedVolume.Collection,
|
||||
Priority: types.TaskPriorityNormal,
|
||||
Reason: fmt.Sprintf("Cluster imbalance detected for %s disk type",
|
||||
diskType),
|
||||
ScheduleAt: time.Now(),
|
||||
}
|
||||
|
||||
// Plan destination if ActiveTopology is available
|
||||
if clusterInfo.ActiveTopology == nil {
|
||||
glog.Warningf("No ActiveTopology available for destination planning in balance detection")
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
// Parse replica placement once for use in both scoring and validation.
|
||||
var volumeRP *super_block.ReplicaPlacement
|
||||
if selectedVolume.ExpectedReplicas > 0 && selectedVolume.ExpectedReplicas <= 255 {
|
||||
if parsed, rpErr := super_block.NewReplicaPlacementFromByte(byte(selectedVolume.ExpectedReplicas)); rpErr == nil && parsed.HasReplication() {
|
||||
volumeRP = parsed
|
||||
}
|
||||
}
|
||||
|
||||
var replicas []types.ReplicaLocation
|
||||
if clusterInfo.VolumeReplicaMap != nil {
|
||||
replicas = clusterInfo.VolumeReplicaMap[selectedVolume.VolumeID]
|
||||
if volumeRP != nil && len(replicas) == 0 {
|
||||
glog.V(1).Infof("BALANCE [%s]: No replica locations found for volume %d, skipping placement validation",
|
||||
diskType, selectedVolume.VolumeID)
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the target server chosen by the detection loop's effective counts.
|
||||
// This keeps destination selection in sync with the greedy algorithm rather
|
||||
// than relying on topology LoadCount which can diverge across iterations.
|
||||
destinationPlan, err := resolveBalanceDestination(clusterInfo.ActiveTopology, selectedVolume, targetServer)
|
||||
if err != nil {
|
||||
// Fall back to score-based planning if the preferred target can't be resolved
|
||||
glog.V(1).Infof("BALANCE [%s]: Cannot resolve target %s for volume %d, falling back to score-based planning: %v",
|
||||
diskType, targetServer, selectedVolume.VolumeID, err)
|
||||
destinationPlan, err = planBalanceDestination(clusterInfo.ActiveTopology, selectedVolume, volumeRP, replicas, allowedServers)
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to plan balance destination for volume %d: %v", selectedVolume.VolumeID, err)
|
||||
return nil, ""
|
||||
}
|
||||
}
|
||||
|
||||
// Verify the resolved destination. If it falls outside the filtered scope
|
||||
// or violates replica placement, fall back to the score-based planner and
|
||||
// pick the best candidate that is actually valid.
|
||||
if !isValidBalanceDestination(destinationPlan, allowedServers, volumeRP, replicas, selectedVolume.Server) {
|
||||
switch {
|
||||
case destinationPlan == nil:
|
||||
glog.V(1).Infof("BALANCE [%s]: Planned destination for volume %d is nil, falling back",
|
||||
diskType, selectedVolume.VolumeID)
|
||||
case !isAllowedBalanceTarget(destinationPlan.TargetNode, allowedServers):
|
||||
glog.V(1).Infof("BALANCE [%s]: Planned destination %s for volume %d is outside filtered scope, falling back",
|
||||
diskType, destinationPlan.TargetNode, selectedVolume.VolumeID)
|
||||
case volumeRP != nil && len(replicas) > 0:
|
||||
glog.V(1).Infof("BALANCE [%s]: Destination %s violates replica placement for volume %d (rp=%03d), falling back",
|
||||
diskType, destinationPlan.TargetNode, selectedVolume.VolumeID, selectedVolume.ExpectedReplicas)
|
||||
}
|
||||
|
||||
destinationPlan, err = planBalanceDestination(clusterInfo.ActiveTopology, selectedVolume, volumeRP, replicas, allowedServers)
|
||||
if err != nil {
|
||||
glog.Warningf("BALANCE [%s]: Failed to plan fallback destination for volume %d: %v", diskType, selectedVolume.VolumeID, err)
|
||||
return nil, ""
|
||||
}
|
||||
if !isValidBalanceDestination(destinationPlan, allowedServers, volumeRP, replicas, selectedVolume.Server) {
|
||||
if destinationPlan == nil {
|
||||
glog.V(1).Infof("BALANCE [%s]: Fallback destination for volume %d is nil",
|
||||
diskType, selectedVolume.VolumeID)
|
||||
} else {
|
||||
glog.V(1).Infof("BALANCE [%s]: Fallback destination %s is not valid for volume %d",
|
||||
diskType, destinationPlan.TargetNode, selectedVolume.VolumeID)
|
||||
}
|
||||
return nil, ""
|
||||
}
|
||||
}
|
||||
|
||||
// Find the actual disk containing the volume on the source server
|
||||
sourceDisk, found := base.FindVolumeDisk(clusterInfo.ActiveTopology, selectedVolume.VolumeID, selectedVolume.Collection, selectedVolume.Server)
|
||||
if !found {
|
||||
glog.Warningf("BALANCE [%s]: Could not find volume %d (collection: %s) on source server %s - unable to create balance task",
|
||||
diskType, selectedVolume.VolumeID, selectedVolume.Collection, selectedVolume.Server)
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
// Update reason with full details now that we have destination info
|
||||
task.Reason = fmt.Sprintf("Cluster imbalance detected for %s: move volume %d from %s to %s",
|
||||
diskType, selectedVolume.VolumeID, selectedVolume.Server, destinationPlan.TargetNode)
|
||||
|
||||
// Create typed parameters with unified source and target information
|
||||
task.TypedParams = &worker_pb.TaskParams{
|
||||
TaskId: taskID,
|
||||
VolumeId: selectedVolume.VolumeID,
|
||||
Collection: selectedVolume.Collection,
|
||||
VolumeSize: selectedVolume.Size,
|
||||
|
||||
Sources: []*worker_pb.TaskSource{
|
||||
{
|
||||
Node: selectedVolume.ServerAddress,
|
||||
DiskId: sourceDisk,
|
||||
VolumeId: selectedVolume.VolumeID,
|
||||
EstimatedSize: selectedVolume.Size,
|
||||
DataCenter: selectedVolume.DataCenter,
|
||||
Rack: selectedVolume.Rack,
|
||||
},
|
||||
},
|
||||
Targets: []*worker_pb.TaskTarget{
|
||||
{
|
||||
Node: destinationPlan.TargetAddress,
|
||||
DiskId: destinationPlan.TargetDisk,
|
||||
VolumeId: selectedVolume.VolumeID,
|
||||
EstimatedSize: destinationPlan.ExpectedSize,
|
||||
DataCenter: destinationPlan.TargetDC,
|
||||
Rack: destinationPlan.TargetRack,
|
||||
},
|
||||
},
|
||||
|
||||
TaskParams: &worker_pb.TaskParams_BalanceParams{
|
||||
BalanceParams: &worker_pb.BalanceTaskParams{
|
||||
ForceMove: false,
|
||||
TimeoutSeconds: 600, // 10 minutes default
|
||||
IoBytePerSecond: ioBytePerSecond,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Planned balance destination for volume %d: %s -> %s",
|
||||
selectedVolume.VolumeID, selectedVolume.Server, destinationPlan.TargetNode)
|
||||
|
||||
// Add pending balance task to ActiveTopology for capacity management
|
||||
targetDisk := destinationPlan.TargetDisk
|
||||
|
||||
err = clusterInfo.ActiveTopology.AddPendingTask(topology.TaskSpec{
|
||||
TaskID: taskID,
|
||||
TaskType: topology.TaskTypeBalance,
|
||||
VolumeID: selectedVolume.VolumeID,
|
||||
VolumeSize: int64(selectedVolume.Size),
|
||||
Sources: []topology.TaskSourceSpec{
|
||||
{ServerID: selectedVolume.Server, DiskID: sourceDisk},
|
||||
},
|
||||
Destinations: []topology.TaskDestinationSpec{
|
||||
{ServerID: destinationPlan.TargetNode, DiskID: targetDisk},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
glog.Warningf("BALANCE [%s]: Failed to add pending task for volume %d: %v", diskType, selectedVolume.VolumeID, err)
|
||||
return nil, ""
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Added pending balance task %s to ActiveTopology for volume %d: %s:%d -> %s:%d",
|
||||
taskID, selectedVolume.VolumeID, selectedVolume.Server, sourceDisk, destinationPlan.TargetNode, targetDisk)
|
||||
|
||||
return task, destinationPlan.TargetNode
|
||||
}
|
||||
|
||||
// resolveBalanceDestination resolves the destination for a balance operation
|
||||
// when the target server is already known (chosen by the detection loop's
|
||||
// effective volume counts). It finds a disk on the target server that can
|
||||
// accept another volume after accounting for ALL pending and assigned tasks
|
||||
// on that disk (via ActiveTopology.GetEffectiveAvailableCapacity) — not the
|
||||
// static VolumeCount from the topology snapshot. This keeps destination
|
||||
// planning consistent with the loop's effective-count bookkeeping and
|
||||
// prevents over-scheduling when multiple moves are planned within the same
|
||||
// detection cycle.
|
||||
func resolveBalanceDestination(activeTopology *topology.ActiveTopology, selectedVolume *types.VolumeHealthMetrics, targetServer string) (*topology.DestinationPlan, error) {
|
||||
nodeDisks := activeTopology.GetNodeDisks(targetServer)
|
||||
if len(nodeDisks) == 0 {
|
||||
return nil, fmt.Errorf("target server %s not found in topology", targetServer)
|
||||
}
|
||||
|
||||
var eligibleDisk *topology.DiskInfo
|
||||
for _, disk := range nodeDisks {
|
||||
if disk == nil || disk.DiskInfo == nil {
|
||||
continue
|
||||
}
|
||||
if disk.DiskType != selectedVolume.DiskType {
|
||||
continue
|
||||
}
|
||||
// Use effective capacity so that prior moves planned in this same
|
||||
// detection cycle (already registered as pending tasks) reduce the
|
||||
// available slots. A disk with VolumeCount << MaxVolumeCount in the
|
||||
// topology snapshot can still be effectively full if many in-flight
|
||||
// tasks already target it.
|
||||
if disk.DiskInfo.MaxVolumeCount > 0 {
|
||||
available := activeTopology.GetEffectiveAvailableCapacity(disk.NodeID, disk.DiskID)
|
||||
if available <= 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
eligibleDisk = disk
|
||||
break
|
||||
}
|
||||
|
||||
if eligibleDisk == nil {
|
||||
return nil, fmt.Errorf("target server %s has no available disk of type %s", targetServer, selectedVolume.DiskType)
|
||||
}
|
||||
|
||||
targetAddress, err := util.ResolveServerAddress(eligibleDisk.NodeID, activeTopology)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve address for target server %s: %v", eligibleDisk.NodeID, err)
|
||||
}
|
||||
return &topology.DestinationPlan{
|
||||
TargetNode: eligibleDisk.NodeID,
|
||||
TargetAddress: targetAddress,
|
||||
TargetDisk: eligibleDisk.DiskID,
|
||||
TargetRack: eligibleDisk.Rack,
|
||||
TargetDC: eligibleDisk.DataCenter,
|
||||
ExpectedSize: selectedVolume.Size,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// planBalanceDestination plans the destination for a balance operation using
|
||||
// score-based selection. Used as a fallback when the preferred target cannot
|
||||
// be resolved, and for single-move scenarios outside the detection loop.
|
||||
// rp may be nil when the volume has no replication constraint. When replica
|
||||
// locations are known, candidates that would violate placement are filtered
|
||||
// out before scoring.
|
||||
func planBalanceDestination(activeTopology *topology.ActiveTopology, selectedVolume *types.VolumeHealthMetrics, rp *super_block.ReplicaPlacement, replicas []types.ReplicaLocation, allowedServers map[string]int) (*topology.DestinationPlan, error) {
|
||||
// Get source node information from topology
|
||||
var sourceRack, sourceDC string
|
||||
|
||||
// Extract rack and DC from topology info
|
||||
topologyInfo := activeTopology.GetTopologyInfo()
|
||||
if topologyInfo != nil {
|
||||
for _, dc := range topologyInfo.DataCenterInfos {
|
||||
for _, rack := range dc.RackInfos {
|
||||
for _, dataNodeInfo := range rack.DataNodeInfos {
|
||||
if dataNodeInfo.Id == selectedVolume.Server {
|
||||
sourceDC = dc.Id
|
||||
sourceRack = rack.Id
|
||||
break
|
||||
}
|
||||
}
|
||||
if sourceRack != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if sourceDC != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get available disks, excluding the source node
|
||||
availableDisks := activeTopology.GetAvailableDisks(topology.TaskTypeBalance, selectedVolume.Server)
|
||||
if len(availableDisks) == 0 {
|
||||
return nil, fmt.Errorf("no available disks for balance operation")
|
||||
}
|
||||
|
||||
// Sort available disks by NodeID then DiskID for deterministic tie-breaking
|
||||
sort.Slice(availableDisks, func(i, j int) bool {
|
||||
if availableDisks[i].NodeID != availableDisks[j].NodeID {
|
||||
return availableDisks[i].NodeID < availableDisks[j].NodeID
|
||||
}
|
||||
return availableDisks[i].DiskID < availableDisks[j].DiskID
|
||||
})
|
||||
|
||||
// Find the best destination disk based on balance criteria
|
||||
var bestDisk *topology.DiskInfo
|
||||
bestScore := math.Inf(-1)
|
||||
|
||||
for _, disk := range availableDisks {
|
||||
// Ensure disk type matches
|
||||
if disk.DiskType != selectedVolume.DiskType {
|
||||
continue
|
||||
}
|
||||
if !isAllowedBalanceTarget(disk.NodeID, allowedServers) {
|
||||
continue
|
||||
}
|
||||
if rp != nil && len(replicas) > 0 {
|
||||
target := types.ReplicaLocation{
|
||||
DataCenter: disk.DataCenter,
|
||||
Rack: disk.Rack,
|
||||
NodeID: disk.NodeID,
|
||||
Host: hostFromAddress(disk.Address, disk.NodeID),
|
||||
}
|
||||
if !IsGoodMove(rp, replicas, selectedVolume.Server, target) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
score := calculateBalanceScore(disk, sourceRack, sourceDC, selectedVolume.Size, rp)
|
||||
if score > bestScore {
|
||||
bestScore = score
|
||||
bestDisk = disk
|
||||
}
|
||||
}
|
||||
|
||||
if bestDisk == nil {
|
||||
return nil, fmt.Errorf("no suitable destination found for balance operation")
|
||||
}
|
||||
|
||||
// Get the target server address
|
||||
targetAddress, err := util.ResolveServerAddress(bestDisk.NodeID, activeTopology)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve address for target server %s: %v", bestDisk.NodeID, err)
|
||||
}
|
||||
|
||||
return &topology.DestinationPlan{
|
||||
TargetNode: bestDisk.NodeID,
|
||||
TargetAddress: targetAddress,
|
||||
TargetDisk: bestDisk.DiskID,
|
||||
TargetRack: bestDisk.Rack,
|
||||
TargetDC: bestDisk.DataCenter,
|
||||
ExpectedSize: selectedVolume.Size,
|
||||
PlacementScore: bestScore,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// calculateBalanceScore calculates placement score for balance operations.
|
||||
// LoadCount reflects pending+assigned tasks on the disk, so we factor it into
|
||||
// the utilization estimate to avoid stacking multiple moves onto the same target.
|
||||
// rp may be nil when the volume has no replication constraint; in that case the
|
||||
// scorer defaults to preferring cross-rack/DC distribution.
|
||||
func calculateBalanceScore(disk *topology.DiskInfo, sourceRack, sourceDC string, volumeSize uint64, rp *super_block.ReplicaPlacement) float64 {
|
||||
if disk.DiskInfo == nil {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
score := 0.0
|
||||
|
||||
// Prefer disks with lower effective volume count (current + pending moves).
|
||||
// LoadCount is included so that disks already targeted by planned moves
|
||||
// appear more utilized, naturally spreading work across targets.
|
||||
if disk.DiskInfo.MaxVolumeCount > 0 {
|
||||
effectiveVolumeCount := float64(disk.DiskInfo.VolumeCount) + float64(disk.LoadCount)
|
||||
utilization := effectiveVolumeCount / float64(disk.DiskInfo.MaxVolumeCount)
|
||||
score += (1.0 - utilization) * 50.0 // Up to 50 points for low utilization
|
||||
}
|
||||
|
||||
// Rack scoring: respect the replication policy.
|
||||
// If replicas must stay on the same rack (SameRackCount > 0 with no
|
||||
// cross-rack requirement), prefer same-rack destinations. Otherwise
|
||||
// prefer different racks for better distribution.
|
||||
sameRack := disk.Rack == sourceRack
|
||||
if rp != nil && rp.DiffRackCount == 0 && rp.SameRackCount > 0 {
|
||||
if sameRack {
|
||||
score += 30.0
|
||||
}
|
||||
} else {
|
||||
if !sameRack {
|
||||
score += 30.0
|
||||
}
|
||||
}
|
||||
|
||||
// DC scoring: same idea. If the policy requires all copies in one DC,
|
||||
// prefer same-DC destinations. Otherwise prefer different DCs.
|
||||
sameDC := disk.DataCenter == sourceDC
|
||||
if rp != nil && rp.DiffDataCenterCount == 0 && (rp.SameRackCount > 0 || rp.DiffRackCount > 0) {
|
||||
if sameDC {
|
||||
score += 20.0
|
||||
}
|
||||
} else {
|
||||
if !sameDC {
|
||||
score += 20.0
|
||||
}
|
||||
}
|
||||
|
||||
return score
|
||||
}
|
||||
|
||||
func isAllowedBalanceTarget(nodeID string, allowedServers map[string]int) bool {
|
||||
if len(allowedServers) == 0 {
|
||||
return true
|
||||
}
|
||||
_, ok := allowedServers[nodeID]
|
||||
return ok
|
||||
}
|
||||
|
||||
func isValidBalanceDestination(plan *topology.DestinationPlan, allowedServers map[string]int, rp *super_block.ReplicaPlacement, replicas []types.ReplicaLocation, sourceNodeID string) bool {
|
||||
if plan == nil {
|
||||
return false
|
||||
}
|
||||
if !isAllowedBalanceTarget(plan.TargetNode, allowedServers) {
|
||||
return false
|
||||
}
|
||||
if rp == nil || len(replicas) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
target := types.ReplicaLocation{
|
||||
DataCenter: plan.TargetDC,
|
||||
Rack: plan.TargetRack,
|
||||
NodeID: plan.TargetNode,
|
||||
Host: hostFromAddress(plan.TargetAddress, plan.TargetNode),
|
||||
}
|
||||
return IsGoodMove(rp, replicas, sourceNodeID, target)
|
||||
}
|
||||
|
||||
// parseCSVSet splits a comma-separated string into a set of trimmed, non-empty values.
|
||||
513
weed/worker/tasks/balance/detection_integration_test.go
Normal file
513
weed/worker/tasks/balance/detection_integration_test.go
Normal file
@@ -0,0 +1,513 @@
|
||||
package balance
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
const (
|
||||
fullnessThreshold = 1.01
|
||||
stateActive = "ACTIVE"
|
||||
stateFull = "FULL"
|
||||
)
|
||||
|
||||
// filterByState filters metrics by volume state for testing.
|
||||
func filterByState(metrics []*types.VolumeHealthMetrics, state string) []*types.VolumeHealthMetrics {
|
||||
if state != stateActive && state != stateFull {
|
||||
return metrics
|
||||
}
|
||||
var out []*types.VolumeHealthMetrics
|
||||
for _, m := range metrics {
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
if state == stateActive && m.FullnessRatio < fullnessThreshold {
|
||||
out = append(out, m)
|
||||
}
|
||||
if state == stateFull && m.FullnessRatio >= fullnessThreshold {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Integration tests that exercise multiple features together:
|
||||
// DC/rack/node filters, volume state filtering, replica placement validation,
|
||||
// and collection scoping all interacting within a single detection run.
|
||||
|
||||
// makeVolumesWithOptions generates metrics with additional options.
|
||||
type volumeOption func(m *types.VolumeHealthMetrics)
|
||||
|
||||
func withFullness(ratio float64) volumeOption {
|
||||
return func(m *types.VolumeHealthMetrics) { m.FullnessRatio = ratio }
|
||||
}
|
||||
|
||||
func withReplicas(rp int) volumeOption {
|
||||
return func(m *types.VolumeHealthMetrics) { m.ExpectedReplicas = rp }
|
||||
}
|
||||
|
||||
func makeVolumesWith(server, diskType, dc, rack, collection string, volumeIDBase uint32, n int, opts ...volumeOption) []*types.VolumeHealthMetrics {
|
||||
vols := makeVolumes(server, diskType, dc, rack, collection, volumeIDBase, n)
|
||||
for _, v := range vols {
|
||||
for _, opt := range opts {
|
||||
opt(v)
|
||||
}
|
||||
}
|
||||
return vols
|
||||
}
|
||||
|
||||
// buildReplicaMap builds a VolumeReplicaMap from metrics (each metric is one replica location).
|
||||
func buildReplicaMap(metrics []*types.VolumeHealthMetrics) map[uint32][]types.ReplicaLocation {
|
||||
m := make(map[uint32][]types.ReplicaLocation)
|
||||
for _, metric := range metrics {
|
||||
m[metric.VolumeID] = append(m[metric.VolumeID], types.ReplicaLocation{
|
||||
DataCenter: metric.DataCenter,
|
||||
Rack: metric.Rack,
|
||||
NodeID: metric.Server,
|
||||
})
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// TestIntegration_DCFilterWithVolumeState tests that DC filtering and volume
|
||||
// state filtering compose correctly: only ACTIVE volumes in the specified DC
|
||||
// participate in balancing.
|
||||
func TestIntegration_DCFilterWithVolumeState(t *testing.T) {
|
||||
servers := []serverSpec{
|
||||
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
|
||||
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack1"},
|
||||
{id: "node-c", diskType: "hdd", diskID: 3, dc: "dc2", rack: "rack1"},
|
||||
}
|
||||
|
||||
var allMetrics []*types.VolumeHealthMetrics
|
||||
// dc1: node-a has 40 active volumes, node-b has 10 active volumes
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 1, 40, withFullness(0.5))...)
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-b", "hdd", "dc1", "rack1", "c1", 100, 10, withFullness(0.5))...)
|
||||
// dc1: node-a also has 20 FULL volumes that should be excluded
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 200, 20, withFullness(1.5))...)
|
||||
// dc2: node-c has 50 active volumes (should be excluded by DC filter)
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-c", "hdd", "dc2", "rack1", "c1", 300, 50, withFullness(0.5))...)
|
||||
|
||||
// Apply volume state filter (ACTIVE only) first
|
||||
activeMetrics := filterByState(allMetrics, stateActive)
|
||||
// Then apply DC filter
|
||||
dcMetrics := make([]*types.VolumeHealthMetrics, 0)
|
||||
for _, m := range activeMetrics {
|
||||
if m.DataCenter == "dc1" {
|
||||
dcMetrics = append(dcMetrics, m)
|
||||
}
|
||||
}
|
||||
|
||||
at := buildTopology(servers, allMetrics)
|
||||
clusterInfo := &types.ClusterInfo{ActiveTopology: at}
|
||||
|
||||
conf := defaultConf()
|
||||
conf.DataCenterFilter = "dc1"
|
||||
|
||||
tasks, _, err := Detection(dcMetrics, clusterInfo, conf, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("Detection failed: %v", err)
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
t.Fatal("Expected balance tasks for 40/10 active-only imbalance in dc1, got 0")
|
||||
}
|
||||
|
||||
for _, task := range tasks {
|
||||
if task.Server == "node-c" {
|
||||
t.Error("node-c (dc2) should not be a source")
|
||||
}
|
||||
if task.TypedParams != nil {
|
||||
for _, tgt := range task.TypedParams.Targets {
|
||||
if strings.Contains(tgt.Node, "node-c") {
|
||||
t.Error("node-c (dc2) should not be a target")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Verify convergence uses only the 50 active dc1 volumes (40+10),
|
||||
// not the 20 full volumes
|
||||
effective := computeEffectiveCounts(servers[:2], dcMetrics, tasks)
|
||||
total := 0
|
||||
maxC, minC := 0, len(dcMetrics)
|
||||
for _, c := range effective {
|
||||
total += c
|
||||
if c > maxC {
|
||||
maxC = c
|
||||
}
|
||||
if c < minC {
|
||||
minC = c
|
||||
}
|
||||
}
|
||||
if total != 50 {
|
||||
t.Errorf("Expected 50 total active volumes in dc1, got %d", total)
|
||||
}
|
||||
t.Logf("DC+state filter: %d tasks, effective=%v", len(tasks), effective)
|
||||
}
|
||||
|
||||
// TestIntegration_NodeFilterWithCollections tests that node filtering works
|
||||
// correctly when volumes span multiple collections.
|
||||
func TestIntegration_NodeFilterWithCollections(t *testing.T) {
|
||||
servers := []serverSpec{
|
||||
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
|
||||
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack1"},
|
||||
{id: "node-c", diskType: "hdd", diskID: 3, dc: "dc1", rack: "rack1"},
|
||||
}
|
||||
|
||||
var allMetrics []*types.VolumeHealthMetrics
|
||||
// node-a: 30 "photos" + 20 "videos" = 50 total
|
||||
allMetrics = append(allMetrics, makeVolumes("node-a", "hdd", "dc1", "rack1", "photos", 1, 30)...)
|
||||
allMetrics = append(allMetrics, makeVolumes("node-a", "hdd", "dc1", "rack1", "videos", 100, 20)...)
|
||||
// node-b: 5 "photos" + 5 "videos" = 10 total
|
||||
allMetrics = append(allMetrics, makeVolumes("node-b", "hdd", "dc1", "rack1", "photos", 200, 5)...)
|
||||
allMetrics = append(allMetrics, makeVolumes("node-b", "hdd", "dc1", "rack1", "videos", 300, 5)...)
|
||||
// node-c: 40 volumes (should be excluded by node filter)
|
||||
allMetrics = append(allMetrics, makeVolumes("node-c", "hdd", "dc1", "rack1", "photos", 400, 40)...)
|
||||
|
||||
// Apply node filter
|
||||
filteredMetrics := make([]*types.VolumeHealthMetrics, 0)
|
||||
for _, m := range allMetrics {
|
||||
if m.Server == "node-a" || m.Server == "node-b" {
|
||||
filteredMetrics = append(filteredMetrics, m)
|
||||
}
|
||||
}
|
||||
|
||||
at := buildTopology(servers, allMetrics)
|
||||
clusterInfo := &types.ClusterInfo{ActiveTopology: at}
|
||||
|
||||
conf := defaultConf()
|
||||
conf.NodeFilter = "node-a,node-b"
|
||||
|
||||
tasks, _, err := Detection(filteredMetrics, clusterInfo, conf, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("Detection failed: %v", err)
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
t.Fatal("Expected tasks for 50/10 imbalance within node-a,node-b")
|
||||
}
|
||||
|
||||
// All moves should be between node-a and node-b only
|
||||
for _, task := range tasks {
|
||||
if task.Server != "node-a" && task.Server != "node-b" {
|
||||
t.Errorf("Source %s should be node-a or node-b", task.Server)
|
||||
}
|
||||
if task.TypedParams != nil {
|
||||
for _, tgt := range task.TypedParams.Targets {
|
||||
if !strings.Contains(tgt.Node, "node-a") && !strings.Contains(tgt.Node, "node-b") {
|
||||
t.Errorf("Target %s should be node-a or node-b", tgt.Node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assertNoDuplicateVolumes(t, tasks)
|
||||
t.Logf("Node filter with mixed collections: %d tasks", len(tasks))
|
||||
}
|
||||
|
||||
// TestIntegration_ReplicaPlacementWithDCFilter tests that replica placement
|
||||
// validation prevents moves that would violate replication policy even when
|
||||
// DC filtering restricts the available servers.
|
||||
func TestIntegration_ReplicaPlacementWithDCFilter(t *testing.T) {
|
||||
// Setup: 2 DCs, volumes with rp=100 (1 replica in different DC)
|
||||
// DC filter restricts to dc1 only.
|
||||
// Replicas: each volume has one copy in dc1 and one in dc2.
|
||||
// The balancer should NOT move volumes within dc1 to a node that would
|
||||
// violate the cross-DC placement requirement.
|
||||
servers := []serverSpec{
|
||||
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
|
||||
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack1"},
|
||||
{id: "node-c", diskType: "hdd", diskID: 3, dc: "dc2", rack: "rack1"},
|
||||
}
|
||||
|
||||
// node-a: 30 volumes with rp=100, node-b: 5 volumes with rp=100
|
||||
// Each volume also has a replica on node-c (dc2)
|
||||
var allMetrics []*types.VolumeHealthMetrics
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 1, 30, withReplicas(100))...)
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-b", "hdd", "dc1", "rack1", "c1", 100, 5, withReplicas(100))...)
|
||||
// dc2 replicas (not part of filtered metrics, but in replica map)
|
||||
dc2Replicas := makeVolumesWith("node-c", "hdd", "dc2", "rack1", "c1", 1, 30, withReplicas(100))
|
||||
|
||||
// Build replica map: volumes 1-30 have replicas on node-a AND node-c
|
||||
replicaMap := buildReplicaMap(allMetrics)
|
||||
for _, r := range dc2Replicas {
|
||||
replicaMap[r.VolumeID] = append(replicaMap[r.VolumeID], types.ReplicaLocation{
|
||||
DataCenter: r.DataCenter,
|
||||
Rack: r.Rack,
|
||||
NodeID: r.Server,
|
||||
})
|
||||
}
|
||||
|
||||
// Filter to dc1 only
|
||||
dc1Metrics := make([]*types.VolumeHealthMetrics, 0)
|
||||
for _, m := range allMetrics {
|
||||
if m.DataCenter == "dc1" {
|
||||
dc1Metrics = append(dc1Metrics, m)
|
||||
}
|
||||
}
|
||||
|
||||
at := buildTopology(servers, allMetrics)
|
||||
clusterInfo := &types.ClusterInfo{
|
||||
ActiveTopology: at,
|
||||
VolumeReplicaMap: replicaMap,
|
||||
}
|
||||
|
||||
conf := defaultConf()
|
||||
conf.DataCenterFilter = "dc1"
|
||||
|
||||
tasks, _, err := Detection(dc1Metrics, clusterInfo, conf, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("Detection failed: %v", err)
|
||||
}
|
||||
|
||||
// With rp=100, moving a volume from node-a to node-b is valid because
|
||||
// the cross-DC replica on node-c is preserved. The balancer should
|
||||
// produce moves that keep the dc2 replica intact.
|
||||
if len(tasks) == 0 {
|
||||
t.Fatal("Expected tasks: 30/5 imbalance with valid cross-DC replicas")
|
||||
}
|
||||
|
||||
for _, task := range tasks {
|
||||
// All sources should be from dc1
|
||||
if task.Server != "node-a" && task.Server != "node-b" {
|
||||
t.Errorf("Source %s should be in dc1", task.Server)
|
||||
}
|
||||
// All targets should be in dc1
|
||||
if task.TypedParams != nil {
|
||||
for _, tgt := range task.TypedParams.Targets {
|
||||
if strings.Contains(tgt.Node, "node-c") {
|
||||
t.Errorf("Target should not be node-c (dc2) with dc1 filter")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assertNoDuplicateVolumes(t, tasks)
|
||||
t.Logf("Replica placement + DC filter: %d tasks", len(tasks))
|
||||
}
|
||||
|
||||
// TestIntegration_RackFilterWithReplicaPlacement tests that rack filtering
|
||||
// and replica placement validation (rp=010) work together. With rp=010,
|
||||
// replicas must be on different racks. When rack filtering restricts to one
|
||||
// rack, the balancer should still produce valid moves within that rack for
|
||||
// volumes whose replicas satisfy the cross-rack requirement via nodes outside
|
||||
// the filter.
|
||||
func TestIntegration_RackFilterWithReplicaPlacement(t *testing.T) {
|
||||
servers := []serverSpec{
|
||||
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
|
||||
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack1"},
|
||||
{id: "node-c", diskType: "hdd", diskID: 3, dc: "dc1", rack: "rack2"},
|
||||
}
|
||||
|
||||
// rack1: node-a has 30 volumes, node-b has 5
|
||||
// Each volume also has a replica on node-c (rack2), satisfying rp=010
|
||||
var allMetrics []*types.VolumeHealthMetrics
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 1, 30, withReplicas(10))...)
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-b", "hdd", "dc1", "rack1", "c1", 100, 5, withReplicas(10))...)
|
||||
rack2Replicas := makeVolumesWith("node-c", "hdd", "dc1", "rack2", "c1", 1, 30, withReplicas(10))
|
||||
|
||||
replicaMap := buildReplicaMap(allMetrics)
|
||||
for _, r := range rack2Replicas {
|
||||
replicaMap[r.VolumeID] = append(replicaMap[r.VolumeID], types.ReplicaLocation{
|
||||
DataCenter: r.DataCenter,
|
||||
Rack: r.Rack,
|
||||
NodeID: r.Server,
|
||||
})
|
||||
}
|
||||
|
||||
// Filter to rack1 only
|
||||
rack1Metrics := make([]*types.VolumeHealthMetrics, 0)
|
||||
for _, m := range allMetrics {
|
||||
if m.Rack == "rack1" {
|
||||
rack1Metrics = append(rack1Metrics, m)
|
||||
}
|
||||
}
|
||||
|
||||
at := buildTopology(servers, allMetrics)
|
||||
clusterInfo := &types.ClusterInfo{
|
||||
ActiveTopology: at,
|
||||
VolumeReplicaMap: replicaMap,
|
||||
}
|
||||
|
||||
conf := defaultConf()
|
||||
conf.RackFilter = "rack1"
|
||||
|
||||
tasks, _, err := Detection(rack1Metrics, clusterInfo, conf, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("Detection failed: %v", err)
|
||||
}
|
||||
|
||||
// Moving within rack1 (node-a → node-b) is valid because the cross-rack
|
||||
// replica on node-c (rack2) is preserved.
|
||||
if len(tasks) == 0 {
|
||||
t.Fatal("Expected tasks for 30/5 imbalance within rack1")
|
||||
}
|
||||
|
||||
for _, task := range tasks {
|
||||
if task.Server == "node-c" {
|
||||
t.Error("node-c (rack2) should not be a source with rack1 filter")
|
||||
}
|
||||
if task.TypedParams != nil {
|
||||
for _, tgt := range task.TypedParams.Targets {
|
||||
if strings.Contains(tgt.Node, "node-c") {
|
||||
t.Error("node-c (rack2) should not be a target with rack1 filter")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assertNoDuplicateVolumes(t, tasks)
|
||||
t.Logf("Rack filter + replica placement: %d tasks", len(tasks))
|
||||
}
|
||||
|
||||
// TestIntegration_AllFactors exercises all filtering dimensions simultaneously:
|
||||
// DC filter, rack filter, volume state filter, replica placement validation,
|
||||
// and mixed collections.
|
||||
func TestIntegration_AllFactors(t *testing.T) {
|
||||
servers := []serverSpec{
|
||||
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
|
||||
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack1"},
|
||||
{id: "node-c", diskType: "hdd", diskID: 3, dc: "dc1", rack: "rack2"}, // excluded by rack filter
|
||||
{id: "node-d", diskType: "hdd", diskID: 4, dc: "dc2", rack: "rack1"}, // excluded by DC filter
|
||||
}
|
||||
|
||||
var allMetrics []*types.VolumeHealthMetrics
|
||||
|
||||
// node-a: 25 active "photos" + 10 full "photos" (full excluded by state filter)
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "photos", 1, 25, withFullness(0.5), withReplicas(100))...)
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "photos", 200, 10, withFullness(1.5), withReplicas(100))...)
|
||||
|
||||
// node-b: 5 active "photos"
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-b", "hdd", "dc1", "rack1", "photos", 100, 5, withFullness(0.5), withReplicas(100))...)
|
||||
|
||||
// node-c: 20 volumes in dc1/rack2 (excluded by rack filter)
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-c", "hdd", "dc1", "rack2", "photos", 300, 20, withFullness(0.5))...)
|
||||
|
||||
// node-d: 30 volumes in dc2 (excluded by DC filter, but provides cross-DC replicas)
|
||||
dc2Replicas := makeVolumesWith("node-d", "hdd", "dc2", "rack1", "photos", 1, 25, withFullness(0.5), withReplicas(100))
|
||||
|
||||
// Build replica map: volumes 1-25 have cross-DC replicas on node-d
|
||||
replicaMap := buildReplicaMap(allMetrics)
|
||||
for _, r := range dc2Replicas {
|
||||
replicaMap[r.VolumeID] = append(replicaMap[r.VolumeID], types.ReplicaLocation{
|
||||
DataCenter: r.DataCenter,
|
||||
Rack: r.Rack,
|
||||
NodeID: r.Server,
|
||||
})
|
||||
}
|
||||
|
||||
// Apply all filters: ACTIVE state, dc1, rack1
|
||||
filtered := filterByState(allMetrics, stateActive)
|
||||
var finalMetrics []*types.VolumeHealthMetrics
|
||||
for _, m := range filtered {
|
||||
if m.DataCenter == "dc1" && m.Rack == "rack1" {
|
||||
finalMetrics = append(finalMetrics, m)
|
||||
}
|
||||
}
|
||||
|
||||
at := buildTopology(servers, allMetrics)
|
||||
clusterInfo := &types.ClusterInfo{
|
||||
ActiveTopology: at,
|
||||
VolumeReplicaMap: replicaMap,
|
||||
}
|
||||
|
||||
conf := defaultConf()
|
||||
conf.DataCenterFilter = "dc1"
|
||||
conf.RackFilter = "rack1"
|
||||
|
||||
tasks, _, err := Detection(finalMetrics, clusterInfo, conf, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("Detection failed: %v", err)
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
t.Fatal("Expected tasks for 25/5 active imbalance in dc1/rack1")
|
||||
}
|
||||
|
||||
// Verify all moves stay within dc1/rack1 scope
|
||||
for i, task := range tasks {
|
||||
if task.Server != "node-a" && task.Server != "node-b" {
|
||||
t.Errorf("Task %d: source %s should be node-a or node-b", i, task.Server)
|
||||
}
|
||||
if task.TypedParams != nil {
|
||||
for _, tgt := range task.TypedParams.Targets {
|
||||
node := tgt.Node
|
||||
if !strings.Contains(node, "node-a") && !strings.Contains(node, "node-b") {
|
||||
t.Errorf("Task %d: target %s should be node-a or node-b", i, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assertNoDuplicateVolumes(t, tasks)
|
||||
|
||||
// Verify convergence
|
||||
effective := computeEffectiveCounts(servers[:2], finalMetrics, tasks)
|
||||
total := 0
|
||||
maxC, minC := 0, len(finalMetrics)
|
||||
for _, c := range effective {
|
||||
total += c
|
||||
if c > maxC {
|
||||
maxC = c
|
||||
}
|
||||
if c < minC {
|
||||
minC = c
|
||||
}
|
||||
}
|
||||
|
||||
// Should have balanced the 30 active dc1/rack1 volumes (25+5)
|
||||
if total != 30 {
|
||||
t.Errorf("Expected 30 total filtered volumes, got %d", total)
|
||||
}
|
||||
avg := float64(total) / float64(len(effective))
|
||||
imbalance := float64(maxC-minC) / avg
|
||||
if imbalance > conf.ImbalanceThreshold {
|
||||
t.Errorf("Still imbalanced: effective=%v, imbalance=%.1f%% (threshold=%.1f%%)",
|
||||
effective, imbalance*100, conf.ImbalanceThreshold*100)
|
||||
}
|
||||
|
||||
t.Logf("All factors combined: %d tasks, effective=%v", len(tasks), effective)
|
||||
}
|
||||
|
||||
// TestIntegration_FullVolumesOnlyBalancing verifies that with FULL state filter,
|
||||
// only full volumes participate in balancing.
|
||||
func TestIntegration_FullVolumesOnlyBalancing(t *testing.T) {
|
||||
servers := []serverSpec{
|
||||
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
|
||||
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack1"},
|
||||
}
|
||||
|
||||
var allMetrics []*types.VolumeHealthMetrics
|
||||
// node-a: 10 active + 30 full
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 1, 10, withFullness(0.5))...)
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 100, 30, withFullness(1.5))...)
|
||||
// node-b: 10 active + 5 full
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-b", "hdd", "dc1", "rack1", "c1", 200, 10, withFullness(0.5))...)
|
||||
allMetrics = append(allMetrics, makeVolumesWith("node-b", "hdd", "dc1", "rack1", "c1", 300, 5, withFullness(1.5))...)
|
||||
|
||||
// Filter to FULL only
|
||||
fullMetrics := filterByState(allMetrics, stateFull)
|
||||
|
||||
at := buildTopology(servers, allMetrics)
|
||||
clusterInfo := &types.ClusterInfo{ActiveTopology: at}
|
||||
|
||||
tasks, _, err := Detection(fullMetrics, clusterInfo, defaultConf(), 100)
|
||||
if err != nil {
|
||||
t.Fatalf("Detection failed: %v", err)
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
t.Fatal("Expected tasks for 30/5 full volume imbalance")
|
||||
}
|
||||
|
||||
// Verify only full volumes are moved (IDs 100-129 from node-a, 300-304 from node-b)
|
||||
for _, task := range tasks {
|
||||
vid := task.VolumeID
|
||||
// Active volumes are IDs 1-10 (node-a) and 200-209 (node-b)
|
||||
if (vid >= 1 && vid <= 10) || (vid >= 200 && vid <= 209) {
|
||||
t.Errorf("Task moved active volume %d, should only move full volumes", vid)
|
||||
}
|
||||
}
|
||||
|
||||
assertNoDuplicateVolumes(t, tasks)
|
||||
t.Logf("Full-only balancing: %d tasks", len(tasks))
|
||||
}
|
||||
1487
weed/worker/tasks/balance/detection_test.go
Normal file
1487
weed/worker/tasks/balance/detection_test.go
Normal file
File diff suppressed because it is too large
Load Diff
158
weed/worker/tasks/balance/execution.go
Normal file
158
weed/worker/tasks/balance/execution.go
Normal file
@@ -0,0 +1,158 @@
|
||||
package balance
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// TypedTask implements balance operation with typed protobuf parameters
|
||||
type TypedTask struct {
|
||||
*base.BaseTypedTask
|
||||
|
||||
// Task state from protobuf
|
||||
sourceServer string
|
||||
destNode string
|
||||
volumeID uint32
|
||||
collection string
|
||||
estimatedSize uint64
|
||||
forceMove bool
|
||||
timeoutSeconds int32
|
||||
}
|
||||
|
||||
// NewTypedTask creates a new typed balance task
|
||||
func NewTypedTask() types.TypedTaskInterface {
|
||||
task := &TypedTask{
|
||||
BaseTypedTask: base.NewBaseTypedTask(types.TaskTypeBalance),
|
||||
}
|
||||
return task
|
||||
}
|
||||
|
||||
// ValidateTyped validates the typed parameters for balance task
|
||||
func (t *TypedTask) ValidateTyped(params *worker_pb.TaskParams) error {
|
||||
// Basic validation from base class
|
||||
if err := t.BaseTypedTask.ValidateTyped(params); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check that we have balance-specific parameters
|
||||
balanceParams := params.GetBalanceParams()
|
||||
if balanceParams == nil {
|
||||
return fmt.Errorf("balance_params is required for balance task")
|
||||
}
|
||||
|
||||
// Validate sources and targets
|
||||
if len(params.Sources) == 0 {
|
||||
return fmt.Errorf("at least one source is required for balance task")
|
||||
}
|
||||
if len(params.Targets) == 0 {
|
||||
return fmt.Errorf("at least one target is required for balance task")
|
||||
}
|
||||
|
||||
// Validate that source and target have volume IDs
|
||||
if params.Sources[0].VolumeId == 0 {
|
||||
return fmt.Errorf("source volume_id is required for balance task")
|
||||
}
|
||||
if params.Targets[0].VolumeId == 0 {
|
||||
return fmt.Errorf("target volume_id is required for balance task")
|
||||
}
|
||||
|
||||
// Validate timeout
|
||||
if balanceParams.TimeoutSeconds <= 0 {
|
||||
return fmt.Errorf("timeout_seconds must be greater than 0")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// EstimateTimeTyped estimates the time needed for balance operation based on protobuf parameters
|
||||
func (t *TypedTask) EstimateTimeTyped(params *worker_pb.TaskParams) time.Duration {
|
||||
balanceParams := params.GetBalanceParams()
|
||||
if balanceParams != nil {
|
||||
// Use the timeout from parameters if specified
|
||||
if balanceParams.TimeoutSeconds > 0 {
|
||||
return time.Duration(balanceParams.TimeoutSeconds) * time.Second
|
||||
}
|
||||
}
|
||||
|
||||
// Estimate based on volume size from sources (1 minute per GB)
|
||||
if len(params.Sources) > 0 {
|
||||
source := params.Sources[0]
|
||||
if source.EstimatedSize > 0 {
|
||||
gbSize := source.EstimatedSize / (1024 * 1024 * 1024)
|
||||
return time.Duration(gbSize) * time.Minute
|
||||
}
|
||||
}
|
||||
|
||||
// Default estimation
|
||||
return 10 * time.Minute
|
||||
}
|
||||
|
||||
// ExecuteTyped implements the balance operation with typed parameters
|
||||
func (t *TypedTask) ExecuteTyped(params *worker_pb.TaskParams) error {
|
||||
// Extract basic parameters
|
||||
t.volumeID = params.VolumeId
|
||||
t.collection = params.Collection
|
||||
|
||||
// Ensure sources and targets are present (should be guaranteed by validation)
|
||||
if len(params.Sources) == 0 {
|
||||
return fmt.Errorf("at least one source is required for balance task (ExecuteTyped)")
|
||||
}
|
||||
if len(params.Targets) == 0 {
|
||||
return fmt.Errorf("at least one target is required for balance task (ExecuteTyped)")
|
||||
}
|
||||
|
||||
// Extract source and target information
|
||||
t.sourceServer = params.Sources[0].Node
|
||||
t.estimatedSize = params.Sources[0].EstimatedSize
|
||||
t.destNode = params.Targets[0].Node
|
||||
// Extract balance-specific parameters
|
||||
balanceParams := params.GetBalanceParams()
|
||||
if balanceParams != nil {
|
||||
t.forceMove = balanceParams.ForceMove
|
||||
t.timeoutSeconds = balanceParams.TimeoutSeconds
|
||||
}
|
||||
|
||||
glog.Infof("Starting typed balance task for volume %d: %s -> %s (collection: %s, size: %d bytes)",
|
||||
t.volumeID, t.sourceServer, t.destNode, t.collection, t.estimatedSize)
|
||||
|
||||
// Simulate balance operation with progress updates
|
||||
steps := []struct {
|
||||
name string
|
||||
duration time.Duration
|
||||
progress float64
|
||||
}{
|
||||
{"Analyzing cluster state", 2 * time.Second, 15},
|
||||
{"Verifying destination capacity", 1 * time.Second, 25},
|
||||
{"Starting volume migration", 1 * time.Second, 35},
|
||||
{"Moving volume data", 6 * time.Second, 75},
|
||||
{"Updating cluster metadata", 2 * time.Second, 95},
|
||||
{"Verifying balance completion", 1 * time.Second, 100},
|
||||
}
|
||||
|
||||
for _, step := range steps {
|
||||
if t.IsCancelled() {
|
||||
return fmt.Errorf("balance task cancelled during: %s", step.name)
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Balance task step: %s", step.name)
|
||||
t.SetProgress(step.progress)
|
||||
|
||||
// Simulate work
|
||||
time.Sleep(step.duration)
|
||||
}
|
||||
|
||||
glog.Infof("Typed balance task completed successfully for volume %d: %s -> %s",
|
||||
t.volumeID, t.sourceServer, t.destNode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Register the typed task in the global registry
|
||||
func init() {
|
||||
types.RegisterGlobalTypedTask(types.TaskTypeBalance, NewTypedTask)
|
||||
glog.V(1).Infof("Registered typed balance task")
|
||||
}
|
||||
1516
weed/worker/tasks/balance/plugin_handler.go
Normal file
1516
weed/worker/tasks/balance/plugin_handler.go
Normal file
File diff suppressed because it is too large
Load Diff
828
weed/worker/tasks/balance/plugin_handler_test.go
Normal file
828
weed/worker/tasks/balance/plugin_handler_test.go
Normal file
@@ -0,0 +1,828 @@
|
||||
package balance
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func TestDecodeVolumeBalanceTaskParamsFromPayload(t *testing.T) {
|
||||
expected := &worker_pb.TaskParams{
|
||||
TaskId: "task-1",
|
||||
VolumeId: 42,
|
||||
Collection: "photos",
|
||||
Sources: []*worker_pb.TaskSource{
|
||||
{
|
||||
Node: "10.0.0.1:8080",
|
||||
VolumeId: 42,
|
||||
},
|
||||
},
|
||||
Targets: []*worker_pb.TaskTarget{
|
||||
{
|
||||
Node: "10.0.0.2:8080",
|
||||
VolumeId: 42,
|
||||
},
|
||||
},
|
||||
TaskParams: &worker_pb.TaskParams_BalanceParams{
|
||||
BalanceParams: &worker_pb.BalanceTaskParams{
|
||||
ForceMove: true,
|
||||
TimeoutSeconds: 1200,
|
||||
},
|
||||
},
|
||||
}
|
||||
payload, err := proto.Marshal(expected)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal payload: %v", err)
|
||||
}
|
||||
|
||||
job := &plugin_pb.JobSpec{
|
||||
JobId: "job-from-admin",
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
|
||||
},
|
||||
}
|
||||
|
||||
actual, err := decodeVolumeBalanceTaskParams(job)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeVolumeBalanceTaskParams() err = %v", err)
|
||||
}
|
||||
if !proto.Equal(expected, actual) {
|
||||
t.Fatalf("decoded params mismatch\nexpected: %+v\nactual: %+v", expected, actual)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeVolumeBalanceTaskParamsFallback(t *testing.T) {
|
||||
job := &plugin_pb.JobSpec{
|
||||
JobId: "job-2",
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"volume_id": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7}},
|
||||
"source_server": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "127.0.0.1:8080"}},
|
||||
"target_server": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "127.0.0.2:8080"}},
|
||||
"collection": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "videos"}},
|
||||
},
|
||||
}
|
||||
|
||||
params, err := decodeVolumeBalanceTaskParams(job)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeVolumeBalanceTaskParams() err = %v", err)
|
||||
}
|
||||
if params.TaskId != "job-2" || params.VolumeId != 7 || params.Collection != "videos" {
|
||||
t.Fatalf("unexpected basic params: %+v", params)
|
||||
}
|
||||
if len(params.Sources) != 1 || params.Sources[0].Node != "127.0.0.1:8080" {
|
||||
t.Fatalf("unexpected sources: %+v", params.Sources)
|
||||
}
|
||||
if len(params.Targets) != 1 || params.Targets[0].Node != "127.0.0.2:8080" {
|
||||
t.Fatalf("unexpected targets: %+v", params.Targets)
|
||||
}
|
||||
if params.GetBalanceParams() == nil {
|
||||
t.Fatalf("expected fallback balance params")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveBalanceWorkerConfig(t *testing.T) {
|
||||
values := map[string]*plugin_pb.ConfigValue{
|
||||
"imbalance_threshold": {
|
||||
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.45},
|
||||
},
|
||||
"min_server_count": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 5},
|
||||
},
|
||||
}
|
||||
|
||||
cfg := deriveBalanceWorkerConfig(values)
|
||||
if cfg.TaskConfig.ImbalanceThreshold != 0.45 {
|
||||
t.Fatalf("expected imbalance_threshold 0.45, got %v", cfg.TaskConfig.ImbalanceThreshold)
|
||||
}
|
||||
if cfg.TaskConfig.MinServerCount != 5 {
|
||||
t.Fatalf("expected min_server_count 5, got %d", cfg.TaskConfig.MinServerCount)
|
||||
}
|
||||
// Defaults for batch config when not specified
|
||||
if cfg.MaxConcurrentMoves != defaultMaxConcurrentMoves {
|
||||
t.Fatalf("expected default max_concurrent_moves %d, got %d", defaultMaxConcurrentMoves, cfg.MaxConcurrentMoves)
|
||||
}
|
||||
if cfg.BatchSize != 20 {
|
||||
t.Fatalf("expected default batch_size 20, got %d", cfg.BatchSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveBalanceWorkerConfigBatchFields(t *testing.T) {
|
||||
values := map[string]*plugin_pb.ConfigValue{
|
||||
"max_concurrent_moves": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 10},
|
||||
},
|
||||
"batch_size": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 50},
|
||||
},
|
||||
}
|
||||
|
||||
cfg := deriveBalanceWorkerConfig(values)
|
||||
if cfg.MaxConcurrentMoves != 10 {
|
||||
t.Fatalf("expected max_concurrent_moves 10, got %d", cfg.MaxConcurrentMoves)
|
||||
}
|
||||
if cfg.BatchSize != 50 {
|
||||
t.Fatalf("expected batch_size 50, got %d", cfg.BatchSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveBalanceWorkerConfigBatchClamping(t *testing.T) {
|
||||
values := map[string]*plugin_pb.ConfigValue{
|
||||
"max_concurrent_moves": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 999},
|
||||
},
|
||||
"batch_size": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0},
|
||||
},
|
||||
}
|
||||
|
||||
cfg := deriveBalanceWorkerConfig(values)
|
||||
if cfg.MaxConcurrentMoves != 50 {
|
||||
t.Fatalf("expected max_concurrent_moves clamped to 50, got %d", cfg.MaxConcurrentMoves)
|
||||
}
|
||||
if cfg.BatchSize != 1 {
|
||||
t.Fatalf("expected batch_size clamped to 1, got %d", cfg.BatchSize)
|
||||
}
|
||||
}
|
||||
|
||||
func makeDetectionResult(volumeID uint32, source, target, collection string) *workertypes.TaskDetectionResult {
|
||||
return &workertypes.TaskDetectionResult{
|
||||
TaskID: fmt.Sprintf("balance-%d", volumeID),
|
||||
TaskType: workertypes.TaskTypeBalance,
|
||||
VolumeID: volumeID,
|
||||
Server: source,
|
||||
Collection: collection,
|
||||
Priority: workertypes.TaskPriorityNormal,
|
||||
Reason: "imbalanced",
|
||||
TypedParams: &worker_pb.TaskParams{
|
||||
VolumeId: volumeID,
|
||||
Collection: collection,
|
||||
VolumeSize: 1024,
|
||||
Sources: []*worker_pb.TaskSource{
|
||||
{Node: source, VolumeId: volumeID},
|
||||
},
|
||||
Targets: []*worker_pb.TaskTarget{
|
||||
{Node: target, VolumeId: volumeID},
|
||||
},
|
||||
TaskParams: &worker_pb.TaskParams_BalanceParams{
|
||||
BalanceParams: &worker_pb.BalanceTaskParams{TimeoutSeconds: 600},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildBatchVolumeBalanceProposals_SingleBatch(t *testing.T) {
|
||||
results := []*workertypes.TaskDetectionResult{
|
||||
makeDetectionResult(1, "s1:8080", "t1:8080", "c1"),
|
||||
makeDetectionResult(2, "s2:8080", "t2:8080", "c1"),
|
||||
makeDetectionResult(3, "s1:8080", "t2:8080", "c1"),
|
||||
}
|
||||
|
||||
proposals := buildBatchVolumeBalanceProposals(results, 10, 5)
|
||||
if len(proposals) != 1 {
|
||||
t.Fatalf("expected 1 batch proposal, got %d", len(proposals))
|
||||
}
|
||||
|
||||
p := proposals[0]
|
||||
if p.Labels["batch"] != "true" {
|
||||
t.Fatalf("expected batch label")
|
||||
}
|
||||
if p.Labels["batch_size"] != "3" {
|
||||
t.Fatalf("expected batch_size label '3', got %q", p.Labels["batch_size"])
|
||||
}
|
||||
|
||||
// Decode and verify moves
|
||||
payload := p.Parameters["task_params_pb"].GetBytesValue()
|
||||
if len(payload) == 0 {
|
||||
t.Fatalf("expected task_params_pb payload")
|
||||
}
|
||||
decoded := &worker_pb.TaskParams{}
|
||||
if err := proto.Unmarshal(payload, decoded); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
moves := decoded.GetBalanceParams().GetMoves()
|
||||
if len(moves) != 3 {
|
||||
t.Fatalf("expected 3 moves, got %d", len(moves))
|
||||
}
|
||||
if moves[0].VolumeId != 1 || moves[1].VolumeId != 2 || moves[2].VolumeId != 3 {
|
||||
t.Fatalf("unexpected volume IDs: %v", moves)
|
||||
}
|
||||
if decoded.GetBalanceParams().MaxConcurrentMoves != 5 {
|
||||
t.Fatalf("expected MaxConcurrentMoves 5, got %d", decoded.GetBalanceParams().MaxConcurrentMoves)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildBatchVolumeBalanceProposals_MultipleBatches(t *testing.T) {
|
||||
results := make([]*workertypes.TaskDetectionResult, 5)
|
||||
for i := range results {
|
||||
results[i] = makeDetectionResult(uint32(i+1), "s1:8080", "t1:8080", "c1")
|
||||
}
|
||||
|
||||
proposals := buildBatchVolumeBalanceProposals(results, 2, 3)
|
||||
// 5 results / batch_size 2 = 3 proposals (2, 2, 1)
|
||||
if len(proposals) != 3 {
|
||||
t.Fatalf("expected 3 proposals, got %d", len(proposals))
|
||||
}
|
||||
|
||||
// First two should be batch proposals
|
||||
if proposals[0].Labels["batch"] != "true" {
|
||||
t.Fatalf("first proposal should be batch")
|
||||
}
|
||||
if proposals[1].Labels["batch"] != "true" {
|
||||
t.Fatalf("second proposal should be batch")
|
||||
}
|
||||
// Last one has only 1 result, should fall back to single-move proposal
|
||||
if proposals[2].Labels["batch"] == "true" {
|
||||
t.Fatalf("last proposal with 1 result should be single-move, not batch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildBatchVolumeBalanceProposals_BatchSizeOne(t *testing.T) {
|
||||
results := []*workertypes.TaskDetectionResult{
|
||||
makeDetectionResult(1, "s1:8080", "t1:8080", "c1"),
|
||||
makeDetectionResult(2, "s2:8080", "t2:8080", "c1"),
|
||||
}
|
||||
|
||||
// batch_size=1 should not be called (Detect guards this), but test the function directly
|
||||
proposals := buildBatchVolumeBalanceProposals(results, 1, 5)
|
||||
// Each result becomes its own single-move proposal
|
||||
if len(proposals) != 2 {
|
||||
t.Fatalf("expected 2 proposals, got %d", len(proposals))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeBalanceDescriptorHasBatchFields(t *testing.T) {
|
||||
descriptor := NewVolumeBalanceHandler(nil).Descriptor()
|
||||
if !workerConfigFormHasField(descriptor.WorkerConfigForm, "max_concurrent_moves") {
|
||||
t.Fatalf("expected max_concurrent_moves in worker config form")
|
||||
}
|
||||
if !workerConfigFormHasField(descriptor.WorkerConfigForm, "batch_size") {
|
||||
t.Fatalf("expected batch_size in worker config form")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildVolumeBalanceProposal(t *testing.T) {
|
||||
params := &worker_pb.TaskParams{
|
||||
TaskId: "balance-task-1",
|
||||
VolumeId: 55,
|
||||
Collection: "images",
|
||||
Sources: []*worker_pb.TaskSource{
|
||||
{
|
||||
Node: "source-a:8080",
|
||||
VolumeId: 55,
|
||||
},
|
||||
},
|
||||
Targets: []*worker_pb.TaskTarget{
|
||||
{
|
||||
Node: "target-b:8080",
|
||||
VolumeId: 55,
|
||||
},
|
||||
},
|
||||
TaskParams: &worker_pb.TaskParams_BalanceParams{
|
||||
BalanceParams: &worker_pb.BalanceTaskParams{
|
||||
TimeoutSeconds: 600,
|
||||
},
|
||||
},
|
||||
}
|
||||
result := &workertypes.TaskDetectionResult{
|
||||
TaskID: "balance-task-1",
|
||||
TaskType: workertypes.TaskTypeBalance,
|
||||
VolumeID: 55,
|
||||
Server: "source-a",
|
||||
Collection: "images",
|
||||
Priority: workertypes.TaskPriorityHigh,
|
||||
Reason: "imbalanced load",
|
||||
TypedParams: params,
|
||||
}
|
||||
|
||||
proposal, err := buildVolumeBalanceProposal(result)
|
||||
if err != nil {
|
||||
t.Fatalf("buildVolumeBalanceProposal() err = %v", err)
|
||||
}
|
||||
if proposal.JobType != "volume_balance" {
|
||||
t.Fatalf("unexpected job type %q", proposal.JobType)
|
||||
}
|
||||
if proposal.DedupeKey == "" {
|
||||
t.Fatalf("expected dedupe key")
|
||||
}
|
||||
if proposal.Parameters["task_params_pb"] == nil {
|
||||
t.Fatalf("expected serialized task params")
|
||||
}
|
||||
if proposal.Labels["source_node"] != "source-a:8080" {
|
||||
t.Fatalf("unexpected source label %q", proposal.Labels["source_node"])
|
||||
}
|
||||
if proposal.Labels["target_node"] != "target-b:8080" {
|
||||
t.Fatalf("unexpected target label %q", proposal.Labels["target_node"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeBalanceHandlerRejectsUnsupportedJobType(t *testing.T) {
|
||||
handler := NewVolumeBalanceHandler(nil)
|
||||
err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
|
||||
JobType: "vacuum",
|
||||
}, noopDetectionSender{})
|
||||
if err == nil {
|
||||
t.Fatalf("expected detect job type mismatch error")
|
||||
}
|
||||
|
||||
err = handler.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
|
||||
Job: &plugin_pb.JobSpec{JobId: "job-1", JobType: "vacuum"},
|
||||
}, noopExecutionSender{})
|
||||
if err == nil {
|
||||
t.Fatalf("expected execute job type mismatch error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitVolumeBalanceDetectionDecisionTraceNoTasks(t *testing.T) {
|
||||
sender := &recordingDetectionSender{}
|
||||
config := NewDefaultConfig()
|
||||
config.ImbalanceThreshold = 0.2
|
||||
config.MinServerCount = 2
|
||||
|
||||
metrics := []*workertypes.VolumeHealthMetrics{
|
||||
{VolumeID: 1, Server: "server-a", DiskType: "hdd"},
|
||||
{VolumeID: 2, Server: "server-a", DiskType: "hdd"},
|
||||
{VolumeID: 3, Server: "server-b", DiskType: "hdd"},
|
||||
{VolumeID: 4, Server: "server-b", DiskType: "hdd"},
|
||||
}
|
||||
|
||||
if err := emitVolumeBalanceDetectionDecisionTrace(sender, metrics, nil, config, nil); err != nil {
|
||||
t.Fatalf("emitVolumeBalanceDetectionDecisionTrace error: %v", err)
|
||||
}
|
||||
if len(sender.events) < 2 {
|
||||
t.Fatalf("expected at least 2 detection events, got %d", len(sender.events))
|
||||
}
|
||||
if sender.events[0].Source != plugin_pb.ActivitySource_ACTIVITY_SOURCE_DETECTOR {
|
||||
t.Fatalf("expected detector source, got %v", sender.events[0].Source)
|
||||
}
|
||||
if !strings.Contains(sender.events[0].Message, "BALANCE: No tasks created for 4 volumes") {
|
||||
t.Fatalf("unexpected summary message: %q", sender.events[0].Message)
|
||||
}
|
||||
foundDiskTypeDecision := false
|
||||
for _, event := range sender.events {
|
||||
if strings.Contains(event.Message, "BALANCE [hdd]: No tasks created - cluster well balanced") {
|
||||
foundDiskTypeDecision = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !foundDiskTypeDecision {
|
||||
t.Fatalf("expected per-disk-type decision message")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeBalanceDescriptorOmitsExecutionTuningFields(t *testing.T) {
|
||||
descriptor := NewVolumeBalanceHandler(nil).Descriptor()
|
||||
if descriptor == nil || descriptor.WorkerConfigForm == nil {
|
||||
t.Fatalf("expected worker config form in descriptor")
|
||||
}
|
||||
if workerConfigFormHasField(descriptor.WorkerConfigForm, "timeout_seconds") {
|
||||
t.Fatalf("unexpected timeout_seconds in volume balance worker config form")
|
||||
}
|
||||
if workerConfigFormHasField(descriptor.WorkerConfigForm, "force_move") {
|
||||
t.Fatalf("unexpected force_move in volume balance worker config form")
|
||||
}
|
||||
}
|
||||
|
||||
type recordingExecutionSender struct {
|
||||
mu sync.Mutex
|
||||
progress []*plugin_pb.JobProgressUpdate
|
||||
completed *plugin_pb.JobCompleted
|
||||
}
|
||||
|
||||
func (r *recordingExecutionSender) SendProgress(p *plugin_pb.JobProgressUpdate) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.progress = append(r.progress, proto.Clone(p).(*plugin_pb.JobProgressUpdate))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recordingExecutionSender) SendCompleted(c *plugin_pb.JobCompleted) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.completed = proto.Clone(c).(*plugin_pb.JobCompleted)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestBuildMoveTaskParams(t *testing.T) {
|
||||
move := &worker_pb.BalanceMoveSpec{
|
||||
VolumeId: 42,
|
||||
SourceNode: "10.0.0.1:8080",
|
||||
TargetNode: "10.0.0.2:8080",
|
||||
Collection: "photos",
|
||||
VolumeSize: 1024 * 1024,
|
||||
}
|
||||
|
||||
outerParams := &worker_pb.BalanceTaskParams{
|
||||
ForceMove: true,
|
||||
TimeoutSeconds: 300,
|
||||
}
|
||||
params := buildMoveTaskParams(move, outerParams)
|
||||
if params.VolumeId != 42 {
|
||||
t.Fatalf("expected volume_id 42, got %d", params.VolumeId)
|
||||
}
|
||||
if params.Collection != "photos" {
|
||||
t.Fatalf("expected collection photos, got %s", params.Collection)
|
||||
}
|
||||
if params.VolumeSize != 1024*1024 {
|
||||
t.Fatalf("expected volume_size %d, got %d", 1024*1024, params.VolumeSize)
|
||||
}
|
||||
if len(params.Sources) != 1 || params.Sources[0].Node != "10.0.0.1:8080" {
|
||||
t.Fatalf("unexpected sources: %+v", params.Sources)
|
||||
}
|
||||
if len(params.Targets) != 1 || params.Targets[0].Node != "10.0.0.2:8080" {
|
||||
t.Fatalf("unexpected targets: %+v", params.Targets)
|
||||
}
|
||||
bp := params.GetBalanceParams()
|
||||
if bp == nil {
|
||||
t.Fatalf("expected balance params")
|
||||
}
|
||||
if bp.TimeoutSeconds != 300 {
|
||||
t.Fatalf("expected timeout 300, got %d", bp.TimeoutSeconds)
|
||||
}
|
||||
if !bp.ForceMove {
|
||||
t.Fatalf("expected force_move to be propagated from outer params")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMoveTaskParamsDefaultTimeout(t *testing.T) {
|
||||
move := &worker_pb.BalanceMoveSpec{
|
||||
VolumeId: 1,
|
||||
SourceNode: "a:8080",
|
||||
TargetNode: "b:8080",
|
||||
}
|
||||
params := buildMoveTaskParams(move, nil)
|
||||
if params.GetBalanceParams().TimeoutSeconds != defaultBalanceTimeoutSeconds {
|
||||
t.Fatalf("expected default timeout %d, got %d", defaultBalanceTimeoutSeconds, params.GetBalanceParams().TimeoutSeconds)
|
||||
}
|
||||
if params.GetBalanceParams().ForceMove {
|
||||
t.Fatalf("expected force_move to default to false with nil outer params")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteDispatchesBatchPath(t *testing.T) {
|
||||
// Build a job with batch moves in BalanceTaskParams
|
||||
bp := &worker_pb.BalanceTaskParams{
|
||||
TimeoutSeconds: 60,
|
||||
MaxConcurrentMoves: 2,
|
||||
Moves: []*worker_pb.BalanceMoveSpec{
|
||||
{VolumeId: 1, SourceNode: "s1:8080", TargetNode: "t1:8080", Collection: "c1"},
|
||||
{VolumeId: 2, SourceNode: "s2:8080", TargetNode: "t2:8080", Collection: "c1"},
|
||||
},
|
||||
}
|
||||
taskParams := &worker_pb.TaskParams{
|
||||
TaskId: "batch-1",
|
||||
TaskParams: &worker_pb.TaskParams_BalanceParams{
|
||||
BalanceParams: bp,
|
||||
},
|
||||
}
|
||||
payload, err := proto.Marshal(taskParams)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
|
||||
job := &plugin_pb.JobSpec{
|
||||
JobId: "batch-job-1",
|
||||
JobType: "volume_balance",
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
|
||||
},
|
||||
}
|
||||
|
||||
handler := NewVolumeBalanceHandler(nil)
|
||||
sender := &recordingExecutionSender{}
|
||||
|
||||
// Execute will enter the batch path. It will fail because there's no real gRPC server,
|
||||
// but we verify it sends the assigned progress and eventually a completion.
|
||||
err = handler.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
|
||||
Job: job,
|
||||
}, sender)
|
||||
if err != nil {
|
||||
t.Fatalf("handler.Execute: %v", err)
|
||||
}
|
||||
|
||||
// Expect an error since no real volume servers exist
|
||||
// But verify the batch path was taken by checking the assigned message
|
||||
sender.mu.Lock()
|
||||
defer sender.mu.Unlock()
|
||||
|
||||
if len(sender.progress) == 0 {
|
||||
t.Fatalf("expected progress messages from batch path")
|
||||
}
|
||||
|
||||
// First progress should be "assigned" with batch info
|
||||
first := sender.progress[0]
|
||||
if first.Stage != "assigned" {
|
||||
t.Fatalf("expected first stage 'assigned', got %q", first.Stage)
|
||||
}
|
||||
if !strings.Contains(first.Message, "batch") || !strings.Contains(first.Message, "2 moves") {
|
||||
t.Fatalf("expected batch assigned message, got %q", first.Message)
|
||||
}
|
||||
|
||||
// Should have a completion with failure details (since no servers)
|
||||
if sender.completed == nil {
|
||||
t.Fatalf("expected completion message")
|
||||
}
|
||||
if sender.completed.Success {
|
||||
t.Fatalf("expected failure since no real gRPC servers")
|
||||
}
|
||||
// Should report 0 succeeded, 2 failed
|
||||
if v, ok := sender.completed.Result.OutputValues["failed"]; !ok || v.GetInt64Value() != 2 {
|
||||
t.Fatalf("expected 2 failed moves, got %+v", sender.completed.Result.OutputValues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSingleMovePathUnchanged(t *testing.T) {
|
||||
// Build a single-move job (no batch moves)
|
||||
taskParams := &worker_pb.TaskParams{
|
||||
TaskId: "single-1",
|
||||
VolumeId: 99,
|
||||
Collection: "videos",
|
||||
Sources: []*worker_pb.TaskSource{
|
||||
{Node: "src:8080", VolumeId: 99},
|
||||
},
|
||||
Targets: []*worker_pb.TaskTarget{
|
||||
{Node: "dst:8080", VolumeId: 99},
|
||||
},
|
||||
TaskParams: &worker_pb.TaskParams_BalanceParams{
|
||||
BalanceParams: &worker_pb.BalanceTaskParams{
|
||||
TimeoutSeconds: 60,
|
||||
},
|
||||
},
|
||||
}
|
||||
payload, err := proto.Marshal(taskParams)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
|
||||
job := &plugin_pb.JobSpec{
|
||||
JobId: "single-job-1",
|
||||
JobType: "volume_balance",
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
|
||||
},
|
||||
}
|
||||
|
||||
handler := NewVolumeBalanceHandler(nil)
|
||||
sender := &recordingExecutionSender{}
|
||||
|
||||
// Execute single-move path. Will fail on gRPC but verify it takes the single-move path.
|
||||
_ = handler.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
|
||||
Job: job,
|
||||
}, sender)
|
||||
|
||||
sender.mu.Lock()
|
||||
defer sender.mu.Unlock()
|
||||
|
||||
if len(sender.progress) == 0 {
|
||||
t.Fatalf("expected progress messages")
|
||||
}
|
||||
|
||||
// Single-move path sends "volume balance job accepted" not "batch volume balance"
|
||||
first := sender.progress[0]
|
||||
if first.Stage != "assigned" {
|
||||
t.Fatalf("expected first stage 'assigned', got %q", first.Stage)
|
||||
}
|
||||
if strings.Contains(first.Message, "batch") {
|
||||
t.Fatalf("single-move path should not mention batch, got %q", first.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterMetricsByLocation(t *testing.T) {
|
||||
metrics := []*workertypes.VolumeHealthMetrics{
|
||||
{VolumeID: 1, Server: "node-a", DataCenter: "dc1", Rack: "rack1"},
|
||||
{VolumeID: 2, Server: "node-b", DataCenter: "dc1", Rack: "rack2"},
|
||||
{VolumeID: 3, Server: "node-c", DataCenter: "dc2", Rack: "rack1"},
|
||||
{VolumeID: 4, Server: "node-d", DataCenter: "dc2", Rack: "rack3"},
|
||||
}
|
||||
|
||||
// Filter by DC
|
||||
filtered := pluginworker.FilterMetricsByLocation(metrics, "dc1", "", "")
|
||||
if len(filtered) != 2 {
|
||||
t.Fatalf("DC filter: expected 2, got %d", len(filtered))
|
||||
}
|
||||
|
||||
// Filter by rack
|
||||
filtered = pluginworker.FilterMetricsByLocation(metrics, "", "rack1,rack2", "")
|
||||
if len(filtered) != 3 {
|
||||
t.Fatalf("rack filter: expected 3, got %d", len(filtered))
|
||||
}
|
||||
|
||||
// Filter by node
|
||||
filtered = pluginworker.FilterMetricsByLocation(metrics, "", "", "node-a,node-c")
|
||||
if len(filtered) != 2 {
|
||||
t.Fatalf("node filter: expected 2, got %d", len(filtered))
|
||||
}
|
||||
|
||||
// Combined DC + rack
|
||||
filtered = pluginworker.FilterMetricsByLocation(metrics, "dc2", "rack3", "")
|
||||
if len(filtered) != 1 {
|
||||
t.Fatalf("DC+rack filter: expected 1, got %d", len(filtered))
|
||||
}
|
||||
|
||||
// Empty filters pass all
|
||||
filtered = pluginworker.FilterMetricsByLocation(metrics, "", "", "")
|
||||
if len(filtered) != 4 {
|
||||
t.Fatalf("no filter: expected 4, got %d", len(filtered))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterMetricsByVolumeState(t *testing.T) {
|
||||
metrics := []*workertypes.VolumeHealthMetrics{
|
||||
{VolumeID: 1, FullnessRatio: 0.5}, // active
|
||||
{VolumeID: 2, FullnessRatio: 1.0}, // active (below 1.01)
|
||||
{VolumeID: 3, FullnessRatio: 1.009}, // active (below 1.01)
|
||||
{VolumeID: 4, FullnessRatio: 1.01}, // full (exactly at threshold)
|
||||
{VolumeID: 5, FullnessRatio: 1.5}, // full
|
||||
{VolumeID: 6, FullnessRatio: 2.0}, // full
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
state string
|
||||
expectedIDs []uint32
|
||||
}{
|
||||
{
|
||||
name: "ALL returns everything",
|
||||
state: "ALL",
|
||||
expectedIDs: []uint32{1, 2, 3, 4, 5, 6},
|
||||
},
|
||||
{
|
||||
name: "empty string returns everything",
|
||||
state: "",
|
||||
expectedIDs: []uint32{1, 2, 3, 4, 5, 6},
|
||||
},
|
||||
{
|
||||
name: "ACTIVE keeps FullnessRatio below 1.01",
|
||||
state: "ACTIVE",
|
||||
expectedIDs: []uint32{1, 2, 3},
|
||||
},
|
||||
{
|
||||
name: "FULL keeps FullnessRatio at or above 1.01",
|
||||
state: "FULL",
|
||||
expectedIDs: []uint32{4, 5, 6},
|
||||
},
|
||||
{
|
||||
name: "unknown value returns everything",
|
||||
state: "INVALID",
|
||||
expectedIDs: []uint32{1, 2, 3, 4, 5, 6},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := pluginworker.FilterMetricsByVolumeState(metrics, pluginworker.VolumeState(tt.state))
|
||||
if len(result) != len(tt.expectedIDs) {
|
||||
t.Fatalf("expected %d metrics, got %d", len(tt.expectedIDs), len(result))
|
||||
}
|
||||
for i, m := range result {
|
||||
if m.VolumeID != tt.expectedIDs[i] {
|
||||
t.Errorf("result[%d].VolumeID = %d, want %d", i, m.VolumeID, tt.expectedIDs[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterMetricsByVolumeState_NilElement(t *testing.T) {
|
||||
metrics := []*workertypes.VolumeHealthMetrics{
|
||||
nil,
|
||||
{VolumeID: 1, FullnessRatio: 0.5},
|
||||
nil,
|
||||
{VolumeID: 2, FullnessRatio: 1.5},
|
||||
}
|
||||
result := pluginworker.FilterMetricsByVolumeState(metrics, pluginworker.VolumeStateActive)
|
||||
if len(result) != 1 || result[0].VolumeID != 1 {
|
||||
t.Fatalf("expected [vol 1] for ACTIVE with nil elements, got %d results", len(result))
|
||||
}
|
||||
result = pluginworker.FilterMetricsByVolumeState(metrics, pluginworker.VolumeStateFull)
|
||||
if len(result) != 1 || result[0].VolumeID != 2 {
|
||||
t.Fatalf("expected [vol 2] for FULL with nil elements, got %d results", len(result))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterMetricsByVolumeState_EmptyInput(t *testing.T) {
|
||||
result := pluginworker.FilterMetricsByVolumeState(nil, pluginworker.VolumeStateActive)
|
||||
if len(result) != 0 {
|
||||
t.Fatalf("expected 0 metrics for nil input, got %d", len(result))
|
||||
}
|
||||
|
||||
result = pluginworker.FilterMetricsByVolumeState([]*workertypes.VolumeHealthMetrics{}, pluginworker.VolumeStateFull)
|
||||
if len(result) != 0 {
|
||||
t.Fatalf("expected 0 metrics for empty input, got %d", len(result))
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeBalanceDescriptorHasVolumeStateField(t *testing.T) {
|
||||
descriptor := NewVolumeBalanceHandler(nil).Descriptor()
|
||||
if descriptor == nil || descriptor.AdminConfigForm == nil {
|
||||
t.Fatalf("expected admin config form in descriptor")
|
||||
}
|
||||
found := false
|
||||
for _, section := range descriptor.AdminConfigForm.Sections {
|
||||
for _, field := range section.Fields {
|
||||
if field.Name == "volume_state" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected volume_state field in admin config form")
|
||||
}
|
||||
defaultVal, ok := descriptor.AdminConfigForm.DefaultValues["volume_state"]
|
||||
if !ok {
|
||||
t.Fatalf("expected volume_state default value")
|
||||
}
|
||||
if defaultVal.GetStringValue() != "ALL" {
|
||||
t.Fatalf("expected volume_state default 'ALL', got %q", defaultVal.GetStringValue())
|
||||
}
|
||||
}
|
||||
|
||||
func workerConfigFormHasField(form *plugin_pb.ConfigForm, fieldName string) bool {
|
||||
if form == nil {
|
||||
return false
|
||||
}
|
||||
for _, section := range form.Sections {
|
||||
if section == nil {
|
||||
continue
|
||||
}
|
||||
for _, field := range section.Fields {
|
||||
if field != nil && field.Name == fieldName {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type noopDetectionSender struct{}
|
||||
|
||||
func (noopDetectionSender) SendProposals(*plugin_pb.DetectionProposals) error { return nil }
|
||||
func (noopDetectionSender) SendComplete(*plugin_pb.DetectionComplete) error { return nil }
|
||||
func (noopDetectionSender) SendActivity(*plugin_pb.ActivityEvent) error { return nil }
|
||||
|
||||
type noopExecutionSender struct{}
|
||||
|
||||
func (noopExecutionSender) SendProgress(*plugin_pb.JobProgressUpdate) error { return nil }
|
||||
func (noopExecutionSender) SendCompleted(*plugin_pb.JobCompleted) error { return nil }
|
||||
|
||||
type recordingDetectionSender struct {
|
||||
proposals *plugin_pb.DetectionProposals
|
||||
complete *plugin_pb.DetectionComplete
|
||||
events []*plugin_pb.ActivityEvent
|
||||
}
|
||||
|
||||
func (r *recordingDetectionSender) SendProposals(proposals *plugin_pb.DetectionProposals) error {
|
||||
r.proposals = proposals
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recordingDetectionSender) SendComplete(complete *plugin_pb.DetectionComplete) error {
|
||||
r.complete = complete
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recordingDetectionSender) SendActivity(event *plugin_pb.ActivityEvent) error {
|
||||
if event != nil {
|
||||
r.events = append(r.events, event)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestCheckMovePreconditions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
locations []string
|
||||
sourceNode string
|
||||
targetNode string
|
||||
wantErr string
|
||||
}{
|
||||
{"valid move", []string{"10.0.0.1:8080", "10.0.0.3:8080"}, "10.0.0.1:8080", "10.0.0.2:8080", ""},
|
||||
{"volume left the source", []string{"10.0.0.3:8080"}, "10.0.0.1:8080", "10.0.0.2:8080", "no longer on source"},
|
||||
{"volume gone entirely", nil, "10.0.0.1:8080", "10.0.0.2:8080", "no longer on source"},
|
||||
{"target already has a replica", []string{"10.0.0.1:8080", "10.0.0.2:8080"}, "10.0.0.1:8080", "10.0.0.2:8080", "already has a replica on target"},
|
||||
{"proposal nodes carry grpc suffix", []string{"10.0.0.1:8080", "10.0.0.3:8080"}, "10.0.0.1:8080.18080", "10.0.0.2:8080.18080", ""},
|
||||
{"grpc-suffixed target already has a replica", []string{"10.0.0.1:8080", "10.0.0.2:8080"}, "10.0.0.1:8080.18080", "10.0.0.2:8080.18080", "already has a replica on target"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := checkMovePreconditions(tt.locations, 5, tt.sourceNode, tt.targetNode)
|
||||
if tt.wantErr == "" {
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tt.wantErr) {
|
||||
t.Fatalf("expected error containing %q, got %v", tt.wantErr, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
95
weed/worker/tasks/balance/register.go
Normal file
95
weed/worker/tasks/balance/register.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package balance
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// Global variable to hold the task definition for configuration updates
|
||||
var globalTaskDef *base.TaskDefinition
|
||||
|
||||
// Auto-register this task when the package is imported
|
||||
func init() {
|
||||
RegisterBalanceTask()
|
||||
|
||||
// Register config updater
|
||||
tasks.AutoRegisterConfigUpdater(types.TaskTypeBalance, UpdateConfigFromPersistence)
|
||||
}
|
||||
|
||||
// RegisterBalanceTask registers the balance task with the new architecture
|
||||
func RegisterBalanceTask() {
|
||||
// Create configuration instance
|
||||
config := NewDefaultConfig()
|
||||
|
||||
// Create shared gRPC dial option using TLS configuration
|
||||
dialOpt := security.LoadClientTLS(util.GetViper(), "grpc.worker")
|
||||
|
||||
// Create complete task definition
|
||||
taskDef := &base.TaskDefinition{
|
||||
Type: types.TaskTypeBalance,
|
||||
Name: "balance",
|
||||
DisplayName: "Volume Balance",
|
||||
Description: "Balances volume distribution across servers",
|
||||
Icon: "fas fa-balance-scale text-warning",
|
||||
Capabilities: []string{"balance", "distribution"},
|
||||
|
||||
Config: config,
|
||||
ConfigSpec: GetConfigSpec(),
|
||||
CreateTask: func(params *worker_pb.TaskParams) (types.Task, error) {
|
||||
if params == nil {
|
||||
return nil, fmt.Errorf("task parameters are required")
|
||||
}
|
||||
if len(params.Sources) == 0 {
|
||||
return nil, fmt.Errorf("at least one source is required for balance task")
|
||||
}
|
||||
return NewBalanceTask(
|
||||
fmt.Sprintf("balance-%d", params.VolumeId),
|
||||
params.Sources[0].Node, // Use first source node
|
||||
params.VolumeId,
|
||||
params.Collection,
|
||||
dialOpt,
|
||||
), nil
|
||||
},
|
||||
DetectionFunc: func(metrics []*types.VolumeHealthMetrics, info *types.ClusterInfo, config base.TaskConfig) ([]*types.TaskDetectionResult, error) {
|
||||
results, _, err := Detection(metrics, info, config, 0)
|
||||
return results, err
|
||||
},
|
||||
ScanInterval: 30 * time.Minute,
|
||||
SchedulingFunc: Scheduling,
|
||||
MaxConcurrent: 1,
|
||||
RepeatInterval: 2 * time.Hour,
|
||||
}
|
||||
|
||||
// Store task definition globally for configuration updates
|
||||
globalTaskDef = taskDef
|
||||
|
||||
// Register everything with a single function call!
|
||||
base.RegisterTask(taskDef)
|
||||
}
|
||||
|
||||
// UpdateConfigFromPersistence updates the balance configuration from persistence
|
||||
func UpdateConfigFromPersistence(configPersistence interface{}) error {
|
||||
if globalTaskDef == nil {
|
||||
return fmt.Errorf("balance task not registered")
|
||||
}
|
||||
|
||||
// Load configuration from persistence
|
||||
newConfig := LoadConfigFromPersistence(configPersistence)
|
||||
if newConfig == nil {
|
||||
return fmt.Errorf("failed to load configuration from persistence")
|
||||
}
|
||||
|
||||
// Update the task definition's config
|
||||
globalTaskDef.Config = newConfig
|
||||
|
||||
glog.V(1).Infof("Updated balance task configuration from persistence")
|
||||
return nil
|
||||
}
|
||||
37
weed/worker/tasks/balance/replica_placement.go
Normal file
37
weed/worker/tasks/balance/replica_placement.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package balance
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/topology/balancer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// hostFromAddress returns the physical machine (host/IP) of a server address,
|
||||
// falling back to the node id when no address is known.
|
||||
func hostFromAddress(address, nodeID string) string {
|
||||
return balancer.HostFromAddress(address, nodeID)
|
||||
}
|
||||
|
||||
// IsGoodMove checks whether moving a volume from sourceNodeID to target would
|
||||
// satisfy the volume's replica placement policy. It is a thin adapter over the
|
||||
// shared placement logic in weed/topology/balancer so the shell command and this
|
||||
// worker cannot drift.
|
||||
func IsGoodMove(rp *super_block.ReplicaPlacement, existingReplicas []types.ReplicaLocation, sourceNodeID string, target types.ReplicaLocation) bool {
|
||||
locs := make([]balancer.Location, len(existingReplicas))
|
||||
for i, r := range existingReplicas {
|
||||
locs[i] = toBalancerLocation(r)
|
||||
}
|
||||
return balancer.IsGoodMove(rp, locs, sourceNodeID, toBalancerLocation(target))
|
||||
}
|
||||
|
||||
// toBalancerLocation copies a worker replica location to the shared placement
|
||||
// type field-by-field, so a future change to either struct is a compile error
|
||||
// here rather than a silent mismatch.
|
||||
func toBalancerLocation(r types.ReplicaLocation) balancer.Location {
|
||||
return balancer.Location{
|
||||
DataCenter: r.DataCenter,
|
||||
Rack: r.Rack,
|
||||
NodeID: r.NodeID,
|
||||
Host: r.Host,
|
||||
}
|
||||
}
|
||||
256
weed/worker/tasks/balance/replica_placement_test.go
Normal file
256
weed/worker/tasks/balance/replica_placement_test.go
Normal file
@@ -0,0 +1,256 @@
|
||||
package balance
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
func rp(t *testing.T, code string) *super_block.ReplicaPlacement {
|
||||
t.Helper()
|
||||
r, err := super_block.NewReplicaPlacementFromString(code)
|
||||
if err != nil {
|
||||
t.Fatalf("invalid replica placement code %q: %v", code, err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func loc(dc, rack, node string) types.ReplicaLocation {
|
||||
return types.ReplicaLocation{DataCenter: dc, Rack: rack, NodeID: node}
|
||||
}
|
||||
|
||||
func TestIsGoodMove_NoReplication(t *testing.T) {
|
||||
// 000 = no replication. Any move is fine.
|
||||
if !IsGoodMove(rp(t, "000"), []types.ReplicaLocation{loc("dc1", "r1", "n1")}, "n1", loc("dc1", "r1", "n2")) {
|
||||
t.Error("000: any move should be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsGoodMove_MachineAntiAffinity(t *testing.T) {
|
||||
// rep 001 allows two copies in one rack, so replica placement alone would permit
|
||||
// moving onto another port of a host that already holds a replica; machine
|
||||
// anti-affinity must reject that and allow a distinct host.
|
||||
existing := []types.ReplicaLocation{
|
||||
{DataCenter: "dc1", Rack: "r1", NodeID: "10.0.0.1:8080", Host: "10.0.0.1"},
|
||||
{DataCenter: "dc1", Rack: "r1", NodeID: "10.0.0.2:8080", Host: "10.0.0.2"},
|
||||
}
|
||||
onSameMachine := types.ReplicaLocation{DataCenter: "dc1", Rack: "r1", NodeID: "10.0.0.1:8081", Host: "10.0.0.1"}
|
||||
if IsGoodMove(rp(t, "001"), existing, "10.0.0.2:8080", onSameMachine) {
|
||||
t.Error("move onto a machine already holding a replica should be rejected")
|
||||
}
|
||||
onOtherMachine := types.ReplicaLocation{DataCenter: "dc1", Rack: "r1", NodeID: "10.0.0.3:8080", Host: "10.0.0.3"}
|
||||
if !IsGoodMove(rp(t, "001"), existing, "10.0.0.2:8080", onOtherMachine) {
|
||||
t.Error("move onto a distinct machine should be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsGoodMove_001_SameRack(t *testing.T) {
|
||||
// 001 = 1 replica on same rack (2 total on same rack)
|
||||
existing := []types.ReplicaLocation{
|
||||
loc("dc1", "r1", "n1"),
|
||||
loc("dc1", "r1", "n2"),
|
||||
}
|
||||
// Move n1 -> n3 on same rack: good
|
||||
if !IsGoodMove(rp(t, "001"), existing, "n1", loc("dc1", "r1", "n3")) {
|
||||
t.Error("001: move to same rack should be allowed")
|
||||
}
|
||||
// Move n1 -> n3 on different rack: bad (would leave only 1 on r1, need 2)
|
||||
if IsGoodMove(rp(t, "001"), existing, "n1", loc("dc1", "r2", "n3")) {
|
||||
t.Error("001: move to different rack should not be allowed when it breaks same-rack count")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsGoodMove_010_DiffRack(t *testing.T) {
|
||||
// 010 = 1 replica on different rack (2 racks total)
|
||||
existing := []types.ReplicaLocation{
|
||||
loc("dc1", "r1", "n1"),
|
||||
loc("dc1", "r2", "n2"),
|
||||
}
|
||||
// Move n1 -> n3 on r2: bad (both replicas on same rack)
|
||||
if IsGoodMove(rp(t, "010"), existing, "n1", loc("dc1", "r2", "n3")) {
|
||||
t.Error("010: move to same rack as other replica should not be allowed")
|
||||
}
|
||||
// Move n1 -> n3 on r3: good (still 2 different racks)
|
||||
if !IsGoodMove(rp(t, "010"), existing, "n1", loc("dc1", "r3", "n3")) {
|
||||
t.Error("010: move to different rack should be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsGoodMove_100_DiffDC(t *testing.T) {
|
||||
// 100 = 1 replica in different DC
|
||||
existing := []types.ReplicaLocation{
|
||||
loc("dc1", "r1", "n1"),
|
||||
loc("dc2", "r1", "n2"),
|
||||
}
|
||||
// Move n1 -> n3 in dc2: bad (both in same DC)
|
||||
if IsGoodMove(rp(t, "100"), existing, "n1", loc("dc2", "r1", "n3")) {
|
||||
t.Error("100: move to same DC as other replica should not be allowed")
|
||||
}
|
||||
// Move n1 -> n3 in dc3: good (different DCs)
|
||||
if !IsGoodMove(rp(t, "100"), existing, "n1", loc("dc3", "r1", "n3")) {
|
||||
t.Error("100: move to different DC should be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsGoodMove_SameNode(t *testing.T) {
|
||||
// Moving to the same node as an existing replica should always be rejected
|
||||
existing := []types.ReplicaLocation{
|
||||
loc("dc1", "r1", "n1"),
|
||||
loc("dc1", "r2", "n2"),
|
||||
}
|
||||
if IsGoodMove(rp(t, "010"), existing, "n1", loc("dc1", "r2", "n2")) {
|
||||
t.Error("should reject move to same node as existing replica")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsGoodMove_011_Composite(t *testing.T) {
|
||||
// 011 = 1 same-rack + 1 different-rack (3 replicas: 2 on same rack, 1 on different)
|
||||
existing := []types.ReplicaLocation{
|
||||
loc("dc1", "r1", "n1"),
|
||||
loc("dc1", "r1", "n2"),
|
||||
loc("dc1", "r2", "n3"),
|
||||
}
|
||||
// Move n1 -> n4 on r1: good (maintains 2 on r1, 1 on r2)
|
||||
if !IsGoodMove(rp(t, "011"), existing, "n1", loc("dc1", "r1", "n4")) {
|
||||
t.Error("011: move within same rack should be allowed")
|
||||
}
|
||||
// Move n3 -> n4 on r1: bad (would have 3 on r1, 0 on different rack)
|
||||
if IsGoodMove(rp(t, "011"), existing, "n3", loc("dc1", "r1", "n4")) {
|
||||
t.Error("011: move that eliminates different-rack replica should not be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsGoodMove_110_Composite(t *testing.T) {
|
||||
// 110 = 1 different-rack + 1 different-DC (3 replicas across 2 DCs and 2 racks)
|
||||
existing := []types.ReplicaLocation{
|
||||
loc("dc1", "r1", "n1"),
|
||||
loc("dc1", "r2", "n2"),
|
||||
loc("dc2", "r1", "n3"),
|
||||
}
|
||||
// Move n1 -> n4 in dc1/r3: good (dc1 still has r2+r3, dc2 has r1)
|
||||
if !IsGoodMove(rp(t, "110"), existing, "n1", loc("dc1", "r3", "n4")) {
|
||||
t.Error("110: move to new rack in same DC should be allowed")
|
||||
}
|
||||
// Move n3 -> n4 in dc1/r1: bad (would lose the different-DC replica)
|
||||
if IsGoodMove(rp(t, "110"), existing, "n3", loc("dc1", "r1", "n4")) {
|
||||
t.Error("110: move that eliminates different-DC replica should not be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsGoodMove_NilReplicaPlacement(t *testing.T) {
|
||||
if !IsGoodMove(nil, []types.ReplicaLocation{loc("dc1", "r1", "n1")}, "n1", loc("dc1", "r1", "n2")) {
|
||||
t.Error("nil replica placement should allow any move")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalculateBalanceScore_ReplicationAware(t *testing.T) {
|
||||
disk := func(dc, rack string) *topology.DiskInfo {
|
||||
return &topology.DiskInfo{
|
||||
DataCenter: dc,
|
||||
Rack: rack,
|
||||
DiskInfo: &master_pb.DiskInfo{MaxVolumeCount: 100, VolumeCount: 50},
|
||||
}
|
||||
}
|
||||
|
||||
// 001: same-rack replication — should prefer same rack and same DC
|
||||
rp001 := rp(t, "001")
|
||||
sameRack := calculateBalanceScore(disk("dc1", "r1"), "r1", "dc1", 0, rp001)
|
||||
diffRack := calculateBalanceScore(disk("dc1", "r2"), "r1", "dc1", 0, rp001)
|
||||
diffDC := calculateBalanceScore(disk("dc2", "r2"), "r1", "dc1", 0, rp001)
|
||||
if sameRack <= diffRack {
|
||||
t.Errorf("001: same-rack score (%v) should exceed different-rack score (%v)", sameRack, diffRack)
|
||||
}
|
||||
if sameRack <= diffDC {
|
||||
t.Errorf("001: same-rack score (%v) should exceed different-DC score (%v)", sameRack, diffDC)
|
||||
}
|
||||
|
||||
// 010: different-rack replication — should prefer different rack, same DC
|
||||
rp010 := rp(t, "010")
|
||||
sameRack = calculateBalanceScore(disk("dc1", "r1"), "r1", "dc1", 0, rp010)
|
||||
diffRack = calculateBalanceScore(disk("dc1", "r2"), "r1", "dc1", 0, rp010)
|
||||
if diffRack <= sameRack {
|
||||
t.Errorf("010: different-rack score (%v) should exceed same-rack score (%v)", diffRack, sameRack)
|
||||
}
|
||||
|
||||
// 100: different-DC replication — should prefer different DC
|
||||
rp100 := rp(t, "100")
|
||||
sameDC := calculateBalanceScore(disk("dc1", "r2"), "r1", "dc1", 0, rp100)
|
||||
diffDCScore := calculateBalanceScore(disk("dc2", "r2"), "r1", "dc1", 0, rp100)
|
||||
if diffDCScore <= sameDC {
|
||||
t.Errorf("100: different-DC score (%v) should exceed same-DC score (%v)", diffDCScore, sameDC)
|
||||
}
|
||||
|
||||
// nil rp: should prefer cross-rack/DC (default behavior)
|
||||
sameRack = calculateBalanceScore(disk("dc1", "r1"), "r1", "dc1", 0, nil)
|
||||
diffRack = calculateBalanceScore(disk("dc1", "r2"), "r1", "dc1", 0, nil)
|
||||
if diffRack <= sameRack {
|
||||
t.Errorf("nil rp: different-rack score (%v) should exceed same-rack score (%v)", diffRack, sameRack)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanBalanceDestination_ChoosesBestValidCompositeDestination(t *testing.T) {
|
||||
servers := []serverSpec{
|
||||
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
|
||||
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack3"},
|
||||
{id: "node-c", diskType: "hdd", diskID: 3, dc: "dc1", rack: "rack1"},
|
||||
}
|
||||
volumes := makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 1, 1, withReplicas(11))
|
||||
replicas := []types.ReplicaLocation{
|
||||
loc("dc1", "rack1", "node-a"),
|
||||
loc("dc1", "rack1", "node-d"),
|
||||
loc("dc1", "rack2", "node-e"),
|
||||
}
|
||||
|
||||
plan, err := planBalanceDestination(buildTopology(servers, volumes), volumes[0], rp(t, "011"), replicas, map[string]int{
|
||||
"node-b": 0,
|
||||
"node-c": 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("planBalanceDestination failed: %v", err)
|
||||
}
|
||||
if plan.TargetNode != "node-c" {
|
||||
t.Fatalf("expected valid same-rack destination node-c, got %s", plan.TargetNode)
|
||||
}
|
||||
if plan.TargetRack != "rack1" {
|
||||
t.Fatalf("expected rack1 destination, got %s", plan.TargetRack)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateBalanceTask_FallbackSelectsValidCompositeDestination(t *testing.T) {
|
||||
servers := []serverSpec{
|
||||
{id: "node-a", diskType: "hdd", diskID: 1, dc: "dc1", rack: "rack1"},
|
||||
{id: "node-b", diskType: "hdd", diskID: 2, dc: "dc1", rack: "rack3"},
|
||||
{id: "node-c", diskType: "hdd", diskID: 3, dc: "dc1", rack: "rack1"},
|
||||
}
|
||||
volumes := makeVolumesWith("node-a", "hdd", "dc1", "rack1", "c1", 1, 1, withReplicas(11))
|
||||
clusterInfo := &types.ClusterInfo{
|
||||
ActiveTopology: buildTopology(servers, volumes),
|
||||
VolumeReplicaMap: map[uint32][]types.ReplicaLocation{
|
||||
1: {
|
||||
loc("dc1", "rack1", "node-a"),
|
||||
loc("dc1", "rack1", "node-d"),
|
||||
loc("dc1", "rack2", "node-e"),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
task, destination := createBalanceTask("hdd", volumes[0], clusterInfo, "node-b", map[string]int{
|
||||
"node-b": 0,
|
||||
"node-c": 0,
|
||||
}, 0)
|
||||
if task == nil {
|
||||
t.Fatal("expected a balance task")
|
||||
}
|
||||
if destination != "node-c" {
|
||||
t.Fatalf("expected fallback destination node-c, got %s", destination)
|
||||
}
|
||||
if len(task.TypedParams.Targets) != 1 {
|
||||
t.Fatalf("expected 1 target, got %d", len(task.TypedParams.Targets))
|
||||
}
|
||||
if got := task.TypedParams.Targets[0].Node; got != "node-c:8080" {
|
||||
t.Fatalf("expected target node-c:8080, got %s", got)
|
||||
}
|
||||
}
|
||||
37
weed/worker/tasks/balance/scheduling.go
Normal file
37
weed/worker/tasks/balance/scheduling.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package balance
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// Scheduling implements the scheduling logic for balance tasks
|
||||
func Scheduling(task *types.TaskInput, runningTasks []*types.TaskInput, availableWorkers []*types.WorkerData, config base.TaskConfig) bool {
|
||||
balanceConfig := config.(*Config)
|
||||
|
||||
// Count running balance tasks
|
||||
runningBalanceCount := 0
|
||||
for _, runningTask := range runningTasks {
|
||||
if runningTask.Type == types.TaskTypeBalance {
|
||||
runningBalanceCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Check concurrency limit
|
||||
if runningBalanceCount >= balanceConfig.MaxConcurrent {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check if we have available workers
|
||||
availableWorkerCount := 0
|
||||
for _, worker := range availableWorkers {
|
||||
for _, capability := range worker.Capabilities {
|
||||
if capability == types.TaskTypeBalance {
|
||||
availableWorkerCount++
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return availableWorkerCount > 0
|
||||
}
|
||||
Reference in New Issue
Block a user