snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,267 @@
package ec_balance
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/admin/config"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
)
// Config extends BaseConfig with EC balance specific settings
type Config struct {
base.BaseConfig
ImbalanceThreshold float64 `json:"imbalance_threshold"`
MinServerCount int `json:"min_server_count"`
CollectionFilter string `json:"collection_filter"`
DiskType string `json:"disk_type"`
PreferredTags []string `json:"preferred_tags"`
ReplicaPlacement string `json:"replica_placement"` // e.g. "020"; empty falls back to the master default replication (even spread only when that default is empty or zero)
IoBytePerSecond int64 `json:"io_byte_per_second"` // limit each shard copy's rate; 0 falls back to the volume server's maintenance rate
DataCenterFilter string `json:"-"` // per-detection-run, not persisted
}
// NewDefaultConfig creates a new default EC balance configuration
func NewDefaultConfig() *Config {
return &Config{
BaseConfig: base.BaseConfig{
Enabled: true,
ScanIntervalSeconds: 60 * 60, // 1 hour
MaxConcurrent: 1,
},
ImbalanceThreshold: 0.2, // 20%
MinServerCount: 3,
CollectionFilter: "",
DiskType: "",
PreferredTags: nil,
}
}
// GetConfigSpec returns the configuration schema for EC balance tasks
func GetConfigSpec() base.ConfigSpec {
return base.ConfigSpec{
Fields: []*config.Field{
{
Name: "enabled",
JSONName: "enabled",
Type: config.FieldTypeBool,
DefaultValue: true,
Required: false,
DisplayName: "Enable EC Shard Balance Tasks",
Description: "Whether EC shard balance tasks should be automatically created",
HelpText: "Toggle this to enable or disable automatic EC shard balancing",
InputType: "checkbox",
CSSClasses: "form-check-input",
},
{
Name: "scan_interval_seconds",
JSONName: "scan_interval_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 60 * 60,
MinValue: 10 * 60,
MaxValue: 24 * 60 * 60,
Required: true,
DisplayName: "Scan Interval",
Description: "How often to scan for EC shard imbalances",
HelpText: "The system will check for EC shard distribution imbalances at this interval",
Placeholder: "1",
Unit: config.UnitHours,
InputType: "interval",
CSSClasses: "form-control",
},
{
Name: "max_concurrent",
JSONName: "max_concurrent",
Type: config.FieldTypeInt,
DefaultValue: 1,
MinValue: 1,
MaxValue: 5,
Required: true,
DisplayName: "Max Concurrent Tasks",
Description: "Maximum number of EC shard balance tasks that can run simultaneously",
HelpText: "Limits the number of EC shard balancing operations running at the same time",
Placeholder: "1 (default)",
Unit: config.UnitCount,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "imbalance_threshold",
JSONName: "imbalance_threshold",
Type: config.FieldTypeFloat,
DefaultValue: 0.2,
MinValue: 0.05,
MaxValue: 0.5,
Required: true,
DisplayName: "Imbalance Threshold",
Description: "Minimum shard count imbalance ratio to trigger balancing",
HelpText: "EC shard distribution imbalances above this threshold will trigger rebalancing",
Placeholder: "0.20 (20%)",
Unit: config.UnitNone,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "min_server_count",
JSONName: "min_server_count",
Type: config.FieldTypeInt,
DefaultValue: 3,
MinValue: 2,
MaxValue: 100,
Required: true,
DisplayName: "Minimum Server Count",
Description: "Minimum number of servers required for EC shard balancing",
HelpText: "EC shard balancing will only occur if there are at least this many servers",
Placeholder: "3 (default)",
Unit: config.UnitCount,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "collection_filter",
JSONName: "collection_filter",
Type: config.FieldTypeString,
DefaultValue: "",
Required: false,
DisplayName: "Collection Filter",
Description: "Only balance EC shards from specific collections (comma-separated names, wildcards, or regex patterns)",
HelpText: "Leave empty to balance all collections, or list the collections to balance",
Placeholder: "pictures,videos",
InputType: "text",
CSSClasses: "form-control",
},
{
Name: "disk_type",
JSONName: "disk_type",
Type: config.FieldTypeString,
DefaultValue: "",
Required: false,
DisplayName: "Disk Type",
Description: "Only balance EC shards on this disk type",
HelpText: "Leave empty for all disk types, or specify hdd or ssd",
Placeholder: "hdd",
InputType: "text",
CSSClasses: "form-control",
},
{
Name: "preferred_tags",
JSONName: "preferred_tags",
Type: config.FieldTypeString,
DefaultValue: "",
Required: false,
DisplayName: "Preferred Disk Tags",
Description: "Comma-separated disk tags to prioritize for shard placement",
HelpText: "EC shards will be placed on disks with these tags first, ordered by preference",
Placeholder: "fast,ssd",
InputType: "text",
CSSClasses: "form-control",
},
{
Name: "replica_placement",
JSONName: "replica_placement",
Type: config.FieldTypeString,
DefaultValue: "",
Required: false,
DisplayName: "Replica Placement",
Description: "EC shard replica placement constraint (e.g. 020)",
HelpText: "Leave empty to use the master default replication (even spread only when that default is empty or zero). When set, limits shards per rack/node per the placement digits (dc/rack/node)",
Placeholder: "020",
InputType: "text",
CSSClasses: "form-control",
},
{
Name: "io_byte_per_second",
JSONName: "io_byte_per_second",
Type: config.FieldTypeInt,
DefaultValue: 0,
MinValue: 0,
Required: false,
DisplayName: "Shard Copy IO Limit (bytes/sec)",
Description: "Limit each EC shard copy's rate",
HelpText: "0 falls back to each volume server's own maintenance rate (-maintenanceBytePerSecond)",
Placeholder: "0 (server maintenance rate)",
Unit: config.UnitNone,
InputType: "number",
CSSClasses: "form-control",
},
},
}
}
// ToTaskPolicy converts configuration to a TaskPolicy protobuf message
func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
preferredTagsCopy := append([]string(nil), c.PreferredTags...)
return &worker_pb.TaskPolicy{
Enabled: c.Enabled,
MaxConcurrent: int32(c.MaxConcurrent),
RepeatIntervalSeconds: int32(c.ScanIntervalSeconds),
CheckIntervalSeconds: int32(c.ScanIntervalSeconds),
TaskConfig: &worker_pb.TaskPolicy_EcBalanceConfig{
EcBalanceConfig: &worker_pb.EcBalanceTaskConfig{
ImbalanceThreshold: c.ImbalanceThreshold,
MinServerCount: int32(c.MinServerCount),
CollectionFilter: c.CollectionFilter,
DiskType: c.DiskType,
PreferredTags: preferredTagsCopy,
ReplicaPlacement: c.ReplicaPlacement,
IoBytePerSecond: c.IoBytePerSecond,
},
},
}
}
// FromTaskPolicy loads configuration from a TaskPolicy protobuf message
func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
if policy == nil {
return fmt.Errorf("policy is nil")
}
c.Enabled = policy.Enabled
c.MaxConcurrent = int(policy.MaxConcurrent)
c.ScanIntervalSeconds = int(policy.RepeatIntervalSeconds)
if ecbConfig := policy.GetEcBalanceConfig(); ecbConfig != nil {
c.ImbalanceThreshold = ecbConfig.ImbalanceThreshold
c.MinServerCount = int(ecbConfig.MinServerCount)
c.CollectionFilter = ecbConfig.CollectionFilter
c.DiskType = ecbConfig.DiskType
c.PreferredTags = append([]string(nil), ecbConfig.PreferredTags...)
c.ReplicaPlacement = ecbConfig.ReplicaPlacement
c.IoBytePerSecond = ecbConfig.IoBytePerSecond
}
return nil
}
// LoadConfigFromPersistence loads configuration from the persistence layer if available
func LoadConfigFromPersistence(configPersistence interface{}) *Config {
cfg := NewDefaultConfig()
if persistence, ok := configPersistence.(interface {
LoadEcBalanceTaskPolicy() (*worker_pb.TaskPolicy, error)
}); ok {
policy, err := persistence.LoadEcBalanceTaskPolicy()
switch {
case err != nil:
glog.Warningf("Could not read the persisted EC balance configuration, falling back to defaults: %v", err)
case policy == nil:
glog.V(1).Infof("No EC balance configuration persisted yet, using defaults")
default:
if err := cfg.FromTaskPolicy(policy); err != nil {
glog.Warningf("Could not apply the persisted EC balance configuration, falling back to defaults: %v", err)
} else {
glog.V(1).Infof("Loaded EC balance configuration from persistence")
return cfg
}
}
} else if configPersistence != nil {
// A store was handed in but does not expose the accessor, so the persisted
// settings are silently ignored - always a wiring bug, never a normal state.
glog.Warningf("%T cannot provide the persisted EC balance configuration: it has no LoadEcBalanceTaskPolicy() method, "+
"so the compiled-in defaults are used and any saved EC balance settings are ignored", configPersistence)
}
glog.V(1).Infof("Using default EC balance configuration")
return cfg
}

View File

@@ -0,0 +1,128 @@
package ec_balance
import (
"context"
"testing"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
)
// The dedup phase deletes a shard it believes is duplicated elsewhere. It picks
// its victims from the master's topology, and the topology can name a location
// that holds nothing: a volume server answering "not found ec volume id N" for
// a shard the master lists is the observable form of that. Deleting on the
// strength of a phantom peer removes the only real copy.
//
// The move phase already refuses to work on trust — it verifies the shard
// registered on the destination before removing the source. These tests hold
// dedup to the same standard, because it is the more dangerous of the two: it
// deletes without copying anything first.
const (
dedupTestVolumeID = uint32(42561)
dedupTestCollection = "pm-itatiaiucu-01"
)
func dedupParams(sourceNode, keepNode string, shardID uint32) *worker_pb.TaskParams {
// Dedup is signalled by source and target being the same node and disk.
loc := &worker_pb.TaskSource{Node: sourceNode, DiskId: 0, ShardIds: []uint32{shardID}}
return &worker_pb.TaskParams{
VolumeId: dedupTestVolumeID,
Collection: dedupTestCollection,
Sources: []*worker_pb.TaskSource{loc},
Targets: []*worker_pb.TaskTarget{{Node: sourceNode, DiskId: 0, ShardIds: []uint32{shardID}}},
// The dedup branch is only reachable when EC params are present; without
// them the job silently takes the copy-and-move path instead.
TaskParams: &worker_pb.TaskParams_EcBalanceParams{
EcBalanceParams: &worker_pb.EcBalanceTaskParams{DedupKeepNode: keepNode},
},
}
}
func newDedupTask() *ECBalanceTask {
return NewECBalanceTask("dedup-test", dedupTestVolumeID, dedupTestCollection,
grpc.WithTransportCredentials(insecure.NewCredentials()))
}
// The failure that cost this cluster shards: the topology claims shard 0 lives
// on two nodes, only one of them actually has it, and dedup is pointed at the
// one that does. Nothing else in the cluster holds shard 0, so deleting it is
// unrecoverable — and no copy was made first, because dedup never makes one.
func TestDedupRefusesToDeleteTheOnlyRealCopy(t *testing.T) {
holder := startFakeEcVolumeServer(t, dedupTestVolumeID, 0)
// The peer the topology believes also has shard 0. Its disks are empty,
// exactly like a server answering "not found ec volume id".
phantom := startFakeEcVolumeServer(t, dedupTestVolumeID)
if !holder.has(dedupTestVolumeID, 0) {
t.Fatal("precondition: holder should start with shard 0")
}
if phantom.has(dedupTestVolumeID, 0) {
t.Fatal("precondition: phantom should hold nothing")
}
err := newDedupTask().Execute(context.Background(), dedupParams(holder.address(), phantom.address(), 0))
if holder.has(dedupTestVolumeID, 0) {
return // refused, or verified and backed off — either is safe
}
t.Fatalf("dedup deleted the only copy of shard %d.0 (deleted=%v, err=%v); "+
"the surviving copy was never confirmed to exist",
dedupTestVolumeID, holder.deletedShards(), err)
}
// A shard genuinely present on two servers is safe to thin out: one copy is
// redundant and removing it loses nothing. Dedup has to keep working here, or
// the guard above has simply disabled the feature.
func TestDedupRemovesAGenuineDuplicate(t *testing.T) {
victim := startFakeEcVolumeServer(t, dedupTestVolumeID, 0)
survivor := startFakeEcVolumeServer(t, dedupTestVolumeID, 0)
if err := newDedupTask().Execute(context.Background(), dedupParams(victim.address(), survivor.address(), 0)); err != nil {
t.Fatalf("dedup of a real duplicate should succeed, got %v", err)
}
if victim.has(dedupTestVolumeID, 0) {
t.Error("dedup left the duplicate in place; capacity is not reclaimed")
}
if !survivor.has(dedupTestVolumeID, 0) {
t.Error("dedup removed the surviving copy instead of the duplicate")
}
}
// An unreachable peer is not evidence of a surviving copy. Treating a failed
// query as "the other side has it" is how a network blip becomes data loss.
func TestDedupRefusesWhenThePeerCannotBeQueried(t *testing.T) {
holder := startFakeEcVolumeServer(t, dedupTestVolumeID, 0)
unreachable := startFakeEcVolumeServer(t, dedupTestVolumeID, 0)
unreachable.stop() // peer down; its inventory is unknown, not empty
err := newDedupTask().Execute(context.Background(), dedupParams(holder.address(), unreachable.address(), 0))
if holder.has(dedupTestVolumeID, 0) {
return // kept the shard while the peer's state is unknown
}
t.Fatalf("dedup deleted shard %d.0 while the peer was unreachable (deleted=%v, err=%v)",
dedupTestVolumeID, holder.deletedShards(), err)
}
// Volume ids are allocated cluster-wide, but the inventory RPC is keyed by
// volume id alone, so a server holding the same number for a different
// collection answers "yes, I have that shard" to a question about this one.
// Accepting that would delete the last real copy on the strength of an
// unrelated volume.
func TestDedupRefusesWhenTheKeepNodeHoldsAnotherCollection(t *testing.T) {
holder := startFakeEcVolumeServer(t, dedupTestVolumeID, 0)
// Same volume id, different collection: a plausible-looking but wrong match.
impostor := startFakeEcVolumeServerInCollection(t, "morro-agudo-01", dedupTestVolumeID, 0)
err := newDedupTask().Execute(context.Background(), dedupParams(holder.address(), impostor.address(), 0))
if holder.has(dedupTestVolumeID, 0) {
return // refused: the keep node's shard belongs to another collection
}
t.Fatalf("dedup deleted shard %d.0 after matching another collection's volume (deleted=%v, err=%v)",
dedupTestVolumeID, holder.deletedShards(), err)
}

View File

@@ -0,0 +1,331 @@
package ec_balance
import (
"context"
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
storagetypes "github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/topology/balancer"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// Detection builds an EC balance topology snapshot from the cluster's active
// topology, runs the shared ecbalancer planner, and converts the planned moves
// into worker task proposals. The balancing policy lives in
// weed/storage/erasure_coding/ecbalancer, shared with the shell ec.balance
// command so the two cannot drift.
func Detection(
ctx context.Context,
metrics []*types.VolumeHealthMetrics,
clusterInfo *types.ClusterInfo,
config base.TaskConfig,
maxResults int,
) ([]*types.TaskDetectionResult, bool, error) {
if !config.IsEnabled() {
return nil, false, nil
}
ecConfig := config.(*Config)
if maxResults < 0 {
maxResults = 0
}
if clusterInfo == nil || clusterInfo.ActiveTopology == nil {
return nil, false, fmt.Errorf("active topology not available for EC balance detection")
}
topoInfo := clusterInfo.ActiveTopology.GetTopologyInfo()
if topoInfo == nil {
return nil, false, fmt.Errorf("topology info not available")
}
allowedCollections, err := wildcard.CompileCollectionMatcher(ecConfig.CollectionFilter)
if err != nil {
return nil, false, err
}
topo, nodeCount, volumeRatio := buildBalancerTopology(topoInfo, ecConfig, allowedCollections)
if nodeCount < ecConfig.MinServerCount {
glog.V(1).Infof("EC balance: only %d servers, need at least %d", nodeCount, ecConfig.MinServerCount)
return nil, false, nil
}
replicaPlacement := resolveReplicaPlacement(ecConfig, clusterInfo)
if ctx != nil {
if err := ctx.Err(); err != nil {
return nil, false, err
}
}
// Canonical disk type for placement/execution: "hdd" -> "" (HardDriveType),
// matching the topology's disk keys and the volume server's move RPCs.
normalizedDiskType := storagetypes.ToDiskType(ecConfig.DiskType).String()
moves := ecbalancer.Plan(topo, ecbalancer.Options{
DiskType: normalizedDiskType,
ImbalanceThreshold: ecConfig.ImbalanceThreshold,
ReplicaPlacement: replicaPlacement,
Ratio: func(collection string) (int, int) {
return resolveECRatio(clusterInfo, collection)
},
// Prefer each volume's own heartbeat-reported ratio over the collection
// default so a mixed-ratio collection is spread per volume; 0 defers to
// resolveECRatio (and is the always-0 OSS case).
VolumeRatio: volumeRatio,
// Move incrementally across detection cycles rather than draining a rack
// in one batch; the scheduler re-evaluates each cycle.
GlobalMaxMovesPerRack: 10,
// Balance heterogeneous-capacity racks by fractional fullness.
GlobalUtilizationBased: true,
})
if len(moves) == 0 {
return nil, false, nil
}
hasMore := false
if maxResults > 0 && len(moves) > maxResults {
moves = moves[:maxResults]
hasMore = true
}
now := time.Now()
results := make([]*types.TaskDetectionResult, 0, len(moves))
for i, m := range moves {
taskID := fmt.Sprintf("ec_balance_%d_%d_%s_%s_%d_%d",
m.VolumeID, m.ShardID, m.SourceNode, m.TargetNode, now.UnixNano(), i)
results = append(results, &types.TaskDetectionResult{
TaskID: taskID,
TaskType: types.TaskTypeECBalance,
VolumeID: m.VolumeID,
Server: m.SourceNode,
Collection: m.Collection,
Priority: movePhasePriority(m.Phase),
Reason: fmt.Sprintf("EC shard %d.%d %s: %s → %s",
m.VolumeID, m.ShardID, m.Phase, m.SourceNode, m.TargetNode),
ScheduleAt: now,
TypedParams: &worker_pb.TaskParams{
TaskId: taskID,
VolumeId: m.VolumeID,
Collection: m.Collection,
Sources: []*worker_pb.TaskSource{{
Node: m.SourceNode,
DiskId: m.SourceDisk,
Rack: m.SourceRack,
ShardIds: []uint32{uint32(m.ShardID)},
}},
Targets: []*worker_pb.TaskTarget{{
Node: m.TargetNode,
DiskId: m.TargetDisk,
Rack: m.TargetRack,
ShardIds: []uint32{uint32(m.ShardID)},
}},
TaskParams: &worker_pb.TaskParams_EcBalanceParams{
EcBalanceParams: &worker_pb.EcBalanceTaskParams{
DiskType: normalizedDiskType,
DedupKeepNode: m.KeepNode,
TimeoutSeconds: 600,
IoBytePerSecond: ecConfig.IoBytePerSecond,
},
},
},
})
}
glog.V(1).Infof("EC balance detection: %d moves proposed", len(results))
return results, hasMore, nil
}
// buildBalancerTopology builds an ecbalancer.Topology from the master topology,
// applying the data-center, disk-type, and collection filters. Rack keys are
// dc:rack composites to avoid cross-DC name collisions. Per-disk free capacity
// is split evenly from the node total because the wire collapses same-type disks.
// Returns the topology, the number of eligible nodes (for MinServerCount), and a
// per-volume ratio lookup built from each shard's heartbeat (0,0 when unreported,
// e.g. always in OSS) which Plan prefers over the collection ratio for mixed-ratio
// clusters.
func buildBalancerTopology(topoInfo *master_pb.TopologyInfo, config *Config, allowedCollections *wildcard.CollectionMatcher) (*ecbalancer.Topology, int, func(collection string, vid uint32) (int, int)) {
topo := ecbalancer.NewTopology()
type volRatioKey struct {
collection string
vid uint32
}
volRatios := make(map[volRatioKey][2]int)
// Normalize the disk-type filter: "hdd" (and the default "") map to the
// HardDriveType, which the topology reports under the empty-string key. Keep a
// separate "filter requested" flag so a configured "hdd" still filters to HDD
// disks instead of being mistaken for "all disk types".
filterByDiskType := config.DiskType != ""
wantDiskType := storagetypes.ToDiskType(config.DiskType).String()
nodeCount := 0
for _, dc := range topoInfo.DataCenterInfos {
if config.DataCenterFilter != "" {
matchers := wildcard.CompileWildcardMatchers(config.DataCenterFilter)
if !wildcard.MatchesAnyWildcard(matchers, dc.Id) {
continue
}
}
for _, rack := range dc.RackInfos {
rackKey := dc.Id + ":" + rack.Id
for _, dn := range rack.DataNodeInfos {
freeSlots := 0
diskTypeOf := make(map[uint32]string) // physical disk_id -> disk type
diskShardCount := make(map[uint32]int)
fullDiskTypes := make(map[string]bool) // physically near-full, ineligible as a target
hasMatchingDisk := false
for diskType, diskInfo := range dn.DiskInfos {
if filterByDiskType && diskType != wantDiskType {
continue
}
hasMatchingDisk = true
// Don't place EC shards on a physically near-full disk, even if slot
// math says it has room (an over-set maxVolumeCount hides real
// fullness). Statfs free bytes already include EC shard files.
if balancer.DiskTooFullAfter(diskInfo.DiskTotalBytes, diskInfo.DiskFreeBytes, 0, balancer.DefaultMaxDiskUsagePercent) {
fullDiskTypes[diskType] = true
}
fs := int(diskInfo.MaxVolumeCount-diskInfo.VolumeCount)*erasure_coding.DataShardsCount - countEcShards(diskInfo.EcShardInfos)
if fs > 0 && !fullDiskTypes[diskType] {
freeSlots += fs
}
// Discover physical disks from regular volumes too, so an
// EC-empty disk is still a candidate destination.
for _, vi := range diskInfo.VolumeInfos {
if _, ok := diskTypeOf[vi.DiskId]; !ok {
diskTypeOf[vi.DiskId] = diskType
}
}
for _, eci := range diskInfo.EcShardInfos {
if _, ok := diskTypeOf[eci.DiskId]; !ok {
diskTypeOf[eci.DiskId] = diskType
}
// Disk occupancy counts ALL volumes' shards (capacity model),
// independent of the collection filter below.
diskShardCount[eci.DiskId] += erasure_coding.GetShardCount(eci)
}
}
if !hasMatchingDisk {
continue
}
node := topo.AddNode(dn.Id, dc.Id, rackKey, freeSlots)
// Group servers sharing a host so a volume's shards spread across
// machines, not just nodes (servers on one host are one fault domain).
node.SetHost(pb.NewServerAddressFromDataNode(dn).ToHost())
// Spread the node's free slots only over its non-full disks, so a
// physically full disk type doesn't dilute the share the usable disks
// advertise (its own disks get 0 below). The total is preserved.
nonFullDiskCount := 0
for _, dt := range diskTypeOf {
if !fullDiskTypes[dt] {
nonFullDiskCount++
}
}
perDiskFree := 0
if nonFullDiskCount > 0 && freeSlots > 0 {
perDiskFree = freeSlots / nonFullDiskCount
}
for diskID, diskType := range diskTypeOf {
diskFree := perDiskFree
if fullDiskTypes[diskType] {
diskFree = 0
}
node.AddDisk(diskID, diskType, diskFree, diskShardCount[diskID])
}
// Add shards only for volumes whose collection passes the filter;
// those are the volumes the planner will balance.
for diskType, diskInfo := range dn.DiskInfos {
if filterByDiskType && diskType != wantDiskType {
continue
}
for _, eci := range diskInfo.EcShardInfos {
if !allowedCollections.Matches(eci.Collection) {
continue
}
node.AddShards(eci.Id, eci.Collection, eci.DiskId, erasure_coding.ShardBits(eci.EcIndexBits))
if d, p := ecbalancer.VolumeShardRatio(eci); d > 0 || p > 0 {
volRatios[volRatioKey{eci.Collection, eci.Id}] = [2]int{d, p}
}
}
}
nodeCount++
}
}
}
volumeRatio := func(collection string, vid uint32) (int, int) {
r := volRatios[volRatioKey{collection, vid}]
return r[0], r[1]
}
return topo, nodeCount, volumeRatio
}
// resolveECRatio returns the (dataShards, parityShards) for a collection from the
// admin EC config snapshot when present, else the local default. This keeps the
// enterprise-only custom-ratio plumbing out of the shared planner.
func resolveECRatio(_ *types.ClusterInfo, _ string) (int, int) {
// Custom EC ratios are an enterprise feature; OSS uses the standard scheme.
return normalizeECShardCounts(0, 0)
}
// resolveReplicaPlacement picks the EC shard replica placement constraint: an
// explicit config value wins; otherwise it falls back to the master's default
// replication (matching the shell ec.balance default). A missing, invalid, or
// zero-replication value yields nil, meaning even spread / no constraint.
func resolveReplicaPlacement(ecConfig *Config, clusterInfo *types.ClusterInfo) *super_block.ReplicaPlacement {
clusterDefault := ""
if clusterInfo != nil {
clusterDefault = clusterInfo.DefaultReplicaPlacement
}
return super_block.ResolveReplicaPlacement(ecConfig.ReplicaPlacement, clusterDefault)
}
func normalizeECShardCounts(dataShards, parityShards int) (int, int) {
if dataShards <= 0 {
dataShards = erasure_coding.DataShardsCount
}
if parityShards <= 0 {
parityShards = erasure_coding.ParityShardsCount
}
return dataShards, parityShards
}
func countEcShards(ecShardInfos []*master_pb.VolumeEcShardInformationMessage) int {
count := 0
for _, eci := range ecShardInfos {
count += erasure_coding.GetShardCount(eci)
}
return count
}
func movePhasePriority(phase string) types.TaskPriority {
switch phase {
case "dedup":
return types.TaskPriorityHigh
case "cross_rack":
return types.TaskPriorityMedium
default:
return types.TaskPriorityLow
}
}

View File

@@ -0,0 +1,156 @@
package ec_balance
import (
"context"
"net"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// The EC balance policy itself is tested in the shared ecbalancer package; these
// tests cover the worker adapter: building the planner topology from the master
// topology (filters, capacity) and the Detection entry point.
func ecTopo(node1Collection string) *master_pb.TopologyInfo {
node1 := &master_pb.DataNodeInfo{
Id: "node1",
DiskInfos: map[string]*master_pb.DiskInfo{
"": {Type: "", MaxVolumeCount: 100, EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 100, Collection: node1Collection, DiskId: 0, EcIndexBits: 0x3FFF}, // 14 shards
}},
},
}
node2 := &master_pb.DataNodeInfo{
Id: "node2",
DiskInfos: map[string]*master_pb.DiskInfo{"": {Type: "", MaxVolumeCount: 100}},
}
return &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{
{Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{node1}},
{Id: "rack2", DataNodeInfos: []*master_pb.DataNodeInfo{node2}},
},
}},
}
}
func TestBuildBalancerTopology(t *testing.T) {
config := NewDefaultConfig()
topo, nodeCount, _ := buildBalancerTopology(ecTopo("col1"), config, nil)
if nodeCount != 2 {
t.Fatalf("nodeCount = %d, want 2", nodeCount)
}
moves := ecbalancer.Plan(topo, ecbalancer.Options{ImbalanceThreshold: 0.01})
if len(moves) == 0 {
t.Error("expected cross-rack moves for an all-on-one-rack volume")
}
}
// TestBuildBalancerTopologyGroupsByHost: two volume servers on host 10.0.0.1
// (different ports) plus three other hosts, a 10+4 volume concentrated on the
// 10.0.0.1 machine. Four machines is enough to spread within parity, so after
// planning the 10.0.0.1 machine must hold <=4 shards of the volume -- which only
// holds if its two ports are grouped into one machine (host wired into the build).
func TestBuildBalancerTopologyGroupsByHost(t *testing.T) {
mkNode := func(id string, bits uint32) *master_pb.DataNodeInfo {
di := &master_pb.DiskInfo{Type: "", MaxVolumeCount: 100}
if bits != 0 {
di.EcShardInfos = []*master_pb.VolumeEcShardInformationMessage{{Id: 100, Collection: "col1", DiskId: 0, EcIndexBits: bits}}
}
return &master_pb.DataNodeInfo{Id: id, DiskInfos: map[string]*master_pb.DiskInfo{"": di}}
}
topoInfo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{{Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{
mkNode("10.0.0.1:8080", 0x007F), // shards 0-6 on host 10.0.0.1
mkNode("10.0.0.1:8081", 0x3F80), // shards 7-13 on host 10.0.0.1
mkNode("10.0.0.2:8080", 0),
mkNode("10.0.0.3:8080", 0),
mkNode("10.0.0.4:8080", 0),
}}},
}},
}
topo, _, _ := buildBalancerTopology(topoInfo, NewDefaultConfig(), nil)
moves := ecbalancer.Plan(topo, ecbalancer.Options{ImbalanceThreshold: 0.01})
host := func(nodeID string) string { h, _, _ := net.SplitHostPort(nodeID); return h }
count := map[string]int{"10.0.0.1": 14}
for _, m := range moves {
if m.VolumeID != 100 {
continue
}
count[host(m.SourceNode)]--
if m.SourceNode != m.TargetNode { // non-dedup move
count[host(m.TargetNode)]++
}
}
if count["10.0.0.1"] > 4 {
t.Errorf("machine 10.0.0.1 holds %d shards of the volume after balancing, want <=4 (host grouping not applied)", count["10.0.0.1"])
}
}
func TestBuildBalancerTopologyCollectionFilter(t *testing.T) {
config := NewDefaultConfig()
config.CollectionFilter = "other" // does not match the volume's collection
allowed, err := wildcard.CompileCollectionMatcher(config.CollectionFilter)
if err != nil {
t.Fatalf("CompileCollectionMatcher: %v", err)
}
topo, nodeCount, _ := buildBalancerTopology(ecTopo("col1"), config, allowed)
if nodeCount != 2 {
t.Fatalf("nodeCount = %d, want 2", nodeCount)
}
if moves := ecbalancer.Plan(topo, ecbalancer.Options{ImbalanceThreshold: 0.01}); len(moves) != 0 {
t.Errorf("filtered-out collection should produce no moves, got %d", len(moves))
}
}
func TestDetectionDisabled(t *testing.T) {
config := NewDefaultConfig()
config.Enabled = false
results, hasMore, err := Detection(context.Background(), nil, nil, config, 0)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if hasMore {
t.Error("expected hasMore=false")
}
if len(results) != 0 {
t.Errorf("expected 0 results, got %d", len(results))
}
}
func TestDetectionNilTopology(t *testing.T) {
config := NewDefaultConfig()
clusterInfo := &types.ClusterInfo{ActiveTopology: nil}
if _, _, err := Detection(context.Background(), nil, clusterInfo, config, 0); err == nil {
t.Fatal("expected error for nil topology")
}
}
func TestMovePhasePriority(t *testing.T) {
cases := map[string]types.TaskPriority{
"dedup": types.TaskPriorityHigh,
"cross_rack": types.TaskPriorityMedium,
"within_rack": types.TaskPriorityLow,
"global": types.TaskPriorityLow,
}
for phase, want := range cases {
if got := movePhasePriority(phase); got != want {
t.Errorf("movePhasePriority(%q) = %v, want %v", phase, got, want)
}
}
}
// keep the erasure_coding import meaningful for future adapter tests
var _ = erasure_coding.DataShardsCount

View File

@@ -0,0 +1,230 @@
package ec_balance
import (
"context"
"fmt"
"math"
"sync/atomic"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation/volume_move"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
"github.com/seaweedfs/seaweedfs/weed/worker/types/base"
"google.golang.org/grpc"
)
// ECBalanceTask implements a single EC shard move operation.
// The move sequence — copy+mount on dest, verify the dest registered the
// shards, then unmount+delete on the source — is shared with the shell's
// ec.balance command via weed/operation/volume_move.
type ECBalanceTask struct {
*base.BaseTask
volumeID uint32
collection string
grpcDialOption grpc.DialOption
progress uint64 // atomic; stores float64 bits via math.Float64bits
reporting int32 // atomic; re-entry guard to prevent recursive reportProgress calls
}
// NewECBalanceTask creates a new EC balance task instance
func NewECBalanceTask(id string, volumeID uint32, collection string, grpcDialOption grpc.DialOption) *ECBalanceTask {
return &ECBalanceTask{
BaseTask: base.NewBaseTask(id, types.TaskTypeECBalance),
volumeID: volumeID,
collection: collection,
grpcDialOption: grpcDialOption,
}
}
// Execute performs the EC shard move operation.
func (t *ECBalanceTask) Execute(ctx context.Context, params *worker_pb.TaskParams) error {
if params == nil {
return fmt.Errorf("task parameters are required")
}
if len(params.Sources) == 0 || len(params.Targets) == 0 {
return fmt.Errorf("sources and targets are required for EC shard move")
}
if len(params.Sources) > 1 || len(params.Targets) > 1 {
return fmt.Errorf("batch EC shard moves not supported: got %d sources and %d targets, expected 1 each", len(params.Sources), len(params.Targets))
}
source := params.Sources[0]
target := params.Targets[0]
if len(source.ShardIds) == 0 || len(target.ShardIds) == 0 {
return fmt.Errorf("shard IDs are required in sources and targets")
}
sourceAddr := pb.ServerAddress(source.Node)
targetAddr := pb.ServerAddress(target.Node)
// Range-check before the uint8 narrowing in Uint32ToShardIds: a malformed
// id like 259 would otherwise alias shard 3 and copy/delete a real,
// unrelated shard.
if err := checkShardIdRange(source.ShardIds); err != nil {
return err
}
shardIds := erasure_coding.Uint32ToShardIds(source.ShardIds)
ecParams := params.GetEcBalanceParams()
// Apply configured timeout to the context for all RPC operations
if ecParams != nil && ecParams.TimeoutSeconds > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, time.Duration(ecParams.TimeoutSeconds)*time.Second)
defer cancel()
}
isDedupDelete := ecParams != nil && isDedupPhase(params)
// Guard against a same-node, cross-disk "move". The shared mover skips the
// copy when source and target addresses match, but the EC shard delete is
// node-wide (it removes the shard from every disk on the node), so this
// sequence would erase the shard after never copying it. EC shards also
// cannot be relocated between disks of one node via these RPCs, so such a
// move is meaningless. Reject it rather than lose data.
if source.Node == target.Node && source.DiskId != target.DiskId {
return fmt.Errorf("refusing same-node cross-disk EC shard move for volume %d shard(s) %v on %s (source disk %d, target disk %d): EC shard delete is node-wide and would erase the shard after a skipped copy",
params.VolumeId, source.ShardIds, source.Node, source.DiskId, target.DiskId)
}
glog.Infof("EC balance: moving shard(s) %v of volume %d from %s to %s",
source.ShardIds, params.VolumeId, source.Node, target.Node)
mover := volume_move.NewMover(t.grpcDialOption)
// For dedup, we only unmount+delete from source (no copy needed)
if isDedupDelete {
// Nothing is copied first, so the shard surviving elsewhere is the
// only thing making this safe — and the plan asserting so is not
// evidence. The topology can name a location that holds nothing, and
// deleting on that basis removes the last copy. Confirm the keep node
// has it before deleting here.
if err := t.verifyShardsOnKeepNode(ctx, params.VolumeId, ecParams.GetDedupKeepNode(), source.ShardIds); err != nil {
return err
}
t.reportProgress(25.0, "Removing duplicate EC shard")
if err := mover.RemoveEcShards(ctx, needle.VolumeId(params.VolumeId), t.collection, sourceAddr, shardIds); err != nil {
return fmt.Errorf("remove duplicate shard: %w", err)
}
t.reportProgress(100.0, "Duplicate shard removed")
return nil
}
err := mover.MoveEcShards(ctx, volume_move.EcShardMove{
VolumeId: needle.VolumeId(params.VolumeId),
Collection: params.Collection,
ShardIds: shardIds,
Source: sourceAddr,
Target: targetAddr,
TargetDisk: target.DiskId,
}, volume_move.EcMoveOptions{
IoBytePerSecond: ecParams.GetIoBytePerSecond(),
Progress: t.reportProgress,
})
if err != nil {
return err
}
glog.Infof("EC balance: successfully moved shard(s) %v of volume %d from %s to %s",
source.ShardIds, params.VolumeId, source.Node, target.Node)
return nil
}
// verifyShardsOnKeepNode confirms the node the plan wants to keep the shard on
// actually has every shard about to be deleted elsewhere, for this collection.
func (t *ECBalanceTask) verifyShardsOnKeepNode(ctx context.Context, volumeID uint32, keepNode string, shardIDs []uint32) error {
if keepNode == "" {
return fmt.Errorf("refusing dedup delete of volume %d shard(s) %v: no keep node recorded, so no surviving copy can be confirmed", volumeID, shardIDs)
}
if err := erasure_coding.VerifyShardsOnServer(ctx, t.collection, volumeID,
string(pb.ServerAddress(keepNode)), shardIDs, t.grpcDialOption); err != nil {
return fmt.Errorf("refusing dedup delete: %w", err)
}
return nil
}
// Validate validates the task parameters.
// ECBalanceTask handles exactly one source→target shard move per execution.
func (t *ECBalanceTask) Validate(params *worker_pb.TaskParams) error {
if params == nil {
return fmt.Errorf("ECBalanceTask.Validate: TaskParams are required")
}
if len(params.Sources) != 1 {
return fmt.Errorf("ECBalanceTask.Validate: expected exactly 1 source, got %d", len(params.Sources))
}
if len(params.Targets) != 1 {
return fmt.Errorf("ECBalanceTask.Validate: expected exactly 1 target, got %d", len(params.Targets))
}
if len(params.Sources[0].ShardIds) == 0 {
return fmt.Errorf("ECBalanceTask.Validate: Sources[0].ShardIds is empty")
}
if len(params.Targets[0].ShardIds) == 0 {
return fmt.Errorf("ECBalanceTask.Validate: Targets[0].ShardIds is empty")
}
if err := checkShardIdRange(params.Sources[0].ShardIds); err != nil {
return fmt.Errorf("ECBalanceTask.Validate: %v", err)
}
if err := checkShardIdRange(params.Targets[0].ShardIds); err != nil {
return fmt.Errorf("ECBalanceTask.Validate: %v", err)
}
// A same-node, cross-disk move is unsafe: the node-wide EC shard delete would
// erase the shard after the skipped same-address copy. Such a move cannot be
// expressed by these RPCs anyway. Dedup (same node and disk) is allowed.
if params.Sources[0].Node == params.Targets[0].Node && params.Sources[0].DiskId != params.Targets[0].DiskId {
return fmt.Errorf("ECBalanceTask.Validate: refusing same-node cross-disk move on %s (source disk %d, target disk %d): EC shard delete is node-wide",
params.Sources[0].Node, params.Sources[0].DiskId, params.Targets[0].DiskId)
}
return nil
}
// EstimateTime estimates the time for an EC shard move
func (t *ECBalanceTask) EstimateTime(params *worker_pb.TaskParams) time.Duration {
return 30 * time.Second
}
// GetProgress returns current progress
func (t *ECBalanceTask) GetProgress() float64 {
return math.Float64frombits(atomic.LoadUint64(&t.progress))
}
// reportProgress updates the stored progress and reports it via the callback
func (t *ECBalanceTask) reportProgress(progress float64, stage string) {
if !atomic.CompareAndSwapInt32(&t.reporting, 0, 1) {
return
}
defer atomic.StoreInt32(&t.reporting, 0)
atomic.StoreUint64(&t.progress, math.Float64bits(progress))
t.ReportProgressWithStage(progress, stage)
glog.Infof("EC balance volume %d: [%.2f] %s", t.volumeID, progress, stage)
}
// checkShardIdRange rejects shard ids that would alias a real shard when
// narrowed to the uint8 ShardId (e.g. 259 → 3).
func checkShardIdRange(ids []uint32) error {
for _, id := range ids {
if id >= erasure_coding.MaxShardCount {
return fmt.Errorf("shard id %d out of range (max %d)", id, erasure_coding.MaxShardCount-1)
}
}
return nil
}
// isDedupPhase checks if this is a dedup-phase task: an unmount+delete on a
// single location, encoded by detection as source==target on the same node AND
// the same disk. Comparing the disk too is essential — VolumeEcShardsDelete is
// node-wide (it removes the shard from every disk on the node), so a same-node
// but cross-disk request must NOT be treated as a benign dedup; see Validate
// and Execute, which reject it outright.
func isDedupPhase(params *worker_pb.TaskParams) bool {
if len(params.Sources) > 0 && len(params.Targets) > 0 {
s, t := params.Sources[0], params.Targets[0]
return s.Node == t.Node && s.DiskId == t.DiskId
}
return false
}

View File

@@ -0,0 +1,27 @@
package ec_balance
import (
"context"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
)
func TestECBalanceTaskRejectsOutOfRangeShardIds(t *testing.T) {
// ShardId is a uint8; an unchecked id like 259 would alias shard 3 and
// copy/delete a real, unrelated shard.
params := &worker_pb.TaskParams{
VolumeId: 7,
Sources: []*worker_pb.TaskSource{{Node: "src:8080", ShardIds: []uint32{259}}},
Targets: []*worker_pb.TaskTarget{{Node: "dst:8080", ShardIds: []uint32{259}}},
}
task := NewECBalanceTask("t1", 7, "c1", nil)
if err := task.Validate(params); err == nil || !strings.Contains(err.Error(), "out of range") {
t.Fatalf("Validate: expected out-of-range rejection, got: %v", err)
}
if err := task.Execute(context.Background(), params); err == nil || !strings.Contains(err.Error(), "out of range") {
t.Fatalf("Execute: expected out-of-range rejection, got: %v", err)
}
}

View File

@@ -0,0 +1,140 @@
package ec_balance
import (
"context"
"fmt"
"net"
"sync"
"testing"
"google.golang.org/grpc"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
)
// fakeEcVolumeServer is a volume server that only knows about EC shard
// inventory. It exists so the destructive half of a balance job can be driven
// in-process: what a real volume server has on disk is the thing the master's
// topology can be wrong about, and being wrong about it is what makes a delete
// unsafe. Holding that state here lets a test say "the topology claims this,
// the disk holds that" and watch what the job does.
type fakeEcVolumeServer struct {
volume_server_pb.UnimplementedVolumeServerServer
mu sync.Mutex
// shards actually present on this server's disks, per volume.
shards map[uint32]map[uint32]bool
// collection the held volume belongs to. Reported back on the inventory so
// a caller can tell this server's volume N from another collection's N.
collection string
// deleted/unmounted record what the job asked for, so a test can assert on
// the request even when it was a no-op against the inventory.
deleted []string
unmounted []string
// grpcAddr is host:port; the code under test dials by address string.
grpcAddr string
stop func()
}
func (f *fakeEcVolumeServer) VolumeEcShardsInfo(ctx context.Context, req *volume_server_pb.VolumeEcShardsInfoRequest) (*volume_server_pb.VolumeEcShardsInfoResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
resp := &volume_server_pb.VolumeEcShardsInfoResponse{}
for sid := range f.shards[req.VolumeId] {
resp.EcShardInfos = append(resp.EcShardInfos, &volume_server_pb.EcShardInfo{
VolumeId: req.VolumeId,
ShardId: sid,
Collection: f.collection,
})
}
return resp, nil
}
func (f *fakeEcVolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_server_pb.VolumeEcShardsDeleteRequest) (*volume_server_pb.VolumeEcShardsDeleteResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
for _, sid := range req.ShardIds {
f.deleted = append(f.deleted, fmt.Sprintf("%d.%d", req.VolumeId, sid))
delete(f.shards[req.VolumeId], sid)
}
return &volume_server_pb.VolumeEcShardsDeleteResponse{}, nil
}
func (f *fakeEcVolumeServer) VolumeEcShardsUnmount(ctx context.Context, req *volume_server_pb.VolumeEcShardsUnmountRequest) (*volume_server_pb.VolumeEcShardsUnmountResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
for _, sid := range req.ShardIds {
f.unmounted = append(f.unmounted, fmt.Sprintf("%d.%d", req.VolumeId, sid))
}
return &volume_server_pb.VolumeEcShardsUnmountResponse{}, nil
}
// has reports whether the shard is still on this server's disks.
func (f *fakeEcVolumeServer) has(volumeID, shardID uint32) bool {
f.mu.Lock()
defer f.mu.Unlock()
return f.shards[volumeID][shardID]
}
func (f *fakeEcVolumeServer) deletedShards() []string {
f.mu.Lock()
defer f.mu.Unlock()
return append([]string(nil), f.deleted...)
}
// startFakeEcVolumeServer listens on a loopback port and serves the EC subset
// of the volume server API. present lists the shards this server really holds.
func startFakeEcVolumeServer(t *testing.T, volumeID uint32, present ...uint32) *fakeEcVolumeServer {
return startFakeEcVolumeServerInCollection(t, dedupTestCollection, volumeID, present...)
}
// startFakeEcVolumeServerInCollection is the same, for a named collection, so a
// test can stand up two servers holding the same volume id in different ones.
func startFakeEcVolumeServerInCollection(t *testing.T, collection string, volumeID uint32, present ...uint32) *fakeEcVolumeServer {
t.Helper()
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
f := &fakeEcVolumeServer{
shards: map[uint32]map[uint32]bool{volumeID: {}},
collection: collection,
grpcAddr: listener.Addr().String(),
}
for _, sid := range present {
f.shards[volumeID][sid] = true
}
server := grpc.NewServer()
volume_server_pb.RegisterVolumeServerServer(server, f)
go func() { _ = server.Serve(listener) }()
f.stop = server.Stop
t.Cleanup(server.Stop)
return f
}
// address returns a node id in the "host:httpPort.grpcPort" form the cluster
// uses (e.g. mg01-s3-intelistor-15:8087.18087), so ToGrpcAddress resolves to
// this fake's listener. The http port is unused and only has to parse.
func (f *fakeEcVolumeServer) address() string {
host, port, err := net.SplitHostPort(f.grpcAddr)
if err != nil {
return f.grpcAddr
}
return fmt.Sprintf("%s:1.%s", host, port)
}
// shardBitsOf is a small readability helper for asserting inventory.
func shardBitsOf(f *fakeEcVolumeServer, volumeID uint32) erasure_coding.ShardBits {
f.mu.Lock()
defer f.mu.Unlock()
var b erasure_coding.ShardBits
for sid := range f.shards[volumeID] {
b = b.Set(erasure_coding.ShardId(sid))
}
return b
}

View File

@@ -0,0 +1,335 @@
package ec_balance
import (
"context"
"sort"
"testing"
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// In-process (no real cluster) tests of the detection/planning path against a
// topology shaped the way the master reports a multi-disk cluster: several
// same-type physical disks on a node collapse into a single DiskInfo, with each
// shard's real DiskId surviving only in the per-shard records (issue 9593). They
// run the master-wire-format -> ActiveTopology -> Detection path, then simulate
// executing the planned moves with the volume server's actual semantics
// (VolumeEcShardsDelete is node-wide) to assert no EC shard is ever lost. The
// real-cluster end-to-end equivalent lives in test/erasure_coding.
// nodeSpec describes one volume server: its rack and the volume's shards per
// physical disk (diskID -> shard ids).
type nodeSpec struct {
id string
rack string
disks map[uint32][]int
}
const integDisksPerNode = 6
// buildMasterTopology renders nodeSpecs into a *master_pb.TopologyInfo exactly as
// the master would: one DiskInfo per node keyed by disk type (""), carrying one
// EcShardInfo per (volume, physical disk) plus regular VolumeInfos so every
// physical disk is discoverable even when it holds no EC shards.
func buildMasterTopology(collection string, vid uint32, maxVolPerDisk int, specs []nodeSpec) *master_pb.TopologyInfo {
rackByID := map[string]*master_pb.RackInfo{}
var rackOrder []string
for _, spec := range specs {
var ecShards []*master_pb.VolumeEcShardInformationMessage
for diskID, shards := range spec.disks {
var bits erasure_coding.ShardBits
for _, s := range shards {
bits = bits.Set(erasure_coding.ShardId(s))
}
ecShards = append(ecShards, &master_pb.VolumeEcShardInformationMessage{
Id: vid,
Collection: collection,
DiskId: diskID,
EcIndexBits: uint32(bits),
})
}
// Expose all physical disks (incl. EC-empty ones) via regular volumes.
var volInfos []*master_pb.VolumeInformationMessage
for d := uint32(0); d < integDisksPerNode; d++ {
volInfos = append(volInfos, &master_pb.VolumeInformationMessage{Id: 90000 + d, DiskId: d})
}
dn := &master_pb.DataNodeInfo{
Id: spec.id,
DiskInfos: map[string]*master_pb.DiskInfo{
"": {
Type: "",
MaxVolumeCount: int64(maxVolPerDisk * integDisksPerNode),
VolumeCount: int64(integDisksPerNode),
EcShardInfos: ecShards,
VolumeInfos: volInfos,
},
},
}
r, ok := rackByID[spec.rack]
if !ok {
r = &master_pb.RackInfo{Id: spec.rack}
rackByID[spec.rack] = r
rackOrder = append(rackOrder, spec.rack)
}
r.DataNodeInfos = append(r.DataNodeInfos, dn)
}
dc := &master_pb.DataCenterInfo{Id: "dc1"}
for _, rk := range rackOrder {
dc.RackInfos = append(dc.RackInfos, rackByID[rk])
}
return &master_pb.TopologyInfo{Id: "integ", DataCenterInfos: []*master_pb.DataCenterInfo{dc}}
}
// runDetection wires the topology through a real ActiveTopology and runs Detection.
func runDetection(t *testing.T, topoInfo *master_pb.TopologyInfo, cfg *Config) []*types.TaskDetectionResult {
t.Helper()
at := topology.NewActiveTopology(0)
if err := at.UpdateTopology(topoInfo); err != nil {
t.Fatalf("UpdateTopology: %v", err)
}
results, _, err := Detection(context.Background(), nil, &types.ClusterInfo{ActiveTopology: at}, cfg, 0)
if err != nil {
t.Fatalf("Detection: %v", err)
}
return results
}
// shardModel is nodeID -> diskID -> set of shard ids for one volume.
type shardModel map[string]map[uint32]map[int]bool
func modelFromSpecs(specs []nodeSpec) shardModel {
m := make(shardModel)
for _, spec := range specs {
m[spec.id] = make(map[uint32]map[int]bool)
for diskID, shards := range spec.disks {
set := make(map[int]bool)
for _, s := range shards {
set[s] = true
}
m[spec.id][diskID] = set
}
}
return m
}
func (m shardModel) distinctShards() []int {
seen := map[int]bool{}
for _, disks := range m {
for _, set := range disks {
for s := range set {
seen[s] = true
}
}
}
out := make([]int, 0, len(seen))
for s := range seen {
out = append(out, s)
}
sort.Ints(out)
return out
}
// applyMovesRealistically replays planned moves with the volume server's actual
// behavior: a move copies the shard to the destination disk, then deletes it on
// the source node — and EC shard delete is node-wide (removes the shard from
// every disk on the source node). A dedup move (same node+disk) is delete-only.
func (m shardModel) apply(results []*types.TaskDetectionResult) {
nodeWideDelete := func(node string, shard int) {
for diskID := range m[node] {
delete(m[node][diskID], shard)
}
}
for _, r := range results {
p := r.TypedParams
if p == nil || len(p.Sources) == 0 || len(p.Targets) == 0 {
continue
}
src, dst := p.Sources[0], p.Targets[0]
if len(src.ShardIds) == 0 {
continue
}
shard := int(src.ShardIds[0])
dedup := src.Node == dst.Node && src.DiskId == dst.DiskId
if !dedup {
if m[dst.Node] == nil {
m[dst.Node] = make(map[uint32]map[int]bool)
}
if m[dst.Node][dst.DiskId] == nil {
m[dst.Node][dst.DiskId] = make(map[int]bool)
}
m[dst.Node][dst.DiskId][shard] = true
}
nodeWideDelete(src.Node, shard)
}
}
// TestMultiDiskBalanceNeverLosesShards is the core regression for
// issue 9593: a freshly-encoded volume on a 3-node, 6-disk-per-node cluster,
// balanced and then concentrated, must never lose a shard when the planned moves
// are executed with real node-wide-delete semantics.
func TestMultiDiskBalanceNeverLosesShards(t *testing.T) {
cfg := NewDefaultConfig()
cfg.ImbalanceThreshold = 0.0 // balance to even; surface any move the planner makes
cases := []struct {
name string
specs []nodeSpec
}{
{
// Healthy post-encode spread: 14 shards across 3 nodes (5/5/4), each
// node spreading its shards over distinct physical disks.
name: "balanced across disks",
specs: []nodeSpec{
{id: "n1", rack: "rack1", disks: map[uint32][]int{0: {0}, 1: {1}, 2: {2}, 3: {3}, 4: {4}}},
{id: "n2", rack: "rack1", disks: map[uint32][]int{0: {5}, 1: {6}, 2: {7}, 3: {8}, 4: {9}}},
{id: "n3", rack: "rack1", disks: map[uint32][]int{0: {10}, 1: {11}, 2: {12}, 3: {13}}},
},
},
{
// Worst case from the report: all 14 shards landed on one node's disks;
// the balancer must redistribute without losing any.
name: "concentrated on one node",
specs: []nodeSpec{
{id: "n1", rack: "rack1", disks: map[uint32][]int{0: {0, 1, 2}, 1: {3, 4, 5}, 2: {6, 7}, 3: {8, 9}, 4: {10, 11}, 5: {12, 13}}},
{id: "n2", rack: "rack2", disks: map[uint32][]int{}},
{id: "n3", rack: "rack3", disks: map[uint32][]int{}},
},
},
{
// Multi-rack healthy spread.
name: "spread across racks",
specs: []nodeSpec{
{id: "n1", rack: "rack1", disks: map[uint32][]int{0: {0, 1}, 1: {2, 3}, 2: {4}}},
{id: "n2", rack: "rack2", disks: map[uint32][]int{0: {5, 6}, 1: {7, 8}, 2: {9}}},
{id: "n3", rack: "rack3", disks: map[uint32][]int{0: {10, 11}, 1: {12, 13}}},
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
topoInfo := buildMasterTopology("col1", 28, 50, tc.specs)
results := runDetection(t, topoInfo, cfg)
model := modelFromSpecs(tc.specs)
before := model.distinctShards()
for _, r := range results {
// No move may be a same-node cross-disk move (node-wide delete
// would erase the shard after a skipped copy).
p := r.TypedParams
s, d := p.Sources[0], p.Targets[0]
if s.Node == d.Node && s.DiskId != d.DiskId {
t.Errorf("unsafe same-node cross-disk move: vol %d shard %v on %s disk %d->%d", p.VolumeId, s.ShardIds, s.Node, s.DiskId, d.DiskId)
}
// Source disk must actually hold the shard being moved.
shard := int(s.ShardIds[0])
if !model[s.Node][s.DiskId][shard] && s.Node != d.Node {
t.Errorf("move sources vol %d shard %d from %s disk %d, which does not hold it", p.VolumeId, shard, s.Node, s.DiskId)
}
}
model.apply(results)
after := model.distinctShards()
if len(after) != len(before) {
t.Errorf("[%s] shard loss: had %v (%d), now %v (%d) after %d moves",
tc.name, before, len(before), after, len(after), len(results))
}
})
}
}
// TestConcentratedVolumeSpreadsAcrossNodesAndDisks asserts the
// remediation actually happens: a one-node-concentrated volume is redistributed
// to the other racks, landing on multiple distinct destination disks.
func TestConcentratedVolumeSpreadsAcrossNodesAndDisks(t *testing.T) {
cfg := NewDefaultConfig()
cfg.ImbalanceThreshold = 0.0
specs := []nodeSpec{
{id: "n1", rack: "rack1", disks: map[uint32][]int{0: {0, 1, 2}, 1: {3, 4, 5}, 2: {6, 7}, 3: {8, 9}, 4: {10, 11}, 5: {12, 13}}},
{id: "n2", rack: "rack2", disks: map[uint32][]int{}},
{id: "n3", rack: "rack3", disks: map[uint32][]int{}},
}
results := runDetection(t, buildMasterTopology("col1", 28, 50, specs), cfg)
if len(results) == 0 {
t.Fatal("expected redistribution moves for a one-node-concentrated volume")
}
destNodes := map[string]bool{}
destDisksByNode := map[string]map[uint32]bool{}
for _, r := range results {
d := r.TypedParams.Targets[0]
if d.Node == "n1" {
continue
}
destNodes[d.Node] = true
if destDisksByNode[d.Node] == nil {
destDisksByNode[d.Node] = map[uint32]bool{}
}
destDisksByNode[d.Node][d.DiskId] = true
}
if len(destNodes) < 2 {
t.Errorf("shards spread to only %d destination nodes, want both other racks", len(destNodes))
}
for node, disks := range destDisksByNode {
if len(disks) < 2 {
t.Errorf("destination %s received shards on only %d disk(s); expected spread across disks: %v", node, len(disks), disks)
}
}
}
// TestBuildBalancerTopologyNormalizesHddDiskType guards the disk-type filter:
// the master reports default-HDD disks under the empty-string key, so a config of
// "hdd" must match them (not filter everything out), while "ssd" must exclude them.
func TestBuildBalancerTopologyNormalizesHddDiskType(t *testing.T) {
specs := []nodeSpec{
{id: "n1", rack: "r1", disks: map[uint32][]int{0: {0, 1}}},
{id: "n2", rack: "r1", disks: map[uint32][]int{0: {2}}},
}
topoInfo := buildMasterTopology("c", 100, 50, specs)
if _, n, _ := buildBalancerTopology(topoInfo, &Config{DiskType: "hdd"}, nil); n != 2 {
t.Errorf("disk_type=hdd matched %d nodes on an all-HDD cluster, want 2 (hdd must map to the empty HDD key)", n)
}
if _, n, _ := buildBalancerTopology(topoInfo, &Config{DiskType: ""}, nil); n != 2 {
t.Errorf("disk_type=empty matched %d nodes, want 2 (all)", n)
}
if _, n, _ := buildBalancerTopology(topoInfo, &Config{DiskType: "ssd"}, nil); n != 0 {
t.Errorf("disk_type=ssd matched %d nodes on an all-HDD cluster, want 0", n)
}
}
// TestResolveReplicaPlacementFallsBackToMasterDefault verifies the worker mirrors
// the shell: explicit config wins, otherwise the master's default replication is
// the fallback, and an empty or zero-replication value means no constraint.
func TestResolveReplicaPlacementFallsBackToMasterDefault(t *testing.T) {
cases := []struct {
name string
configRP string
defaultRP string
wantApplied bool
}{
{"explicit config used", "010", "", true},
{"explicit config wins over default", "010", "100", true},
{"falls back to master default", "", "010", true},
{"zero master default = no constraint", "", "000", false},
{"empty everywhere = no constraint", "", "", false},
{"invalid value ignored", "", "nonsense", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
rp := resolveReplicaPlacement(&Config{ReplicaPlacement: tc.configRP},
&types.ClusterInfo{DefaultReplicaPlacement: tc.defaultRP})
if (rp != nil) != tc.wantApplied {
t.Errorf("config=%q default=%q: applied=%v, want %v", tc.configRP, tc.defaultRP, rp != nil, tc.wantApplied)
}
})
}
}

View File

@@ -0,0 +1,58 @@
package ec_balance
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
)
// These cover the worker execution layer (ec_balance_task.go). The shard-balance
// policy and multi-disk placement are tested in the shared ecbalancer package.
// TestValidateRejectsSameNodeCrossDiskMove covers the data-loss trap of a
// same-node, cross-disk shard move. copyAndMountShard skips the copy when source
// and target addresses match, but VolumeEcShardsDelete is node-wide, so executing
// such a move would erase the shard. Validate must reject it, while allowing a
// cross-node move and a same-node/same-disk dedup.
func TestValidateRejectsSameNodeCrossDiskMove(t *testing.T) {
task := NewECBalanceTask("t1", 100, "col1", nil)
mk := func(srcNode string, srcDisk uint32, dstNode string, dstDisk uint32) *worker_pb.TaskParams {
return &worker_pb.TaskParams{
VolumeId: 100,
Sources: []*worker_pb.TaskSource{{Node: srcNode, DiskId: srcDisk, ShardIds: []uint32{3}}},
Targets: []*worker_pb.TaskTarget{{Node: dstNode, DiskId: dstDisk, ShardIds: []uint32{3}}},
}
}
if err := task.Validate(mk("node1", 0, "node1", 3)); err == nil {
t.Error("Validate accepted a same-node cross-disk move; it must reject it to avoid node-wide delete data loss")
}
if err := task.Validate(mk("node1", 0, "node2", 0)); err != nil {
t.Errorf("Validate rejected a legitimate cross-node move: %v", err)
}
if err := task.Validate(mk("node1", 2, "node1", 2)); err != nil {
t.Errorf("Validate rejected a same-node/same-disk dedup: %v", err)
}
}
// TestIsDedupPhaseRequiresSameDisk confirms dedup classification keys on both
// node and disk, so a same-node cross-disk request is never silently routed to
// the unmount+delete path.
func TestIsDedupPhaseRequiresSameDisk(t *testing.T) {
withParams := func(srcDisk, dstDisk uint32) *worker_pb.TaskParams {
return &worker_pb.TaskParams{
Sources: []*worker_pb.TaskSource{{Node: "node1", DiskId: srcDisk}},
Targets: []*worker_pb.TaskTarget{{Node: "node1", DiskId: dstDisk}},
TaskParams: &worker_pb.TaskParams_EcBalanceParams{
EcBalanceParams: &worker_pb.EcBalanceTaskParams{},
},
}
}
if !isDedupPhase(withParams(2, 2)) {
t.Error("same node and disk should be a dedup phase")
}
if isDedupPhase(withParams(0, 3)) {
t.Error("same node but different disk must NOT be a dedup phase")
}
}

View File

@@ -0,0 +1,687 @@
package ec_balance
import (
"context"
"fmt"
"math"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
"github.com/seaweedfs/seaweedfs/weed/util"
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
"google.golang.org/grpc"
"google.golang.org/protobuf/proto"
)
const (
ecBalanceMinImbalanceThreshold = 0.05
ecBalanceMaxImbalanceThreshold = 0.5
ecBalanceMinServerCount = 2
)
func init() {
pluginworker.RegisterHandler(pluginworker.HandlerFactory{
JobType: "ec_balance",
Category: pluginworker.CategoryDefault,
Aliases: []string{"ec-balance", "ec.balance", "ec_shard_balance"},
Build: func(opts pluginworker.HandlerBuildOptions) (pluginworker.JobHandler, error) {
return NewECBalanceHandler(opts.GrpcDialOption), nil
},
})
}
type ecBalanceWorkerConfig struct {
TaskConfig *Config
}
// ECBalanceHandler is the plugin job handler for EC shard balancing.
type ECBalanceHandler struct {
grpcDialOption grpc.DialOption
}
func NewECBalanceHandler(grpcDialOption grpc.DialOption) *ECBalanceHandler {
return &ECBalanceHandler{grpcDialOption: grpcDialOption}
}
func (h *ECBalanceHandler) Capability() *plugin_pb.JobTypeCapability {
return &plugin_pb.JobTypeCapability{
JobType: "ec_balance",
CanDetect: true,
CanExecute: true,
MaxDetectionConcurrency: 1,
MaxExecutionConcurrency: 3,
DisplayName: "EC Shard Balance",
Description: "Balance EC shard distribution across racks and nodes",
Weight: 60,
}
}
func (h *ECBalanceHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
return &plugin_pb.JobTypeDescriptor{
JobType: "ec_balance",
DisplayName: "EC Shard Balance",
Description: "Detect and execute EC shard rebalancing across the cluster",
Icon: "fas fa-balance-scale-left",
DescriptorVersion: 1,
AdminConfigForm: &plugin_pb.ConfigForm{
FormId: "ec-balance-admin",
Title: "EC Shard Balance Admin Config",
Description: "Admin-side controls for EC shard balance detection scope.",
Sections: []*plugin_pb.ConfigSection{
{
SectionId: "scope",
Title: "Scope",
Description: "Optional filters applied before EC shard balance detection.",
Fields: []*plugin_pb.ConfigField{
{
Name: "collection_filter",
Label: "Collection Filter",
Description: "Only balance EC shards in matching collections. Comma-separated list of names, wildcards, or regex patterns.",
Placeholder: "all collections",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
},
{
Name: "data_center_filter",
Label: "Data Center Filter",
Description: "Only balance within matching data centers (wildcard supported).",
Placeholder: "all data centers",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
},
{
Name: "disk_type",
Label: "Disk Type",
Description: "Only balance EC shards on this disk type (hdd, ssd, or empty for all).",
Placeholder: "all disk types",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
},
{
Name: "shard_replica_placement",
Label: "Shard Replica Placement",
Description: "EC shard replica placement constraint (e.g. 020); empty uses the master default replication (even spread only when that default is empty or zero). Mirrors the ec.balance -shardReplicaPlacement flag.",
Placeholder: "master default",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
},
},
},
},
DefaultValues: map[string]*plugin_pb.ConfigValue{
"collection_filter": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""}},
"data_center_filter": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""}},
"disk_type": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""}},
"shard_replica_placement": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""}},
},
},
WorkerConfigForm: &plugin_pb.ConfigForm{
FormId: "ec-balance-worker",
Title: "EC Shard Balance Worker Config",
Description: "Worker-side detection thresholds and execution controls.",
Sections: []*plugin_pb.ConfigSection{
{
SectionId: "thresholds",
Title: "Detection Thresholds",
Description: "Controls for when EC shard balance jobs should be proposed.",
Fields: []*plugin_pb.ConfigField{
{
Name: "imbalance_threshold",
Label: "Imbalance Threshold",
Description: "Minimum shard count imbalance ratio to trigger balancing (0.0-1.0).",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_DOUBLE,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
Required: true,
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: ecBalanceMinImbalanceThreshold}},
MaxValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: ecBalanceMaxImbalanceThreshold}},
},
{
Name: "min_server_count",
Label: "Minimum Server Count",
Description: "Minimum servers required for EC shard balancing.",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
Required: true,
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: ecBalanceMinServerCount}},
},
{
Name: "io_byte_per_second",
Label: "Shard Copy IO Limit (bytes/sec)",
Description: "Limit each EC shard copy's rate in bytes per second. 0 falls back to each volume server's own maintenance rate.",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
},
{
Name: "preferred_tags",
Label: "Preferred Tags",
Description: "Comma-separated disk tags to prioritize for shard placement, ordered by preference.",
Placeholder: "fast,ssd",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
},
},
},
},
DefaultValues: map[string]*plugin_pb.ConfigValue{
"imbalance_threshold": {Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.2}},
"min_server_count": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 3}},
"io_byte_per_second": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
"preferred_tags": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""}},
},
},
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
Enabled: true,
DetectionIntervalMinutes: 30,
DetectionTimeoutSeconds: 300,
MaxJobsPerDetection: 500,
GlobalExecutionConcurrency: 16,
PerWorkerExecutionConcurrency: 4,
RetryLimit: 1,
RetryBackoffSeconds: 30,
JobTypeMaxRuntimeSeconds: 1800,
ExecutionTimeoutSeconds: 1800,
},
WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{
"imbalance_threshold": {Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.2}},
"min_server_count": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 3}},
"io_byte_per_second": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
"preferred_tags": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""}},
},
}
}
func (h *ECBalanceHandler) Detect(
ctx context.Context,
request *plugin_pb.RunDetectionRequest,
sender pluginworker.DetectionSender,
) error {
if request == nil {
return fmt.Errorf("run detection request is nil")
}
if sender == nil {
return fmt.Errorf("detection sender is nil")
}
if request.JobType != "" && request.JobType != "ec_balance" {
return fmt.Errorf("job type %q is not handled by ec_balance worker", request.JobType)
}
workerConfig := deriveECBalanceWorkerConfig(request.GetWorkerConfigValues())
// Apply admin-side scope filters
collectionFilter := strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "collection_filter", ""))
if collectionFilter != "" {
workerConfig.TaskConfig.CollectionFilter = collectionFilter
}
dcFilter := strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "data_center_filter", ""))
if dcFilter != "" {
workerConfig.TaskConfig.DataCenterFilter = dcFilter
}
diskType := strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "disk_type", ""))
if diskType != "" {
workerConfig.TaskConfig.DiskType = diskType
}
replicaPlacement := strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "shard_replica_placement", ""))
if replicaPlacement != "" {
workerConfig.TaskConfig.ReplicaPlacement = replicaPlacement
}
masters := make([]string, 0)
if request.ClusterContext != nil {
masters = append(masters, request.ClusterContext.MasterGrpcAddresses...)
}
metrics, activeTopology, err := h.collectVolumeMetrics(ctx, masters, collectionFilter)
if err != nil {
return err
}
clusterInfo := &workertypes.ClusterInfo{
ActiveTopology: activeTopology,
DefaultReplicaPlacement: pluginworker.FetchDefaultReplicaPlacement(ctx, masters, h.grpcDialOption),
}
maxResults := int(request.MaxResults)
if maxResults < 0 {
maxResults = 0
}
results, hasMore, err := Detection(ctx, metrics, clusterInfo, workerConfig.TaskConfig, maxResults)
if err != nil {
return err
}
if traceErr := emitECBalanceDecisionTrace(sender, workerConfig.TaskConfig, results, maxResults, hasMore); traceErr != nil {
glog.Warningf("Plugin worker failed to emit ec_balance detection trace: %v", traceErr)
}
proposals := make([]*plugin_pb.JobProposal, 0, len(results))
for _, result := range results {
proposal, proposalErr := buildECBalanceProposal(result)
if proposalErr != nil {
glog.Warningf("Plugin worker skip invalid ec_balance proposal: %v", proposalErr)
continue
}
proposals = append(proposals, proposal)
}
if err := sender.SendProposals(&plugin_pb.DetectionProposals{
JobType: "ec_balance",
Proposals: proposals,
HasMore: hasMore,
}); err != nil {
return err
}
return sender.SendComplete(&plugin_pb.DetectionComplete{
JobType: "ec_balance",
Success: true,
TotalProposals: int32(len(proposals)),
})
}
func (h *ECBalanceHandler) Execute(
ctx context.Context,
request *plugin_pb.ExecuteJobRequest,
sender pluginworker.ExecutionSender,
) error {
if request == nil || request.Job == nil {
return fmt.Errorf("execute request/job is nil")
}
if sender == nil {
return fmt.Errorf("execution sender is nil")
}
if request.Job.JobType != "" && request.Job.JobType != "ec_balance" {
return fmt.Errorf("job type %q is not handled by ec_balance worker", request.Job.JobType)
}
params, err := decodeECBalanceTaskParams(request.Job)
if err != nil {
return err
}
if len(params.Sources) == 0 || strings.TrimSpace(params.Sources[0].Node) == "" {
return fmt.Errorf("ec balance source node is required")
}
if len(params.Targets) == 0 || strings.TrimSpace(params.Targets[0].Node) == "" {
return fmt.Errorf("ec balance target node is required")
}
task := NewECBalanceTask(
request.Job.JobId,
params.VolumeId,
params.Collection,
h.grpcDialOption,
)
execCtx, execCancel := context.WithCancel(ctx)
defer execCancel()
task.SetProgressCallback(func(progress float64, stage string) {
message := fmt.Sprintf("ec balance progress %.0f%%", progress)
if strings.TrimSpace(stage) != "" {
message = stage
}
if err := sender.SendProgress(&plugin_pb.JobProgressUpdate{
JobId: request.Job.JobId,
JobType: request.Job.JobType,
State: plugin_pb.JobState_JOB_STATE_RUNNING,
ProgressPercent: progress,
Stage: stage,
Message: message,
Activities: []*plugin_pb.ActivityEvent{
pluginworker.BuildExecutorActivity(stage, message),
},
}); err != nil {
glog.Warningf("EC balance job %s (%s): failed to send progress (%.0f%%, stage=%q): %v, cancelling execution",
request.Job.JobId, request.Job.JobType, progress, stage, err)
execCancel()
}
})
if err := sender.SendProgress(&plugin_pb.JobProgressUpdate{
JobId: request.Job.JobId,
JobType: request.Job.JobType,
State: plugin_pb.JobState_JOB_STATE_ASSIGNED,
ProgressPercent: 0,
Stage: "assigned",
Message: "ec balance job accepted",
Activities: []*plugin_pb.ActivityEvent{
pluginworker.BuildExecutorActivity("assigned", "ec balance job accepted"),
},
}); err != nil {
return err
}
if err := task.Execute(execCtx, params); err != nil {
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
JobId: request.Job.JobId,
JobType: request.Job.JobType,
State: plugin_pb.JobState_JOB_STATE_FAILED,
ProgressPercent: 100,
Stage: "failed",
Message: err.Error(),
Activities: []*plugin_pb.ActivityEvent{
pluginworker.BuildExecutorActivity("failed", err.Error()),
},
})
return err
}
sourceNode := params.Sources[0].Node
targetNode := params.Targets[0].Node
resultSummary := fmt.Sprintf("EC shard balance completed: volume %d shards moved from %s to %s",
params.VolumeId, sourceNode, targetNode)
return sender.SendCompleted(&plugin_pb.JobCompleted{
JobId: request.Job.JobId,
JobType: request.Job.JobType,
Success: true,
Result: &plugin_pb.JobResult{
Summary: resultSummary,
OutputValues: map[string]*plugin_pb.ConfigValue{
"volume_id": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(params.VolumeId)},
},
"source_server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceNode},
},
"target_server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: targetNode},
},
},
},
Activities: []*plugin_pb.ActivityEvent{
pluginworker.BuildExecutorActivity("completed", resultSummary),
},
})
}
func (h *ECBalanceHandler) collectVolumeMetrics(
ctx context.Context,
masterAddresses []string,
collectionFilter string,
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, error) {
metrics, activeTopology, _, err := pluginworker.CollectVolumeMetricsFromMasters(ctx, masterAddresses, collectionFilter, h.grpcDialOption)
return metrics, activeTopology, err
}
func deriveECBalanceWorkerConfig(values map[string]*plugin_pb.ConfigValue) *ecBalanceWorkerConfig {
taskConfig := NewDefaultConfig()
imbalanceThreshold := pluginworker.ReadDoubleConfig(values, "imbalance_threshold", taskConfig.ImbalanceThreshold)
if imbalanceThreshold < ecBalanceMinImbalanceThreshold {
imbalanceThreshold = ecBalanceMinImbalanceThreshold
}
if imbalanceThreshold > ecBalanceMaxImbalanceThreshold {
imbalanceThreshold = ecBalanceMaxImbalanceThreshold
}
taskConfig.ImbalanceThreshold = imbalanceThreshold
minServerCount := pluginworker.ReadIntConfig(values, "min_server_count", taskConfig.MinServerCount)
if minServerCount < ecBalanceMinServerCount {
minServerCount = ecBalanceMinServerCount
}
taskConfig.MinServerCount = minServerCount
ioBytePerSecond := pluginworker.ReadInt64Config(values, "io_byte_per_second", taskConfig.IoBytePerSecond)
if ioBytePerSecond < 0 {
ioBytePerSecond = 0
}
taskConfig.IoBytePerSecond = ioBytePerSecond
taskConfig.PreferredTags = util.NormalizeTagList(pluginworker.ReadStringListConfig(values, "preferred_tags"))
return &ecBalanceWorkerConfig{
TaskConfig: taskConfig,
}
}
func buildECBalanceProposal(result *workertypes.TaskDetectionResult) (*plugin_pb.JobProposal, error) {
if result == nil {
return nil, fmt.Errorf("task detection result is nil")
}
if result.TypedParams == nil {
return nil, fmt.Errorf("missing typed params for volume %d", result.VolumeID)
}
paramsPayload, err := proto.Marshal(result.TypedParams)
if err != nil {
return nil, fmt.Errorf("marshal task params: %w", err)
}
proposalID := strings.TrimSpace(result.TaskID)
if proposalID == "" {
proposalID = fmt.Sprintf("ec-balance-%d-%d", result.VolumeID, time.Now().UnixNano())
}
// Dedupe per (collection, volume), NOT per shard. A volume's EC shards can be
// spread across several disks on one server, and concurrent moves of the same
// volume race on its shared .ecx/.ecj/.vif sidecar files. Keying by volume
// makes the scheduler run only one of a volume's moves at a time — both within
// a detection run and against in-flight jobs — and because the planner emits a
// volume's moves in phase order (dedup, cross-rack, within-rack, global), the
// phases then execute in order across detection cycles. This mirrors the shell,
// which serializes a volume's moves and waits between phases.
dedupeKey := fmt.Sprintf("ec_balance:%d", result.VolumeID)
if result.Collection != "" {
dedupeKey += ":" + result.Collection
}
sourceNode := ""
targetNode := ""
if len(result.TypedParams.Sources) > 0 {
sourceNode = strings.TrimSpace(result.TypedParams.Sources[0].Node)
}
if len(result.TypedParams.Targets) > 0 {
targetNode = strings.TrimSpace(result.TypedParams.Targets[0].Node)
}
summary := fmt.Sprintf("Balance EC shard of volume %d", result.VolumeID)
if sourceNode != "" && targetNode != "" {
summary = fmt.Sprintf("Move EC shard of volume %d: %s → %s", result.VolumeID, sourceNode, targetNode)
}
// EC shard moves only relocate one shard (1/14 of the volume). Budget
// 5 min/GB of full volume size, which conservatively covers the shard
// transfer plus mount/registration overhead.
volumeSizeGB := int64(result.TypedParams.VolumeSize/1024/1024/1024) + 1
estimatedRuntimeSeconds := volumeSizeGB * 5 * 60
return &plugin_pb.JobProposal{
ProposalId: proposalID,
DedupeKey: dedupeKey,
JobType: "ec_balance",
Priority: pluginworker.MapTaskPriority(result.Priority),
Summary: summary,
Detail: strings.TrimSpace(result.Reason),
Parameters: map[string]*plugin_pb.ConfigValue{
"task_params_pb": {
Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: paramsPayload},
},
"volume_id": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(result.VolumeID)},
},
"source_server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceNode},
},
"target_server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: targetNode},
},
"collection": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Collection},
},
"estimated_runtime_seconds": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: estimatedRuntimeSeconds},
},
},
Labels: map[string]string{
"task_type": "ec_balance",
"volume_id": fmt.Sprintf("%d", result.VolumeID),
"collection": result.Collection,
"source_node": sourceNode,
"target_node": targetNode,
},
}, nil
}
func decodeECBalanceTaskParams(job *plugin_pb.JobSpec) (*worker_pb.TaskParams, error) {
if job == nil {
return nil, fmt.Errorf("job spec is nil")
}
// Try protobuf-encoded params first (preferred path)
if payload := pluginworker.ReadBytesConfig(job.Parameters, "task_params_pb"); len(payload) > 0 {
params := &worker_pb.TaskParams{}
if err := proto.Unmarshal(payload, params); err != nil {
return nil, fmt.Errorf("decodeECBalanceTaskParams: unmarshal task_params_pb: %w", err)
}
if params.TaskId == "" {
params.TaskId = job.JobId
}
// Validate execution-critical fields in the deserialized TaskParams
if len(params.Sources) == 0 || strings.TrimSpace(params.Sources[0].Node) == "" {
return nil, fmt.Errorf("decodeECBalanceTaskParams: TaskParams missing Sources[0].Node")
}
if len(params.Sources[0].ShardIds) == 0 {
return nil, fmt.Errorf("decodeECBalanceTaskParams: TaskParams missing Sources[0].ShardIds")
}
if len(params.Targets) == 0 || strings.TrimSpace(params.Targets[0].Node) == "" {
return nil, fmt.Errorf("decodeECBalanceTaskParams: TaskParams missing Targets[0].Node")
}
if len(params.Targets[0].ShardIds) == 0 {
return nil, fmt.Errorf("decodeECBalanceTaskParams: TaskParams missing Targets[0].ShardIds")
}
return params, nil
}
// Legacy fallback: construct TaskParams from individual scalar parameters.
// All execution-critical fields are required.
volumeID := pluginworker.ReadInt64Config(job.Parameters, "volume_id", 0)
sourceNode := strings.TrimSpace(pluginworker.ReadStringConfig(job.Parameters, "source_server", ""))
targetNode := strings.TrimSpace(pluginworker.ReadStringConfig(job.Parameters, "target_server", ""))
collection := pluginworker.ReadStringConfig(job.Parameters, "collection", "")
if volumeID <= 0 || volumeID > math.MaxUint32 {
return nil, fmt.Errorf("decodeECBalanceTaskParams: invalid or missing volume_id: %d", volumeID)
}
if sourceNode == "" {
return nil, fmt.Errorf("decodeECBalanceTaskParams: missing source_server")
}
if targetNode == "" {
return nil, fmt.Errorf("decodeECBalanceTaskParams: missing target_server")
}
shardIDVal, hasShardID := job.Parameters["shard_id"]
if !hasShardID || shardIDVal == nil {
return nil, fmt.Errorf("decodeECBalanceTaskParams: missing shard_id (required for EcBalanceTaskParams)")
}
shardID := pluginworker.ReadInt64Config(job.Parameters, "shard_id", -1)
if shardID < 0 || shardID > math.MaxUint32 {
return nil, fmt.Errorf("decodeECBalanceTaskParams: invalid shard_id: %d", shardID)
}
sourceDiskID := pluginworker.ReadInt64Config(job.Parameters, "source_disk_id", 0)
if sourceDiskID < 0 || sourceDiskID > math.MaxUint32 {
return nil, fmt.Errorf("decodeECBalanceTaskParams: invalid source_disk_id: %d", sourceDiskID)
}
targetDiskID := pluginworker.ReadInt64Config(job.Parameters, "target_disk_id", 0)
if targetDiskID < 0 || targetDiskID > math.MaxUint32 {
return nil, fmt.Errorf("decodeECBalanceTaskParams: invalid target_disk_id: %d", targetDiskID)
}
ioBytePerSecond := pluginworker.ReadInt64Config(job.Parameters, "io_byte_per_second", 0)
return &worker_pb.TaskParams{
TaskId: job.JobId,
VolumeId: uint32(volumeID),
Collection: collection,
Sources: []*worker_pb.TaskSource{{
Node: sourceNode,
DiskId: uint32(sourceDiskID),
ShardIds: []uint32{uint32(shardID)},
}},
Targets: []*worker_pb.TaskTarget{{
Node: targetNode,
DiskId: uint32(targetDiskID),
ShardIds: []uint32{uint32(shardID)},
}},
TaskParams: &worker_pb.TaskParams_EcBalanceParams{
EcBalanceParams: &worker_pb.EcBalanceTaskParams{
TimeoutSeconds: 600,
IoBytePerSecond: ioBytePerSecond,
},
},
}, nil
}
func emitECBalanceDecisionTrace(
sender pluginworker.DetectionSender,
taskConfig *Config,
results []*workertypes.TaskDetectionResult,
maxResults int,
hasMore bool,
) error {
if sender == nil || taskConfig == nil {
return nil
}
// Count moves by phase
phaseCounts := make(map[string]int)
for _, result := range results {
if result.Reason != "" {
// Extract phase from reason string
for _, phase := range []string{"dedup", "cross_rack", "within_rack", "global"} {
if strings.Contains(result.Reason, phase) {
phaseCounts[phase]++
break
}
}
}
}
summarySuffix := ""
if hasMore {
summarySuffix = fmt.Sprintf(" (max_results=%d reached)", maxResults)
}
summaryMessage := fmt.Sprintf(
"EC balance detection: %d moves proposed%s (dedup=%d, cross_rack=%d, within_rack=%d, global=%d)",
len(results),
summarySuffix,
phaseCounts["dedup"],
phaseCounts["cross_rack"],
phaseCounts["within_rack"],
phaseCounts["global"],
)
return sender.SendActivity(pluginworker.BuildDetectorActivity("decision_summary", summaryMessage, map[string]*plugin_pb.ConfigValue{
"total_moves": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(results))},
},
"has_more": {
Kind: &plugin_pb.ConfigValue_BoolValue{BoolValue: hasMore},
},
"dedup_moves": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(phaseCounts["dedup"])},
},
"cross_rack_moves": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(phaseCounts["cross_rack"])},
},
"within_rack_moves": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(phaseCounts["within_rack"])},
},
"global_moves": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(phaseCounts["global"])},
},
"imbalance_threshold": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: taskConfig.ImbalanceThreshold},
},
"min_server_count": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.MinServerCount)},
},
}))
}

View File

@@ -0,0 +1,327 @@
package ec_balance
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
"google.golang.org/protobuf/proto"
)
func TestDeriveECBalanceWorkerConfig(t *testing.T) {
tests := []struct {
name string
values map[string]*plugin_pb.ConfigValue
check func(t *testing.T, config *ecBalanceWorkerConfig)
}{
{
name: "nil values uses defaults",
values: nil,
check: func(t *testing.T, config *ecBalanceWorkerConfig) {
if config.TaskConfig.ImbalanceThreshold != 0.2 {
t.Errorf("expected default threshold 0.2, got %f", config.TaskConfig.ImbalanceThreshold)
}
if config.TaskConfig.MinServerCount != 3 {
t.Errorf("expected default min_server_count 3, got %d", config.TaskConfig.MinServerCount)
}
},
},
{
name: "custom threshold",
values: map[string]*plugin_pb.ConfigValue{
"imbalance_threshold": {Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.3}},
"min_server_count": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 5}},
},
check: func(t *testing.T, config *ecBalanceWorkerConfig) {
if config.TaskConfig.ImbalanceThreshold != 0.3 {
t.Errorf("expected threshold 0.3, got %f", config.TaskConfig.ImbalanceThreshold)
}
if config.TaskConfig.MinServerCount != 5 {
t.Errorf("expected min_server_count 5, got %d", config.TaskConfig.MinServerCount)
}
},
},
{
name: "threshold clamped to min",
values: map[string]*plugin_pb.ConfigValue{
"imbalance_threshold": {Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.01}},
},
check: func(t *testing.T, config *ecBalanceWorkerConfig) {
if config.TaskConfig.ImbalanceThreshold != 0.05 {
t.Errorf("expected clamped threshold 0.05, got %f", config.TaskConfig.ImbalanceThreshold)
}
},
},
{
name: "threshold clamped to max",
values: map[string]*plugin_pb.ConfigValue{
"imbalance_threshold": {Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.9}},
},
check: func(t *testing.T, config *ecBalanceWorkerConfig) {
if config.TaskConfig.ImbalanceThreshold != 0.5 {
t.Errorf("expected clamped threshold 0.5, got %f", config.TaskConfig.ImbalanceThreshold)
}
},
},
{
name: "min_server_count clamped to 2",
values: map[string]*plugin_pb.ConfigValue{
"min_server_count": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}},
},
check: func(t *testing.T, config *ecBalanceWorkerConfig) {
if config.TaskConfig.MinServerCount != 2 {
t.Errorf("expected min_server_count 2, got %d", config.TaskConfig.MinServerCount)
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
config := deriveECBalanceWorkerConfig(tt.values)
tt.check(t, config)
})
}
}
func TestBuildECBalanceProposal(t *testing.T) {
result := &workertypes.TaskDetectionResult{
TaskID: "test-task-123",
TaskType: workertypes.TaskTypeECBalance,
VolumeID: 42,
Collection: "test-col",
Priority: workertypes.TaskPriorityMedium,
Reason: "cross_rack balance",
TypedParams: &worker_pb.TaskParams{
VolumeId: 42,
Collection: "test-col",
Sources: []*worker_pb.TaskSource{{
Node: "source:8080",
ShardIds: []uint32{5},
}},
Targets: []*worker_pb.TaskTarget{{
Node: "target:8080",
ShardIds: []uint32{5},
}},
},
}
proposal, err := buildECBalanceProposal(result)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if proposal.JobType != "ec_balance" {
t.Errorf("expected job type ec_balance, got %s", proposal.JobType)
}
if proposal.ProposalId != "test-task-123" {
t.Errorf("expected proposal ID test-task-123, got %s", proposal.ProposalId)
}
// Dedupe is per (volume, collection) so a volume's shard moves serialize and
// run in phase order, rather than per shard which allowed concurrent same-volume
// moves that race on the volume's .ecx/.ecj/.vif sidecars.
if proposal.DedupeKey != "ec_balance:42:test-col" {
t.Errorf("expected dedupe key ec_balance:42:test-col, got %s", proposal.DedupeKey)
}
if proposal.Labels["source_node"] != "source:8080" {
t.Errorf("expected source_node label source:8080, got %s", proposal.Labels["source_node"])
}
if proposal.Labels["target_node"] != "target:8080" {
t.Errorf("expected target_node label target:8080, got %s", proposal.Labels["target_node"])
}
}
func TestBuildECBalanceProposalNilResult(t *testing.T) {
_, err := buildECBalanceProposal(nil)
if err == nil {
t.Fatal("expected error for nil result")
}
}
func TestDecodeECBalanceTaskParamsFromProtobuf(t *testing.T) {
originalParams := &worker_pb.TaskParams{
TaskId: "test-id",
VolumeId: 100,
Collection: "test-col",
Sources: []*worker_pb.TaskSource{{
Node: "source:8080",
ShardIds: []uint32{3},
}},
Targets: []*worker_pb.TaskTarget{{
Node: "target:8080",
ShardIds: []uint32{3},
}},
TaskParams: &worker_pb.TaskParams_EcBalanceParams{
EcBalanceParams: &worker_pb.EcBalanceTaskParams{
DiskType: "hdd",
TimeoutSeconds: 300,
},
},
}
payload, err := proto.Marshal(originalParams)
if err != nil {
t.Fatalf("failed to marshal: %v", err)
}
job := &plugin_pb.JobSpec{
JobId: "job-1",
Parameters: map[string]*plugin_pb.ConfigValue{
"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
},
}
decoded, err := decodeECBalanceTaskParams(job)
if err != nil {
t.Fatalf("failed to decode: %v", err)
}
if decoded.VolumeId != 100 {
t.Errorf("expected volume_id 100, got %d", decoded.VolumeId)
}
if decoded.Collection != "test-col" {
t.Errorf("expected collection test-col, got %s", decoded.Collection)
}
if len(decoded.Sources) != 1 || decoded.Sources[0].Node != "source:8080" {
t.Error("source mismatch")
}
if len(decoded.Targets) != 1 || decoded.Targets[0].Node != "target:8080" {
t.Error("target mismatch")
}
ecParams := decoded.GetEcBalanceParams()
if ecParams == nil {
t.Fatal("expected ec_balance_params")
}
if ecParams.DiskType != "hdd" {
t.Errorf("expected disk_type hdd, got %s", ecParams.DiskType)
}
}
func TestDecodeECBalanceTaskParamsFallback(t *testing.T) {
job := &plugin_pb.JobSpec{
JobId: "job-2",
Parameters: map[string]*plugin_pb.ConfigValue{
"volume_id": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 200}},
"source_server": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "src:8080"}},
"target_server": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "dst:8080"}},
"collection": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "fallback-col"}},
"shard_id": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7}},
"source_disk_id": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}},
"target_disk_id": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 2}},
},
}
decoded, err := decodeECBalanceTaskParams(job)
if err != nil {
t.Fatalf("failed to decode fallback: %v", err)
}
if decoded.VolumeId != 200 {
t.Errorf("expected volume_id 200, got %d", decoded.VolumeId)
}
if len(decoded.Sources) != 1 || decoded.Sources[0].Node != "src:8080" {
t.Error("source mismatch")
}
if decoded.Sources[0].ShardIds[0] != 7 {
t.Errorf("expected shard_id 7, got %d", decoded.Sources[0].ShardIds[0])
}
if decoded.Sources[0].DiskId != 1 {
t.Errorf("expected source disk_id 1, got %d", decoded.Sources[0].DiskId)
}
if decoded.Targets[0].DiskId != 2 {
t.Errorf("expected target disk_id 2, got %d", decoded.Targets[0].DiskId)
}
}
func TestDecodeECBalanceTaskParamsProtobufValidation(t *testing.T) {
// Protobuf payload with missing ShardIds should fail validation
badParams := &worker_pb.TaskParams{
TaskId: "test-id",
VolumeId: 100,
Sources: []*worker_pb.TaskSource{{Node: "source:8080"}}, // no ShardIds
Targets: []*worker_pb.TaskTarget{{Node: "target:8080", ShardIds: []uint32{3}}},
}
payload, _ := proto.Marshal(badParams)
job := &plugin_pb.JobSpec{
JobId: "job-validate",
Parameters: map[string]*plugin_pb.ConfigValue{
"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
},
}
_, err := decodeECBalanceTaskParams(job)
if err == nil {
t.Fatal("expected error for missing Sources[0].ShardIds in protobuf")
}
}
func TestDecodeECBalanceTaskParamsMissingShardID(t *testing.T) {
job := &plugin_pb.JobSpec{
JobId: "job-3",
Parameters: map[string]*plugin_pb.ConfigValue{
"volume_id": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 300}},
"source_server": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "src:8080"}},
"target_server": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "dst:8080"}},
"collection": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "test-col"}},
// shard_id deliberately missing
},
}
_, err := decodeECBalanceTaskParams(job)
if err == nil {
t.Fatal("expected error for missing shard_id")
}
}
func TestECBalanceHandlerCapability(t *testing.T) {
handler := NewECBalanceHandler(nil)
cap := handler.Capability()
if cap.JobType != "ec_balance" {
t.Errorf("expected job type ec_balance, got %s", cap.JobType)
}
if !cap.CanDetect {
t.Error("expected CanDetect=true")
}
if !cap.CanExecute {
t.Error("expected CanExecute=true")
}
if cap.Weight != 60 {
t.Errorf("expected weight 60, got %d", cap.Weight)
}
}
func TestECBalanceConfigRoundTrip(t *testing.T) {
config := NewDefaultConfig()
config.ImbalanceThreshold = 0.3
config.MinServerCount = 5
config.CollectionFilter = "my_col"
config.DiskType = "ssd"
config.PreferredTags = []string{"fast", "ssd"}
policy := config.ToTaskPolicy()
if policy == nil {
t.Fatal("expected non-nil policy")
}
config2 := NewDefaultConfig()
if err := config2.FromTaskPolicy(policy); err != nil {
t.Fatalf("failed to load from policy: %v", err)
}
if config2.ImbalanceThreshold != 0.3 {
t.Errorf("expected threshold 0.3, got %f", config2.ImbalanceThreshold)
}
if config2.MinServerCount != 5 {
t.Errorf("expected min_server_count 5, got %d", config2.MinServerCount)
}
if config2.CollectionFilter != "my_col" {
t.Errorf("expected collection_filter my_col, got %s", config2.CollectionFilter)
}
if config2.DiskType != "ssd" {
t.Errorf("expected disk_type ssd, got %s", config2.DiskType)
}
if len(config2.PreferredTags) != 2 || config2.PreferredTags[0] != "fast" {
t.Errorf("expected preferred_tags [fast ssd], got %v", config2.PreferredTags)
}
}

View File

@@ -0,0 +1,89 @@
package ec_balance
import (
"context"
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// Global variable to hold the task definition for configuration updates
var globalTaskDef *base.TaskDefinition
// Auto-register this task when the package is imported
func init() {
RegisterECBalanceTask()
// Register config updater
tasks.AutoRegisterConfigUpdater(types.TaskTypeECBalance, UpdateConfigFromPersistence)
}
// RegisterECBalanceTask registers the EC balance task with the task architecture
func RegisterECBalanceTask() {
cfg := NewDefaultConfig()
// Create shared gRPC dial option using TLS configuration
dialOpt := security.LoadClientTLS(util.GetViper(), "grpc.worker")
// Create complete task definition
taskDef := &base.TaskDefinition{
Type: types.TaskTypeECBalance,
Name: "ec_balance",
DisplayName: "EC Shard Balance",
Description: "Balances EC shard distribution across racks and servers",
Icon: "fas fa-balance-scale-left text-info",
Capabilities: []string{"ec_balance", "erasure_coding"},
Config: cfg,
ConfigSpec: GetConfigSpec(),
CreateTask: func(params *worker_pb.TaskParams) (types.Task, error) {
if params == nil {
return nil, fmt.Errorf("task parameters are required")
}
return NewECBalanceTask(
fmt.Sprintf("ec_balance-%d", params.VolumeId),
params.VolumeId,
params.Collection,
dialOpt,
), nil
},
DetectionFunc: func(metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo, config base.TaskConfig) ([]*types.TaskDetectionResult, error) {
results, _, err := Detection(context.Background(), metrics, clusterInfo, config, 0)
return results, err
},
ScanInterval: 1 * time.Hour,
SchedulingFunc: Scheduling,
MaxConcurrent: 1,
RepeatInterval: 4 * time.Hour,
}
// Store task definition globally for configuration updates
globalTaskDef = taskDef
// Register everything with a single function call
base.RegisterTask(taskDef)
}
// UpdateConfigFromPersistence updates the EC balance configuration from persistence
func UpdateConfigFromPersistence(configPersistence interface{}) error {
if globalTaskDef == nil {
return fmt.Errorf("ec balance task not registered")
}
newConfig := LoadConfigFromPersistence(configPersistence)
if newConfig == nil {
return fmt.Errorf("failed to load configuration from persistence")
}
globalTaskDef.Config = newConfig
glog.V(1).Infof("Updated EC balance task configuration from persistence")
return nil
}

View File

@@ -0,0 +1,40 @@
package ec_balance
import (
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// Scheduling implements the scheduling logic for EC balance tasks
func Scheduling(task *types.TaskInput, runningTasks []*types.TaskInput, availableWorkers []*types.WorkerData, config base.TaskConfig) bool {
ecbConfig := config.(*Config)
// Check if we have available workers
if len(availableWorkers) == 0 {
return false
}
// Count running EC balance tasks
runningCount := 0
for _, runningTask := range runningTasks {
if runningTask.Type == types.TaskTypeECBalance {
runningCount++
}
}
// Check concurrency limit
if runningCount >= ecbConfig.MaxConcurrent {
return false
}
// Check if any worker can handle EC balance tasks
for _, worker := range availableWorkers {
for _, capability := range worker.Capabilities {
if capability == types.TaskTypeECBalance {
return true
}
}
}
return false
}