snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,255 @@
package erasure_coding
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/admin/config"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
)
// Config extends BaseConfig with erasure coding specific settings
type Config struct {
base.BaseConfig
QuietForSeconds int `json:"quiet_for_seconds"`
FullnessRatio float64 `json:"fullness_ratio"`
CollectionFilter string `json:"collection_filter"`
MinSizeMB int `json:"min_size_mb"`
PreferredTags []string `json:"preferred_tags"`
ReplicaPlacement string `json:"replica_placement"` // e.g. "020"; empty falls back to the master default replication
}
// NewDefaultConfig creates a new default erasure coding configuration
func NewDefaultConfig() *Config {
return &Config{
BaseConfig: base.BaseConfig{
Enabled: true,
ScanIntervalSeconds: 60 * 60, // 1 hour
MaxConcurrent: 1,
},
QuietForSeconds: 3600, // 1 hour, matching the shell ec.encode -quietFor default
FullnessRatio: 0.95, // 95%, matching the shell ec.encode -fullPercent default
CollectionFilter: "",
MinSizeMB: 30, // 30MB (more reasonable than 100MB)
PreferredTags: nil,
}
}
// GetConfigSpec returns the configuration schema for erasure coding tasks
func GetConfigSpec() base.ConfigSpec {
return base.ConfigSpec{
Fields: []*config.Field{
{
Name: "enabled",
JSONName: "enabled",
Type: config.FieldTypeBool,
DefaultValue: true,
Required: false,
DisplayName: "Enable Erasure Coding Tasks",
Description: "Whether erasure coding tasks should be automatically created",
HelpText: "Toggle this to enable or disable automatic erasure coding task generation",
InputType: "checkbox",
CSSClasses: "form-check-input",
},
{
Name: "scan_interval_seconds",
JSONName: "scan_interval_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 60 * 60,
MinValue: 10 * 60,
MaxValue: 24 * 60 * 60,
Required: true,
DisplayName: "Scan Interval",
Description: "How often to scan for volumes needing erasure coding",
HelpText: "The system will check for volumes that need erasure coding at this interval",
Placeholder: "1",
Unit: config.UnitHours,
InputType: "interval",
CSSClasses: "form-control",
},
{
Name: "max_concurrent",
JSONName: "max_concurrent",
Type: config.FieldTypeInt,
DefaultValue: 1,
MinValue: 1,
MaxValue: 5,
Required: true,
DisplayName: "Max Concurrent Tasks",
Description: "Maximum number of erasure coding tasks that can run simultaneously",
HelpText: "Limits the number of erasure coding operations running at the same time",
Placeholder: "1 (default)",
Unit: config.UnitCount,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "quiet_for_seconds",
JSONName: "quiet_for_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 3600,
MinValue: 1,
MaxValue: 3600,
Required: true,
DisplayName: "Quiet Period",
Description: "Minimum time volume must be quiet before erasure coding",
HelpText: "Volume must not be modified for this duration before erasure coding",
Placeholder: "60",
Unit: config.UnitMinutes,
InputType: "interval",
CSSClasses: "form-control",
},
{
Name: "fullness_ratio",
JSONName: "fullness_ratio",
Type: config.FieldTypeFloat,
DefaultValue: 0.95,
MinValue: 0.0001,
MaxValue: 1.0,
Required: true,
DisplayName: "Fullness Ratio",
Description: "Minimum fullness ratio to trigger erasure coding",
HelpText: "Only volumes with this fullness ratio or higher will be erasure coded",
Placeholder: "0.80 (80%)",
Unit: config.UnitNone,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "collection_filter",
JSONName: "collection_filter",
Type: config.FieldTypeString,
DefaultValue: "",
Required: false,
DisplayName: "Collection Filter",
Description: "Only process volumes from specific collections (comma-separated names, wildcards, or regex patterns)",
HelpText: "Leave empty to process all collections, or list the collections to process",
Placeholder: "pictures,videos",
InputType: "text",
CSSClasses: "form-control",
},
{
Name: "min_size_mb",
JSONName: "min_size_mb",
Type: config.FieldTypeInt,
DefaultValue: 30,
MinValue: 1,
MaxValue: 1000,
Required: true,
DisplayName: "Minimum Size (MB)",
Description: "Minimum volume size to consider for erasure coding",
HelpText: "Only volumes larger than this size will be considered for erasure coding",
Placeholder: "30",
Unit: config.UnitNone,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "preferred_tags",
JSONName: "preferred_tags",
Type: config.FieldTypeString,
DefaultValue: "",
Required: false,
DisplayName: "Preferred Disk Tags",
Description: "Comma-separated disk tags to prioritize for EC shard placement",
HelpText: "EC shards will be placed on disks with these tags first, then fall back to other disks if needed",
Placeholder: "fast,ssd",
InputType: "text",
CSSClasses: "form-control",
},
{
Name: "replica_placement",
JSONName: "replica_placement",
Type: config.FieldTypeString,
DefaultValue: "",
Required: false,
DisplayName: "Replica Placement",
Description: "EC shard replica placement constraint (e.g. 020)",
HelpText: "Leave empty to use the master default replication. When set, the 2nd/3rd digits cap EC shards per rack and per node (best-effort during encode: relaxed rather than failing if the cluster can't satisfy them, then enforced by rebalancing). The 1st (data-center) digit is ignored for EC placement",
Placeholder: "020",
InputType: "text",
CSSClasses: "form-control",
},
},
}
}
// ToTaskPolicy converts configuration to a TaskPolicy protobuf message
func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
// Defensive copy of PreferredTags to prevent external mutation
preferredTagsCopy := append([]string(nil), c.PreferredTags...)
return &worker_pb.TaskPolicy{
Enabled: c.Enabled,
MaxConcurrent: int32(c.MaxConcurrent),
RepeatIntervalSeconds: int32(c.ScanIntervalSeconds),
CheckIntervalSeconds: int32(c.ScanIntervalSeconds),
TaskConfig: &worker_pb.TaskPolicy_ErasureCodingConfig{
ErasureCodingConfig: &worker_pb.ErasureCodingTaskConfig{
FullnessRatio: float64(c.FullnessRatio),
QuietForSeconds: int32(c.QuietForSeconds),
MinVolumeSizeMb: int32(c.MinSizeMB),
CollectionFilter: c.CollectionFilter,
PreferredTags: preferredTagsCopy,
ReplicaPlacement: c.ReplicaPlacement,
},
},
}
}
// FromTaskPolicy loads configuration from a TaskPolicy protobuf message
func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
if policy == nil {
return fmt.Errorf("policy is nil")
}
// Set general TaskPolicy fields
c.Enabled = policy.Enabled
c.MaxConcurrent = int(policy.MaxConcurrent)
c.ScanIntervalSeconds = int(policy.RepeatIntervalSeconds) // Direct seconds-to-seconds mapping
// Set erasure coding-specific fields from the task config
if ecConfig := policy.GetErasureCodingConfig(); ecConfig != nil {
c.FullnessRatio = float64(ecConfig.FullnessRatio)
c.QuietForSeconds = int(ecConfig.QuietForSeconds)
c.MinSizeMB = int(ecConfig.MinVolumeSizeMb)
c.CollectionFilter = ecConfig.CollectionFilter
c.PreferredTags = append([]string(nil), ecConfig.PreferredTags...)
c.ReplicaPlacement = ecConfig.ReplicaPlacement
}
return nil
}
// LoadConfigFromPersistence loads configuration from the persistence layer if available
func LoadConfigFromPersistence(configPersistence interface{}) *Config {
config := NewDefaultConfig()
// Try to load from persistence if available
if persistence, ok := configPersistence.(interface {
LoadErasureCodingTaskPolicy() (*worker_pb.TaskPolicy, error)
}); ok {
policy, err := persistence.LoadErasureCodingTaskPolicy()
switch {
case err != nil:
glog.Warningf("Could not read the persisted erasure coding configuration, falling back to defaults: %v", err)
case policy == nil:
glog.V(1).Infof("No erasure coding configuration persisted yet, using defaults")
default:
if err := config.FromTaskPolicy(policy); err != nil {
glog.Warningf("Could not apply the persisted erasure coding configuration, falling back to defaults: %v", err)
} else {
glog.V(1).Infof("Loaded erasure coding configuration from persistence")
return config
}
}
} else if configPersistence != nil {
// A store was handed in but does not expose the accessor, so the persisted
// settings are silently ignored - always a wiring bug, never a normal state.
glog.Warningf("%T cannot provide the persisted erasure coding configuration: it has no LoadErasureCodingTaskPolicy() method, "+
"so the compiled-in defaults are used and any saved erasure coding settings are ignored", configPersistence)
}
glog.V(1).Infof("Using default erasure coding configuration")
return config
}

View File

@@ -0,0 +1,772 @@
package erasure_coding
import (
"context"
"fmt"
"sort"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
"github.com/seaweedfs/seaweedfs/weed/ec"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
workerutil "github.com/seaweedfs/seaweedfs/weed/worker/tasks/util"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
const (
minProposalsBeforeEarlyStop = 10
maxConsecutivePlanningFailures = 10
)
// Detection implements the detection logic for erasure coding tasks.
// It respects ctx cancellation and can stop early once maxResults is reached.
func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo, config base.TaskConfig, maxResults int) ([]*types.TaskDetectionResult, bool, error) {
if !config.IsEnabled() {
return nil, false, nil
}
if maxResults < 0 {
maxResults = 0
}
ecConfig := config.(*Config)
var results []*types.TaskDetectionResult
hasMore := false
stoppedEarly := false
now := time.Now()
quietThreshold := time.Duration(ecConfig.QuietForSeconds) * time.Second
minSizeBytes := uint64(ecConfig.MinSizeMB) * 1024 * 1024 // Configurable minimum
debugCount := 0
skippedAlreadyEC := 0
skippedTooSmall := 0
skippedCollectionFilter := 0
skippedQuietTime := 0
skippedFullness := 0
skippedRemote := 0
skippedTooFewNodes := 0
consecutivePlanningFailures := 0
// EC shard replica placement: explicit config wins, else the master default.
var replicaPlacement *super_block.ReplicaPlacement
if clusterInfo != nil {
replicaPlacement = super_block.ResolveReplicaPlacement(ecConfig.ReplicaPlacement, clusterInfo.DefaultReplicaPlacement)
}
// EC placement honors only the rack/node digits; the data-center digit can't
// express a useful per-DC EC shard cap (it maxes at 2). Warn once per cycle so a
// 1xx/2xx setting isn't silently ineffective.
if replicaPlacement != nil && replicaPlacement.DiffDataCenterCount > 0 {
glog.Warningf("EC Detection: replica placement data-center digit (%d) is ignored for EC; only rack/node digits are honored", replicaPlacement.DiffDataCenterCount)
}
allowedCollections, err := wildcard.CompileCollectionMatcher(ecConfig.CollectionFilter)
if err != nil {
return nil, false, err
}
// Cluster node count for the min-node safety gate (mirrors the shell ec.encode
// guard that refuses to encode when nodes < parity shards, so shards cannot be
// spread for fault tolerance).
clusterNodeCount := countTopologyNodes(clusterInfo.ActiveTopology)
// Group metrics by VolumeID to handle replicas and select canonical server
volumeGroups := make(map[uint32][]*types.VolumeHealthMetrics)
for _, metric := range metrics {
if ctx != nil {
if err := ctx.Err(); err != nil {
return results, hasMore, err
}
}
volumeGroups[metric.VolumeID] = append(volumeGroups[metric.VolumeID], metric)
}
groupKeys := make([]uint32, 0, len(volumeGroups))
for volumeID := range volumeGroups {
groupKeys = append(groupKeys, volumeID)
}
sort.Slice(groupKeys, func(i, j int) bool { return groupKeys[i] < groupKeys[j] })
// Build the EC placement snapshot once per detection cycle. planECDestinations
// reserves the shards it assigns directly into it, so volumes planned later in
// this cycle see the reduced capacity. Rebuilding it per volume re-walked the
// whole topology (and ActiveTopology's growing pending-task set) every time,
// which is O(volumes × topology) and times out on large clusters. The node
// address map is precomputed for the same reason (ResolveServerAddress rebuilds
// the full node map on every call).
var ecSnapshot *ecbalancer.Topology
var nodeAddresses map[string]string
if clusterInfo != nil && clusterInfo.ActiveTopology != nil {
ecSnapshot = ecbalancer.FromActiveTopology(clusterInfo.ActiveTopology, erasure_coding.DataShardsCount)
nodeAddresses = buildNodeAddressMap(clusterInfo.ActiveTopology)
}
// Iterate over groups to check criteria and creation tasks
for idx, volumeID := range groupKeys {
if ctx != nil {
if err := ctx.Err(); err != nil {
return results, hasMore, err
}
}
if maxResults > 0 && len(results) >= maxResults {
if idx+1 < len(groupKeys) {
hasMore = true
}
stoppedEarly = true
break
}
groupMetrics := volumeGroups[volumeID]
// Prefer the lowest-server credible replica so a 0-byte stub or a
// leftover EC shard set on a lower server can't become canonical and
// strand the volume in skippedTooSmall / skippedAlreadyEC.
metric := selectCanonicalMetric(groupMetrics)
// Skip if already EC volume
if metric.IsECVolume {
skippedAlreadyEC++
continue
}
// Handle the "stuck source" state from #9448: a previous encode
// succeeded but the post-encode source-delete left a regular replica
// behind, so the master heartbeats BOTH the replica AND its EC shards.
// metric.IsECVolume above is set only for the EC-side metric path, so
// the canonical metric we picked is the regular replica with
// IsECVolume=false. Re-proposing an encode in that state collides with
// the mounted shards on the targets ("ec volume %d is mounted; refusing
// overwrite") and the detector re-queues forever.
//
// We only act when the EC shard set is COMPLETE — fewer than
// totalShards present means the existing recovery branch below
// (around the `existingECShards` block) should keep its chance to
// fold the partial shards into the new task. Counting walks
// EcIndexBits to handle a single info entry carrying multiple shards.
if clusterInfo.ActiveTopology != nil {
shardCount := countExistingEcShardsForVolume(clusterInfo.ActiveTopology, metric.VolumeID, metric.Collection)
totalShards := erasure_coding.DataShardsCount + erasure_coding.ParityShardsCount
if shardCount >= totalShards {
glog.Warningf("EC Detection: Volume %d has all %d EC shards in topology; "+
"source replica on %s is orphaned (#9448).",
metric.VolumeID, totalShards, metric.Server)
if clusterInfo.GrpcDialOption != nil {
deleted, cleanupErr := cleanupOrphanSourceReplicas(ctx, clusterInfo, metric, totalShards)
switch {
case cleanupErr != nil:
// Don't fall through to a re-encode — that would just
// collide with the mounted shards again. Surface the
// failure and wait for the next cycle; the source is
// still safe.
glog.Warningf("EC Detection: failed to auto-clean orphaned source for volume %d: %v", metric.VolumeID, cleanupErr)
case deleted > 0:
glog.Infof("EC Detection: auto-cleaned %d orphaned source replica(s) for volume %d after verifying all %d EC shards present", deleted, metric.VolumeID, totalShards)
default:
glog.V(1).Infof("EC Detection: no orphaned regular replicas found in topology for volume %d (collection %q)", metric.VolumeID, metric.Collection)
}
} else {
glog.Warningf("EC Detection: no gRPC dial option available to auto-clean orphaned source for volume %d; "+
"to clean up by hand, send a targeted VolumeDelete RPC to %s only — DO NOT use the cluster-wide `volume.delete` shell command, which would also delete the EC shards.",
metric.VolumeID, metric.Server)
}
skippedAlreadyEC++
continue
}
}
// Check minimum size requirement
if metric.Size < minSizeBytes {
skippedTooSmall++
continue
}
// Check collection filter if specified
if !allowedCollections.Matches(metric.Collection) {
skippedCollectionFilter++
continue
}
// Skip remote/tiered volumes: encoding them would lose the tiering. The
// shell ec.encode excludes remote volumes for the same reason.
if metric.HasRemoteCopy {
skippedRemote++
continue
}
// Min-node safety gate: don't encode when the cluster has fewer nodes than
// this collection's parity shards — shards could not be spread to tolerate
// failures. Mirrors the shell ec.encode guard (node count < parity shards).
if clusterNodeCount > 0 && clusterNodeCount < erasure_coding.ParityShardsCount {
skippedTooFewNodes++
continue
}
// Check quiet duration and fullness criteria
if metric.Age >= quietThreshold && metric.FullnessRatio >= ecConfig.FullnessRatio {
if ctx != nil {
if err := ctx.Err(); err != nil {
return results, hasMore, err
}
}
glog.Infof("EC Detection: Volume %d meets all criteria, attempting to create task", metric.VolumeID)
// Generate task ID for ActiveTopology integration
taskID := fmt.Sprintf("ec_vol_%d_%d", metric.VolumeID, now.Unix())
result := &types.TaskDetectionResult{
TaskID: taskID, // Link to ActiveTopology pending task
TaskType: types.TaskTypeErasureCoding,
VolumeID: metric.VolumeID,
Server: metric.Server,
Collection: metric.Collection,
Priority: types.TaskPriorityLow, // EC is not urgent
Reason: fmt.Sprintf("Volume meets EC criteria: quiet for %.1fs (>%ds), fullness=%.1f%% (>%.1f%%), size=%.1fMB (>%dMB)",
metric.Age.Seconds(), ecConfig.QuietForSeconds, metric.FullnessRatio*100, ecConfig.FullnessRatio*100,
float64(metric.Size)/(1024*1024), ecConfig.MinSizeMB),
ScheduleAt: now,
}
// Plan EC destinations if ActiveTopology is available
if clusterInfo.ActiveTopology != nil {
// Check if ANY task already exists in ActiveTopology for this volume
if clusterInfo.ActiveTopology.HasAnyTask(metric.VolumeID) {
glog.V(2).Infof("EC Detection: Skipping volume %d, task already exists in ActiveTopology", metric.VolumeID)
continue
}
glog.Infof("EC Detection: ActiveTopology available, planning destinations for volume %d", metric.VolumeID)
dataShards := erasure_coding.DataShardsCount
parityShards := erasure_coding.ParityShardsCount
multiPlan, shardsPerPlan, err := planECDestinations(ecSnapshot, nodeAddresses, metric, ecConfig, replicaPlacement, dataShards, parityShards)
if err != nil {
glog.V(2).Infof("Failed to plan EC destinations for volume %d: %v", metric.VolumeID, err)
consecutivePlanningFailures++
if len(results) >= minProposalsBeforeEarlyStop && consecutivePlanningFailures >= maxConsecutivePlanningFailures {
glog.Warningf("EC Detection: stopping early after %d consecutive placement failures with %d proposals already planned", consecutivePlanningFailures, len(results))
hasMore = true
stoppedEarly = true
break
}
continue // Skip this volume if destination planning fails
}
consecutivePlanningFailures = 0
glog.Infof("EC Detection: Successfully planned %d destinations for volume %d", len(multiPlan.Plans), metric.VolumeID)
// Calculate expected shard size for EC operation
// Each data shard will be approximately volumeSize / dataShards
expectedShardSize := uint64(metric.Size) / uint64(dataShards)
// Add pending EC shard task to ActiveTopology for capacity management
// Extract shard destinations from multiPlan
var shardDestinations []string
var shardDiskIDs []uint32
for _, plan := range multiPlan.Plans {
shardDestinations = append(shardDestinations, plan.TargetNode)
shardDiskIDs = append(shardDiskIDs, plan.TargetDisk)
}
// Find all volume replica locations (server + disk) from topology
glog.Infof("EC Detection: Looking for replica locations for volume %d", metric.VolumeID)
replicaLocations := findVolumeReplicaLocations(clusterInfo.ActiveTopology, metric.VolumeID, metric.Collection)
if len(replicaLocations) == 0 {
glog.Warningf("No replica locations found for volume %d, skipping EC", metric.VolumeID)
continue
}
glog.Infof("EC Detection: Found %d replica locations for volume %d", len(replicaLocations), metric.VolumeID)
// Find existing EC shards from previous failed attempts
existingECShards := findExistingECShards(clusterInfo.ActiveTopology, metric.VolumeID, metric.Collection)
// Combine volume replicas and existing EC shards for cleanup
var sources []topology.TaskSourceSpec
// Add volume replicas (will free volume slots)
for _, replica := range replicaLocations {
sources = append(sources, topology.TaskSourceSpec{
ServerID: replica.ServerID,
DiskID: replica.DiskID,
DataCenter: replica.DataCenter,
Rack: replica.Rack,
CleanupType: topology.CleanupVolumeReplica,
})
}
// Add existing EC shards (will free shard slots)
duplicateCheck := make(map[string]bool)
for _, replica := range replicaLocations {
key := fmt.Sprintf("%s:%d", replica.ServerID, replica.DiskID)
duplicateCheck[key] = true
}
for _, shard := range existingECShards {
key := fmt.Sprintf("%s:%d", shard.ServerID, shard.DiskID)
if !duplicateCheck[key] { // Avoid duplicates if EC shards are on same disk as volume replicas
shardIds := append([]uint32(nil), shard.ShardIds...)
// Free exactly the shards on this disk. Without an explicit
// impact the cleanup falls back to CalculateECShardCleanupImpact,
// which credits TotalShardsCount (14) regardless of ratio.
cleanupImpact := topology.StorageSlotChange{ShardSlots: -int32(len(shardIds))}
sources = append(sources, topology.TaskSourceSpec{
ServerID: shard.ServerID,
DiskID: shard.DiskID,
DataCenter: shard.DataCenter,
Rack: shard.Rack,
CleanupType: topology.CleanupECShards,
ShardIds: shardIds,
StorageImpact: &cleanupImpact,
})
duplicateCheck[key] = true
}
}
glog.V(2).Infof("Found %d volume replicas and %d existing EC shards for volume %d (total %d cleanup sources)",
len(replicaLocations), len(existingECShards), metric.VolumeID, len(sources))
// Convert shard destinations to TaskDestinationSpec. A destination may
// hold several shards (small clusters), so reserve capacity for the
// actual per-disk shard count that Place assigned (shardsPerPlan),
// which is exactly what createECTargets writes.
destinations := make([]topology.TaskDestinationSpec, len(shardDestinations))
for i, dest := range shardDestinations {
shardCount := len(shardsPerPlan[i])
shardImpact := topology.CalculateECShardStorageImpact(int32(shardCount), int64(expectedShardSize))
destSize := int64(expectedShardSize) * int64(shardCount)
destinations[i] = topology.TaskDestinationSpec{
ServerID: dest,
DiskID: shardDiskIDs[i],
StorageImpact: &shardImpact,
EstimatedSize: &destSize,
}
}
// Convert sources before mutating topology
sourcesProto, err := convertTaskSourcesToProtobuf(sources, metric.VolumeID, clusterInfo.ActiveTopology)
if err != nil {
glog.Warningf("Failed to convert sources for EC task on volume %d: %v, skipping", metric.VolumeID, err)
continue
}
err = clusterInfo.ActiveTopology.AddPendingTask(topology.TaskSpec{
TaskID: taskID,
TaskType: topology.TaskTypeErasureCoding,
VolumeID: metric.VolumeID,
VolumeSize: int64(metric.Size),
Sources: sources,
Destinations: destinations,
})
if err != nil {
glog.Warningf("Failed to add pending EC shard task to ActiveTopology for volume %d: %v", metric.VolumeID, err)
continue // Skip this volume if topology task addition fails
}
// Cross-volume in-cycle capacity is tracked by ActiveTopology via the
// pending task above, which the next volume's FromActiveTopology snapshot
// reflects; no separate planner reservation is needed.
glog.V(2).Infof("Added pending EC shard task %s to ActiveTopology for volume %d with %d cleanup sources and %d shard destinations",
taskID, metric.VolumeID, len(sources), len(multiPlan.Plans))
// Create unified sources and targets for EC task
result.TypedParams = &worker_pb.TaskParams{
TaskId: taskID, // Link to ActiveTopology pending task
VolumeId: metric.VolumeID,
Collection: metric.Collection,
VolumeSize: metric.Size, // Store original volume size for tracking changes
// Unified sources - all sources that will be processed/cleaned up
Sources: sourcesProto,
// Unified targets - all EC shard destinations
Targets: createECTargets(multiPlan, shardsPerPlan),
TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
ErasureCodingParams: createECTaskParams(dataShards, parityShards, metric.DiskType),
},
}
glog.V(1).Infof("Planned EC destinations for volume %d: %d shards across %d racks, %d DCs",
metric.VolumeID, len(multiPlan.Plans), multiPlan.SuccessfulRack, multiPlan.SuccessfulDCs)
} else {
glog.Warningf("No ActiveTopology available for destination planning in EC detection")
continue // Skip this volume if no topology available
}
glog.Infof("EC Detection: Successfully created EC task for volume %d, adding to results", metric.VolumeID)
results = append(results, result)
} else {
// Count debug reasons
if metric.Age < quietThreshold {
skippedQuietTime++
}
if metric.FullnessRatio < ecConfig.FullnessRatio {
skippedFullness++
}
if debugCount < 5 { // Limit to avoid spam
// Logic moved outside
}
debugCount++
}
}
// Log debug summary if no tasks were created
if len(results) == 0 && len(metrics) > 0 && !stoppedEarly {
totalVolumes := len(metrics)
glog.V(1).Infof("EC detection: No tasks created for %d volumes (skipped: %d already EC, %d too small, %d filtered, %d not quiet, %d not full, %d remote, %d too few nodes)",
totalVolumes, skippedAlreadyEC, skippedTooSmall, skippedCollectionFilter, skippedQuietTime, skippedFullness, skippedRemote, skippedTooFewNodes)
// Show details for first few volumes
for i, metric := range metrics {
if i >= 3 || metric.IsECVolume { // Limit to first 3 non-EC volumes
continue
}
sizeMB := float64(metric.Size) / (1024 * 1024)
glog.V(1).Infof("ERASURE CODING: Volume %d: size=%.1fMB (need ≥%dMB), age=%s (need ≥%s), fullness=%.1f%% (need ≥%.1f%%)",
metric.VolumeID, sizeMB, ecConfig.MinSizeMB, metric.Age.Truncate(time.Minute), quietThreshold.Truncate(time.Minute),
metric.FullnessRatio*100, ecConfig.FullnessRatio*100)
}
}
return results, hasMore, nil
}
// countTopologyNodes counts volume-server nodes in the active topology, used by
// the min-node safety gate.
func countTopologyNodes(at *topology.ActiveTopology) int {
if at == nil {
return 0
}
topo := at.GetTopologyInfo()
if topo == nil {
return 0
}
n := 0
for _, dc := range topo.DataCenterInfos {
for _, rack := range dc.RackInfos {
n += len(rack.DataNodeInfos)
}
}
return n
}
// buildNodeAddressMap resolves every node's server address once per detection
// cycle. ResolveServerAddress rebuilds the full node map on each call, so
// resolving per shard destination is O(destinations × topology); callers build
// this map once and look up addresses from it.
func buildNodeAddressMap(at *topology.ActiveTopology) map[string]string {
if at == nil {
return nil
}
allNodes := at.GetAllNodes()
m := make(map[string]string, len(allNodes))
for id, n := range allNodes {
m[id] = string(pb.NewServerAddressFromDataNode(n))
}
return m
}
// planECDestinations places all shards of the volume via the shared ecbalancer
// policy and returns the per-disk destination plans plus, parallel to them, the
// shard ids ecbalancer.Place assigned to each disk (so createECTargets and the
// capacity reservations use the real assignment, not a round-robin guess).
//
// snap is the cycle-wide placement snapshot, built once by the caller and reused
// across volumes: Place reserves the shards it assigns into it, so later volumes
// in the same detection cycle see the reduced capacity. Rebuilding it per volume
// is O(volumes × topology) and times out on large clusters.
//
// Encode is lenient (PlaceDurabilityFirst): it relaxes caps/anti-affinity/RP as
// needed rather than fail, and prefers the source disk type but spills if that
// type can't hold every shard. rp is the resolved replica placement (may be nil).
func planECDestinations(snap *ecbalancer.Topology, nodeAddresses map[string]string, metric *types.VolumeHealthMetrics, ecConfig *Config, rp *super_block.ReplicaPlacement, dataShards, parityShards int) (*topology.MultiDestinationPlan, [][]uint32, error) {
if snap == nil {
return nil, nil, fmt.Errorf("EC placement snapshot not available")
}
if dataShards <= 0 || parityShards <= 0 {
return nil, nil, fmt.Errorf("invalid EC ratio: dataShards=%d parityShards=%d", dataShards, parityShards)
}
totalShards := dataShards + parityShards
// Survive losing one disk: each disk holds at most parityShards shards,
// so we need at least ceil(totalShards / parityShards) disks.
minTotalDisks := (totalShards + parityShards - 1) / parityShards
expectedShardSize := uint64(metric.Size) / uint64(dataShards)
// Encode is greenfield: any EC shards already present for this volume are stale
// leftovers from a prior failed attempt, which the task deletes
// (cleanupStaleEcShards) before distributing the new shards. Release them so they
// don't occupy capacity or skew anti-affinity / per-disk caps during planning.
snap.ReleaseVolumeShards(metric.Collection, metric.VolumeID)
need := make([]int, totalShards)
for i := range need {
need[i] = i
}
res, err := snap.Place(metric.VolumeID, metric.Collection, need, ecbalancer.Constraints{
DiskType: metric.DiskType,
DiskTypePolicy: ecbalancer.DiskTypePrefer,
PreferredTags: ecConfig.PreferredTags,
ReplicaPlacement: rp,
Ratio: func(string) (int, int) { return dataShards, parityShards },
}, ecbalancer.PlaceDurabilityFirst)
if err != nil {
return nil, nil, err
}
if res.SpilledToOtherDiskType {
glog.Warningf("EC volume %d: placed shards outside preferred disk type %q", metric.VolumeID, metric.DiskType)
}
if res.SpilledOutsidePreferredTags {
glog.Warningf("EC volume %d: placed shards outside preferred tags %v", metric.VolumeID, ecConfig.PreferredTags)
}
if len(res.Relaxed) > 0 {
// Encode is best-effort (PlaceDurabilityFirst): it relaxes these constraints
// rather than defer when the cluster can't satisfy them. Surface it so a tight
// replica placement isn't silently weakened; rebalancing tightens the spread.
glog.Warningf("EC volume %d: placed with relaxed constraints %v; replica placement not fully satisfied (rebalancing will adjust)", metric.VolumeID, res.Relaxed)
}
// Group the per-shard destinations into one plan per (node,disk), iterating
// shard ids in order for determinism.
type diskGroup struct {
node, rack, dc string
diskID uint32
shards []uint32
}
type diskKey struct {
node string
diskID uint32
}
groups := make(map[diskKey]*diskGroup, totalShards)
order := make([]diskKey, 0, totalShards)
for sid := 0; sid < totalShards; sid++ {
d, ok := res.Destinations[sid]
if !ok {
return nil, nil, fmt.Errorf("EC volume %d: shard %d was not placed", metric.VolumeID, sid)
}
key := diskKey{node: d.Node, diskID: d.DiskID}
g := groups[key]
if g == nil {
g = &diskGroup{node: d.Node, rack: d.Rack, dc: d.DataCenter, diskID: d.DiskID}
groups[key] = g
order = append(order, key)
}
g.shards = append(g.shards, uint32(sid))
}
if len(order) < minTotalDisks {
return nil, nil, fmt.Errorf("placed onto %d disks, but EC %d+%d needs at least %d so no disk holds more than %d shards",
len(order), dataShards, parityShards, minTotalDisks, parityShards)
}
var plans []*topology.DestinationPlan
shardsPerPlan := make([][]uint32, 0, len(order))
rackCount := make(map[string]int)
dcCount := make(map[string]int)
for _, key := range order {
g := groups[key]
targetAddress, ok := nodeAddresses[g.node]
if !ok {
return nil, nil, fmt.Errorf("failed to resolve address for target server %s", g.node)
}
plans = append(plans, &topology.DestinationPlan{
TargetNode: g.node,
TargetAddress: targetAddress,
TargetDisk: g.diskID,
TargetRack: g.rack,
TargetDC: g.dc,
ExpectedSize: expectedShardSize,
})
shardsPerPlan = append(shardsPerPlan, g.shards)
rackCount[fmt.Sprintf("%s:%s", g.dc, g.rack)]++
dcCount[g.dc]++
}
glog.V(1).Infof("Planned EC destinations for volume %d (size=%d bytes): expected shard size=%d bytes, %d shards across %d disks, %d racks, %d DCs",
metric.VolumeID, metric.Size, expectedShardSize, totalShards, len(plans), len(rackCount), len(dcCount))
return &topology.MultiDestinationPlan{
Plans: plans,
TotalShards: totalShards,
SuccessfulRack: len(rackCount),
SuccessfulDCs: len(dcCount),
}, shardsPerPlan, nil
}
// createECTargets builds TaskTargets from the per-disk plans and the shard ids
// ecbalancer.Place assigned to each (shardsPerPlan is parallel to multiPlan.Plans).
func createECTargets(multiPlan *topology.MultiDestinationPlan, shardsPerPlan [][]uint32) []*worker_pb.TaskTarget {
targets := make([]*worker_pb.TaskTarget, 0, len(multiPlan.Plans))
for i, plan := range multiPlan.Plans {
shardIDs := shardsPerPlan[i]
targets = append(targets, &worker_pb.TaskTarget{
Node: plan.TargetAddress,
DiskId: plan.TargetDisk,
Rack: plan.TargetRack,
DataCenter: plan.TargetDC,
ShardIds: shardIDs,
EstimatedSize: plan.ExpectedSize,
})
glog.V(2).Infof("EC planning: target %s disk %d assigned shards %v", plan.TargetNode, plan.TargetDisk, shardIDs)
}
return targets
}
// convertTaskSourcesToProtobuf converts topology.TaskSourceSpec to worker_pb.TaskSource
func convertTaskSourcesToProtobuf(sources []topology.TaskSourceSpec, volumeID uint32, activeTopology *topology.ActiveTopology) ([]*worker_pb.TaskSource, error) {
var protobufSources []*worker_pb.TaskSource
for _, source := range sources {
serverAddress, err := workerutil.ResolveServerAddress(source.ServerID, activeTopology)
if err != nil {
return nil, fmt.Errorf("failed to resolve address for source server %s: %v", source.ServerID, err)
}
pbSource := &worker_pb.TaskSource{
Node: serverAddress,
DiskId: source.DiskID,
DataCenter: source.DataCenter,
Rack: source.Rack,
}
// Convert storage impact to estimated size
if source.EstimatedSize != nil {
pbSource.EstimatedSize = uint64(*source.EstimatedSize)
}
// Populated ShardIds is the wire-level marker that flags an
// EC-shard cleanup source; the worker routes it through
// cleanupStaleEcShards and skips it in getReplicas.
switch source.CleanupType {
case topology.CleanupVolumeReplica:
pbSource.VolumeId = volumeID
case topology.CleanupECShards:
pbSource.VolumeId = volumeID
pbSource.ShardIds = append([]uint32(nil), source.ShardIds...)
}
protobufSources = append(protobufSources, pbSource)
}
return protobufSources, nil
}
// createECTaskParams creates clean EC task parameters (destinations now in unified targets).
// sourceDiskType is forwarded to VolumeEcShardsMount so the resulting EC volume
// reports under the source's disk type rather than the target location's (#9423).
func createECTaskParams(dataShards, parityShards int, sourceDiskType string) *worker_pb.ErasureCodingTaskParams {
return &worker_pb.ErasureCodingTaskParams{
DataShards: int32(dataShards),
ParityShards: int32(parityShards),
SourceDiskType: sourceDiskType,
}
}
// findVolumeReplicaLocations finds all replica locations (server + disk) for the specified volume
// Uses O(1) indexed lookup for optimal performance on large clusters.
func findVolumeReplicaLocations(activeTopology *topology.ActiveTopology, volumeID uint32, collection string) []topology.VolumeReplica {
if activeTopology == nil {
return nil
}
return activeTopology.GetVolumeLocations(volumeID, collection)
}
// findExistingECShards finds existing EC shards for a volume (from previous failed EC attempts)
// Uses O(1) indexed lookup for optimal performance on large clusters.
func findExistingECShards(activeTopology *topology.ActiveTopology, volumeID uint32, collection string) []topology.VolumeReplica {
if activeTopology == nil {
return nil
}
return activeTopology.GetECShardLocations(volumeID, collection)
}
// cleanupOrphanSourceReplicas deletes any regular volume replicas still
// present in the topology for (volumeID, collection) after re-verifying that
// the full EC shard set is intact. Caller must hold expectedShards equal to
// the configured totalShards count. Issues VolumeDelete RPC to each replica
// server's address — that endpoint only touches the regular volume on the
// targeted server, never EC shards (those live in a separate store path).
// The cluster-wide `volume.delete` shell command is what would have nuked
// the EC shards too; the targeted RPC used here is safe by construction.
// Returns the count of replicas successfully deleted plus any error.
func cleanupOrphanSourceReplicas(ctx context.Context, clusterInfo *types.ClusterInfo, metric *types.VolumeHealthMetrics, expectedShards int) (int, error) {
if clusterInfo == nil || clusterInfo.ActiveTopology == nil {
return 0, fmt.Errorf("active topology unavailable")
}
if clusterInfo.GrpcDialOption == nil {
return 0, fmt.Errorf("grpc dial option unavailable")
}
// Re-verify shard completeness right before acting. Defensive: detection
// processes many volumes sequentially and the topology snapshot we built
// at start-of-detection could have lost shards in between (a volume
// server going down between iterations). Refusing to delete the source
// when we can no longer prove the shards are complete is the safer
// failure mode — the source replica is the only complete copy.
actualShards := countExistingEcShardsForVolume(clusterInfo.ActiveTopology, metric.VolumeID, metric.Collection)
if actualShards < expectedShards {
return 0, fmt.Errorf("EC shard set shrank between detection and cleanup (%d < %d); refusing to delete source replica", actualShards, expectedShards)
}
replicas := findVolumeReplicaLocations(clusterInfo.ActiveTopology, metric.VolumeID, metric.Collection)
if len(replicas) == 0 {
return 0, nil
}
deleted := 0
var deleteErrors []string
for _, replica := range replicas {
serverAddress := replica.ServerID
var isReadOnly bool
err := operation.WithVolumeServerClient(false, pb.ServerAddress(serverAddress), clusterInfo.GrpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
// Re-probe before deleting: only a still-readonly source replica is
// safe to remove. One that came back writable may have accepted
// writes the EC shards do not contain, so deleting it loses data.
status, statusErr := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: metric.VolumeID})
if statusErr != nil {
return statusErr
}
isReadOnly = status.GetIsReadOnly()
if !isReadOnly {
return nil
}
_, deleteErr := client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
VolumeId: metric.VolumeID,
OnlyEmpty: false,
})
return deleteErr
})
if err != nil {
deleteErrors = append(deleteErrors, fmt.Sprintf("server %s: %v", serverAddress, err))
continue
}
if !isReadOnly {
glog.Warningf("EC Detection: source replica for volume %d on %s is writable; not deleting (may hold writes the EC shards lack)", metric.VolumeID, serverAddress)
continue
}
deleted++
glog.V(1).Infof("EC Detection: deleted orphan regular replica for volume %d on %s", metric.VolumeID, serverAddress)
}
if len(deleteErrors) > 0 {
return deleted, fmt.Errorf("%d of %d replica delete(s) failed: %s", len(deleteErrors), len(replicas), strings.Join(deleteErrors, "; "))
}
return deleted, nil
}
// countExistingEcShardsForVolume counts (volumeID, collection) EC shards in the
// topology, counting only the single largest encode generation; see
// ec.CountExistingEcShardsForVolume for the generation semantics.
func countExistingEcShardsForVolume(activeTopology *topology.ActiveTopology, volumeID uint32, collection string) int {
if activeTopology == nil {
return 0
}
return ec.CountExistingEcShardsForVolume(activeTopology.GetTopologyInfo(), volumeID, collection)
}

View File

@@ -0,0 +1,93 @@
package erasure_coding
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestPlanECDestinationsPrefersSourceDiskType_FullCluster covers the
// #9423 happy path through the worker plumbing: every node carries both
// an HDD and an SSD disk, so the cluster has enough SSD slots to fully
// satisfy a 10+4 placement. With metric.DiskType="ssd", every selected
// destination must land on an SSD disk.
func TestPlanECDestinationsPrefersSourceDiskType_FullCluster(t *testing.T) {
// 14 nodes × 2 disks per node (hdd + ssd) — enough SSD slots alone
// for a 10+4 layout with one-shard-per-(server,disk) diversity.
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd", "ssd"}, 100, 0, "")
metric := &types.VolumeHealthMetrics{
VolumeID: 1,
Server: "10.0.0.1:8080",
Size: 100 * 1024 * 1024,
Collection: "",
DiskType: "ssd", // the property being plumbed end-to-end
}
snap := ecbalancer.FromActiveTopology(activeTopology, erasure_coding.DataShardsCount)
nodeAddresses := buildNodeAddressMap(activeTopology)
plan, _, err := planECDestinations(snap, nodeAddresses, metric, NewDefaultConfig(), nil, erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
require.NoError(t, err)
require.Len(t, plan.Plans, erasure_coding.TotalShardsCount)
// buildActiveTopology assigns DiskID by index into the diskTypes
// slice, so SSD is DiskID 1. Every plan entry must point at an SSD
// disk — anything else means metric.DiskType was dropped on the way
// to the placement layer.
for _, p := range plan.Plans {
assert.Equalf(t, uint32(1), p.TargetDisk,
"target %s diskID = %d, want SSD disk (DiskID=1)", p.TargetNode, p.TargetDisk)
}
}
// TestPlanECDestinationsSpillsToOtherDiskType_WhenPreferredScarce pins
// the prefer-with-spillover policy: only one node in the cluster has an
// SSD disk; the rest are HDD-only. A 10+4 placement must still succeed
// (no hard fail), filling the one SSD slot first and the remaining 13
// from HDD nodes. Without the spillover behavior this would either
// error out or starve the placement.
func TestPlanECDestinationsSpillsToOtherDiskType_WhenPreferredScarce(t *testing.T) {
// Start with every node carrying both HDD and SSD, then strip SSD
// from all but the first node so the SSD pool is too small alone.
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd", "ssd"}, 100, 0, "")
topo := activeTopology.GetTopologyInfo()
for _, dc := range topo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for i, node := range rack.DataNodeInfos {
if i == 0 {
continue // keep the first node's SSD
}
delete(node.DiskInfos, "ssd")
}
}
}
require.NoError(t, activeTopology.UpdateTopology(topo))
metric := &types.VolumeHealthMetrics{
VolumeID: 2,
Server: "10.0.0.1:8080",
Size: 100 * 1024 * 1024,
Collection: "",
DiskType: "ssd",
}
snap := ecbalancer.FromActiveTopology(activeTopology, erasure_coding.DataShardsCount)
nodeAddresses := buildNodeAddressMap(activeTopology)
plan, _, err := planECDestinations(snap, nodeAddresses, metric, NewDefaultConfig(), nil, erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
require.NoError(t, err)
require.Len(t, plan.Plans, erasure_coding.TotalShardsCount)
// Exactly one SSD target (the only SSD node), the rest on HDD.
ssdHits := 0
for _, p := range plan.Plans {
if p.TargetDisk == 1 {
ssdHits++
}
}
assert.Equalf(t, 1, ssdHits,
"expected exactly 1 SSD placement (the only SSD node in the cluster), got %d", ssdHits)
}

View File

@@ -0,0 +1,92 @@
package erasure_coding
import (
"fmt"
"testing"
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/stretchr/testify/require"
)
type ecShardSet struct {
bits uint32
encodeTsNs int64
}
func ecBits(shardIds ...int) uint32 {
var b uint32
for _, id := range shardIds {
b |= uint32(1) << uint(id)
}
return b
}
// buildEcTopology builds an ActiveTopology where each shard set lives on its own
// node, so countExistingEcShardsForVolume can be exercised across encode
// generations spread over different disks.
func buildEcTopology(t *testing.T, vid uint32, collection string, sets []ecShardSet) *topology.ActiveTopology {
t.Helper()
at := topology.NewActiveTopology(10)
nodes := make([]*master_pb.DataNodeInfo, 0, len(sets))
for i, set := range sets {
nodes = append(nodes, &master_pb.DataNodeInfo{
Id: fmt.Sprintf("10.0.0.%d:8080", i+1),
DiskInfos: map[string]*master_pb.DiskInfo{
"hdd": {
DiskId: 0,
MaxVolumeCount: 100,
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: vid, Collection: collection, EcIndexBits: set.bits, EncodeTsNs: set.encodeTsNs},
},
},
},
})
}
topologyInfo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{{Id: "rack1", DataNodeInfos: nodes}},
}},
}
require.NoError(t, at.UpdateTopology(topologyInfo))
return at
}
// Two interrupted encode runs whose shard sets overlap in count must NOT be
// unioned into a false-complete set. countExistingEcShardsForVolume returns the
// largest single generation's shard count, not the cross-run union.
func TestCountExistingEcShards_DoesNotUnionAcrossGenerations(t *testing.T) {
const vid = uint32(42)
// run A holds shards 0-6 (7 shards, encode_ts 1); run B holds shards 7-13
// (7 shards, encode_ts 2). The union would be all 14; the per-generation max
// is 7, so the orphan-source delete trigger (>= 14) must not fire.
at := buildEcTopology(t, vid, "", []ecShardSet{
{bits: ecBits(0, 1, 2, 3, 4, 5, 6), encodeTsNs: 1},
{bits: ecBits(7, 8, 9, 10, 11, 12, 13), encodeTsNs: 2},
})
require.Equal(t, 7, countExistingEcShardsForVolume(at, vid, ""))
}
// A single complete generation still counts as complete, so genuine stuck-source
// cleanup keeps working.
func TestCountExistingEcShards_SingleCompleteGeneration(t *testing.T) {
const vid = uint32(43)
at := buildEcTopology(t, vid, "", []ecShardSet{
{bits: ecBits(0, 1, 2, 3, 4, 5, 6), encodeTsNs: 5},
{bits: ecBits(7, 8, 9, 10, 11, 12, 13), encodeTsNs: 5},
})
require.Equal(t, 14, countExistingEcShardsForVolume(at, vid, ""))
}
// Shards from pre-upgrade servers report encode_ts_ns==0; a complete 0-generation
// set is still recognized as complete (its own bucket).
func TestCountExistingEcShards_PreUpgradeZeroGeneration(t *testing.T) {
const vid = uint32(44)
at := buildEcTopology(t, vid, "", []ecShardSet{
{bits: ecBits(0, 1, 2, 3, 4, 5, 6), encodeTsNs: 0},
{bits: ecBits(7, 8, 9, 10, 11, 12, 13), encodeTsNs: 0},
})
require.Equal(t, 14, countExistingEcShardsForVolume(at, vid, ""))
}

View File

@@ -0,0 +1,499 @@
package erasure_coding
import (
"context"
"fmt"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestPlanECDestinationsUsesPlanner(t *testing.T) {
activeTopology := buildActiveTopology(t, 7, []string{"hdd", "ssd"}, 100, 0, "")
metric := &types.VolumeHealthMetrics{
VolumeID: 1,
Server: "10.0.0.1:8080",
Size: 100 * 1024 * 1024,
Collection: "",
}
snap := ecbalancer.FromActiveTopology(activeTopology, erasure_coding.DataShardsCount)
nodeAddresses := buildNodeAddressMap(activeTopology)
plan, shardsPerPlan, err := planECDestinations(snap, nodeAddresses, metric, NewDefaultConfig(), nil, erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
require.NoError(t, err)
require.NotNil(t, plan)
requireAllShardsPlaced(t, plan, shardsPerPlan)
}
// TestDetectionSkipsWhenECShardsAlreadyExist guards against issue #9448: a
// regular replica that survived a previous successful EC encode (source
// delete didn't clean it up for some reason) gets re-proposed for encoding,
// the new encode collides with the already-mounted shards on the targets
// ("ec volume %d is mounted; refusing overwrite"), and detection loops
// forever on the same volume. Detection must see the existing shards and
// skip the volume so an admin can clean it up out-of-band.
//
// The guard fires ONLY when the EC shard set is complete (count >=
// totalShards), so a partially-distributed previous attempt still falls
// through to the existing recovery branch in the encode path.
func TestDetectionSkipsWhenECShardsAlreadyExist(t *testing.T) {
const volumeID uint32 = 42
activeTopology := buildStuckSourceTopology(t, volumeID, erasure_coding.TotalShardsCount)
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
metrics := buildStuckSourceMetrics(volumeID, "127.0.0.1:8080")
results, hasMore, err := Detection(context.Background(), metrics, clusterInfo, NewDefaultConfig(), 0)
require.NoError(t, err)
require.False(t, hasMore)
require.Empty(t, results, "stuck source replica with all EC shards present must not yield a new encoding proposal")
}
// TestDetectionAllowsRegularReplicaWhenShardsPartial covers the partial-EC
// branch of the #9448 guard: when fewer than totalShards exist, the volume
// is allowed to flow through to the normal encoding path so the existing
// recovery branch (the `existingECShards` block in the encode arm) can fold
// the partial shards into the new task. A bug here would either (a) skip
// the volume entirely or (b) emit a proposal that later collides on the
// mounted shards.
func TestDetectionAllowsRegularReplicaWhenShardsPartial(t *testing.T) {
const volumeID uint32 = 43
activeTopology := buildStuckSourceTopology(t, volumeID, erasure_coding.DataShardsCount-1)
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
metrics := buildStuckSourceMetrics(volumeID, "127.0.0.1:8080")
results, _, err := Detection(context.Background(), metrics, clusterInfo, NewDefaultConfig(), 0)
require.NoError(t, err)
// Partial shards are not a "stuck source" — the encode arm must keep
// its chance to either propose a fresh task that folds the partial
// shards into cleanup, or fail planning on the constrained topology.
// We don't require len(results) > 0 because the constrained topology
// (one disk per node, the orphaned shards already taking slots) can
// legitimately fail destination planning. The assertion that matters
// is: the #9448 guard did NOT silently swallow the volume into a
// skippedAlreadyEC counter, and any emitted result is still an EC
// task and not a no-op.
for _, r := range results {
require.Equal(t, types.TaskTypeErasureCoding, r.TaskType, "any emitted result should still be an EC task, not a no-op")
}
}
// buildStuckSourceTopology constructs a topology that mimics the #9448 stuck
// state: a regular volume replica on node 0 plus `presentShardCount` EC
// shards distributed across nodes 0..presentShardCount-1.
func buildStuckSourceTopology(t *testing.T, volumeID uint32, presentShardCount int) *topology.ActiveTopology {
t.Helper()
require.LessOrEqual(t, presentShardCount, erasure_coding.TotalShardsCount)
activeTopology := topology.NewActiveTopology(10)
nodes := make([]*master_pb.DataNodeInfo, 0, erasure_coding.TotalShardsCount)
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
nodeID := fmt.Sprintf("127.0.0.1:%d", 8080+i)
diskInfo := &master_pb.DiskInfo{
DiskId: 0,
VolumeCount: 1,
MaxVolumeCount: 100,
}
if i < presentShardCount {
diskInfo.EcShardInfos = []*master_pb.VolumeEcShardInformationMessage{{
Id: volumeID,
Collection: "",
EcIndexBits: uint32(1) << uint(i),
DiskId: 0,
}}
}
if i == 0 {
diskInfo.VolumeInfos = []*master_pb.VolumeInformationMessage{{
Id: volumeID,
DiskId: 0,
DiskType: "hdd",
Size: 200 * 1024 * 1024,
}}
}
nodes = append(nodes, &master_pb.DataNodeInfo{
Id: nodeID,
DiskInfos: map[string]*master_pb.DiskInfo{"hdd": diskInfo},
})
}
require.NoError(t, activeTopology.UpdateTopology(&master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{{
Id: "rack1",
DataNodeInfos: nodes,
}},
}},
}))
return activeTopology
}
// buildStuckSourceMetrics returns a metric that already satisfies the EC
// criteria (Age, FullnessRatio, Size), with `Age` derived from `LastModified`
// so the two fields stay consistent for any reader.
func buildStuckSourceMetrics(volumeID uint32, server string) []*types.VolumeHealthMetrics {
lastModified := time.Now().Add(-2 * time.Hour)
return []*types.VolumeHealthMetrics{{
VolumeID: volumeID,
Server: server,
Size: 200 * 1024 * 1024,
Collection: "",
FullnessRatio: 0.96,
LastModified: lastModified,
Age: time.Since(lastModified),
}}
}
// TestCountExistingEcShardsForVolume verifies that the helper walks the
// EcIndexBits bitmap (not just len(EcShardInfos)) so it correctly counts
// distinct shard ids even when a single info entry on one disk carries
// multiple shards.
func TestCountExistingEcShardsForVolume(t *testing.T) {
const volumeID uint32 = 99
activeTopology := topology.NewActiveTopology(10)
require.NoError(t, activeTopology.UpdateTopology(&master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{{
Id: "rack1",
DataNodeInfos: []*master_pb.DataNodeInfo{
{
Id: "127.0.0.1:8080",
DiskInfos: map[string]*master_pb.DiskInfo{
"hdd": {
DiskId: 0,
MaxVolumeCount: 100,
// One info entry, three shards present (ids 0, 2, 5).
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{{
Id: volumeID,
Collection: "",
EcIndexBits: (uint32(1) << 0) | (uint32(1) << 2) | (uint32(1) << 5),
DiskId: 0,
}},
},
},
},
{
Id: "127.0.0.1:8081",
DiskInfos: map[string]*master_pb.DiskInfo{
"hdd": {
DiskId: 0,
MaxVolumeCount: 100,
// One info entry, one shard (id 3) — overlaps with neither.
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{{
Id: volumeID,
Collection: "",
EcIndexBits: uint32(1) << 3,
DiskId: 0,
}},
},
},
},
},
}},
}},
}))
assert.Equal(t, 4, countExistingEcShardsForVolume(activeTopology, volumeID, ""))
assert.Equal(t, 0, countExistingEcShardsForVolume(activeTopology, volumeID, "other-collection"))
assert.Equal(t, 0, countExistingEcShardsForVolume(nil, volumeID, ""))
}
func TestDetectionContextCancellation(t *testing.T) {
activeTopology := buildActiveTopology(t, 5, []string{"hdd", "ssd"}, 50, 0, "")
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
metrics := buildVolumeMetricsForIDs(50)
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, _, err := Detection(ctx, metrics, clusterInfo, NewDefaultConfig(), 0)
require.ErrorIs(t, err, context.Canceled)
}
// The admin UI collection filter takes a list, and the two mode sentinels must
// not be read as a collection name.
func TestDetectionCollectionFilter(t *testing.T) {
for _, tc := range []struct {
filter string
want int
}{
{filter: "", want: 3},
{filter: "ALL_COLLECTIONS", want: 3},
{filter: "collection-a", want: 3},
{filter: "collection-a,collection-b", want: 3},
{filter: " collection-b , collection-a ", want: 3},
{filter: "collection-*", want: 3},
{filter: "collection-b,collection-c", want: 0},
{filter: "collection-ab", want: 0},
} {
t.Run(tc.filter, func(t *testing.T) {
// A fresh topology per case: planned destinations reserve capacity.
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd"}, 20, 0, "collection-a")
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
metrics := buildVolumeMetricsForIDs(3)
for _, metric := range metrics {
metric.Collection = "collection-a"
}
config := NewDefaultConfig()
config.CollectionFilter = tc.filter
results, _, err := Detection(context.Background(), metrics, clusterInfo, config, 0)
require.NoError(t, err)
assert.Len(t, results, tc.want)
})
}
}
func TestDetectionMaxResultsHonorsLimit(t *testing.T) {
// One node per shard so each shard gets its own disk (#9369).
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd"}, 20, 0, "")
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
metrics := buildVolumeMetricsForIDs(3)
results, hasMore, err := Detection(context.Background(), metrics, clusterInfo, NewDefaultConfig(), 1)
require.NoError(t, err)
assert.Len(t, results, 1)
assert.True(t, hasMore)
}
// #9369: 7 servers × 2 physical HDDs must yield 14 distinct (server, disk_id)
// destinations, not 7 destinations doubled up on the same disk.
func TestPlanECDestinationsSpreadsAcrossPhysicalDisks(t *testing.T) {
const numServers = 7
const disksPerServer = 2
activeTopology := topology.NewActiveTopology(10)
nodes := make([]*master_pb.DataNodeInfo, 0, numServers)
for i := 1; i <= numServers; i++ {
volumeInfos := make([]*master_pb.VolumeInformationMessage, 0, disksPerServer)
for d := uint32(0); d < disksPerServer; d++ {
volumeInfos = append(volumeInfos, &master_pb.VolumeInformationMessage{
Id: uint32(i*100 + int(d)),
DiskId: d,
DiskType: "hdd",
})
}
nodes = append(nodes, &master_pb.DataNodeInfo{
Id: fmt.Sprintf("127.0.0.1:%d", 8080+i),
DiskInfos: map[string]*master_pb.DiskInfo{
"hdd": {
DiskId: 0,
VolumeCount: int64(disksPerServer),
MaxVolumeCount: 200,
VolumeInfos: volumeInfos,
},
},
})
}
require.NoError(t, activeTopology.UpdateTopology(&master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{{
Id: "rack1",
DataNodeInfos: nodes,
}},
}},
}))
metric := &types.VolumeHealthMetrics{
VolumeID: 42,
Server: "127.0.0.1:8081",
Size: 100 * 1024 * 1024,
Collection: "",
}
snap := ecbalancer.FromActiveTopology(activeTopology, erasure_coding.DataShardsCount)
nodeAddresses := buildNodeAddressMap(activeTopology)
plan, shardsPerPlan, err := planECDestinations(snap, nodeAddresses, metric, NewDefaultConfig(), nil, erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
require.NoError(t, err)
require.NotNil(t, plan)
requireAllShardsPlaced(t, plan, shardsPerPlan)
}
func TestPlanECDestinationsFailsWithInsufficientCapacity(t *testing.T) {
activeTopology := buildActiveTopology(t, 1, []string{"hdd"}, 1, 1, "")
metric := &types.VolumeHealthMetrics{
VolumeID: 2,
Server: "10.0.0.1:8080",
Size: 10 * 1024 * 1024,
Collection: "",
}
snap := ecbalancer.FromActiveTopology(activeTopology, erasure_coding.DataShardsCount)
nodeAddresses := buildNodeAddressMap(activeTopology)
_, _, err := planECDestinations(snap, nodeAddresses, metric, NewDefaultConfig(), nil, erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
require.Error(t, err)
}
// #9586: with fewer single-disk servers than total shards, EC must still plan
// by packing several shards onto a disk (ec.encode's "4,4,3,3" fallback) rather
// than refusing. The reporter has 8 single-disk servers across 3 racks and a
// 10+4 scheme — 8 disks for 14 shards. minTotalDisks (ceil(14/4)=4) keeps any
// disk under parityShards shards, so durability holds.
func TestPlanECDestinationsPacksWhenFewerDisksThanShards(t *testing.T) {
const numServers = 8
// rack3 holds 4 servers, rack1 and rack2 hold 2 each, mirroring the report.
racks := []string{"rack3", "rack3", "rack3", "rack3", "rack1", "rack1", "rack2", "rack2"}
activeTopology := topology.NewActiveTopology(10)
rackNodes := make(map[string][]*master_pb.DataNodeInfo)
for i := 0; i < numServers; i++ {
nodeID := fmt.Sprintf("192.168.1.%d:%d", 143+i/3, 8080+i)
rackNodes[racks[i]] = append(rackNodes[racks[i]], &master_pb.DataNodeInfo{
Id: nodeID,
DiskInfos: map[string]*master_pb.DiskInfo{
"hdd": {
DiskId: 0,
VolumeCount: 1,
MaxVolumeCount: 200,
VolumeInfos: []*master_pb.VolumeInformationMessage{{
Id: uint32(i + 1),
DiskId: 0,
DiskType: "hdd",
}},
},
},
})
}
rackInfos := make([]*master_pb.RackInfo, 0, len(rackNodes))
for _, rackID := range []string{"rack1", "rack2", "rack3"} {
rackInfos = append(rackInfos, &master_pb.RackInfo{Id: rackID, DataNodeInfos: rackNodes[rackID]})
}
require.NoError(t, activeTopology.UpdateTopology(&master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{Id: "dc1", RackInfos: rackInfos}},
}))
metric := &types.VolumeHealthMetrics{
VolumeID: 4569,
Server: "192.168.1.145:8081",
Size: 100 * 1024 * 1024,
Collection: "",
}
snap := ecbalancer.FromActiveTopology(activeTopology, erasure_coding.DataShardsCount)
nodeAddresses := buildNodeAddressMap(activeTopology)
plan, shardsPerPlan, err := planECDestinations(snap, nodeAddresses, metric, NewDefaultConfig(), nil, erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
require.NoError(t, err)
require.NotNil(t, plan)
// Packed onto the available disks: more than one shard per disk but never more
// than the 8 disks, and at least the durability floor of distinct disks.
require.LessOrEqual(t, len(plan.Plans), numServers)
require.GreaterOrEqual(t, len(plan.Plans), (erasure_coding.TotalShardsCount+erasure_coding.ParityShardsCount-1)/erasure_coding.ParityShardsCount)
// createECTargets must cover all 14 shards exactly once, packing onto the
// available disks without any disk exceeding parityShards shards.
targets := createECTargets(plan, shardsPerPlan)
require.Equal(t, len(plan.Plans), len(targets))
seenShards := make(map[uint32]bool)
for _, target := range targets {
require.LessOrEqual(t, len(target.ShardIds), erasure_coding.ParityShardsCount,
"no disk may hold more than parityShards shards, else losing it loses the volume")
for _, shardId := range target.ShardIds {
require.False(t, seenShards[shardId], "shard %d assigned to more than one target", shardId)
seenShards[shardId] = true
}
}
require.Len(t, seenShards, erasure_coding.TotalShardsCount, "every shard must be placed exactly once")
}
// requireAllShardsPlaced asserts every EC shard landed exactly once, on a distinct
// (node,disk) target, with no disk holding more than parityShards shards (so losing
// any one disk cannot lose the volume). shardsPerPlan is parallel to plan.Plans.
func requireAllShardsPlaced(t *testing.T, plan *topology.MultiDestinationPlan, shardsPerPlan [][]uint32) {
t.Helper()
require.Equal(t, len(plan.Plans), len(shardsPerPlan), "one shard list per plan entry")
keys := make(map[string]bool, len(plan.Plans))
seen := make(map[uint32]bool)
for i, p := range plan.Plans {
key := fmt.Sprintf("%s:%d", p.TargetNode, p.TargetDisk)
require.False(t, keys[key], "duplicate (node,disk) target %s", key)
keys[key] = true
require.LessOrEqual(t, len(shardsPerPlan[i]), erasure_coding.ParityShardsCount,
"disk %s holds %d shards, over parityShards", key, len(shardsPerPlan[i]))
for _, s := range shardsPerPlan[i] {
require.False(t, seen[s], "shard %d placed more than once", s)
seen[s] = true
}
}
require.Len(t, seen, erasure_coding.TotalShardsCount, "every shard must be placed exactly once")
}
func buildVolumeMetricsForIDs(count int) []*types.VolumeHealthMetrics {
metrics := make([]*types.VolumeHealthMetrics, 0, count)
now := time.Now()
for id := 1; id <= count; id++ {
metrics = append(metrics, &types.VolumeHealthMetrics{
VolumeID: uint32(id),
Server: "10.0.0.1:8080",
Size: 200 * 1024 * 1024,
Collection: "",
FullnessRatio: 0.96,
LastModified: now.Add(-2 * time.Hour),
Age: 2 * time.Hour,
})
}
return metrics
}
func buildActiveTopology(t *testing.T, nodeCount int, diskTypes []string, maxVolumeCount, usedVolumeCount int64, collection string) *topology.ActiveTopology {
t.Helper()
activeTopology := topology.NewActiveTopology(10)
nodes := make([]*master_pb.DataNodeInfo, 0, nodeCount)
for i := 1; i <= nodeCount; i++ {
diskInfos := make(map[string]*master_pb.DiskInfo)
for diskIndex, diskType := range diskTypes {
used := usedVolumeCount
if used > maxVolumeCount {
used = maxVolumeCount
}
volumeInfos := make([]*master_pb.VolumeInformationMessage, 0, 200)
for vid := 1; vid <= 200; vid++ {
volumeInfos = append(volumeInfos, &master_pb.VolumeInformationMessage{
Id: uint32(vid),
Collection: collection,
DiskId: uint32(diskIndex),
})
}
diskInfos[diskType] = &master_pb.DiskInfo{
DiskId: uint32(diskIndex),
VolumeCount: used,
MaxVolumeCount: maxVolumeCount,
VolumeInfos: volumeInfos,
}
}
nodes = append(nodes, &master_pb.DataNodeInfo{
Id: fmt.Sprintf("10.0.0.%d:8080", i),
DiskInfos: diskInfos,
})
}
topologyInfo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{
{
Id: "rack1",
DataNodeInfos: nodes,
},
},
},
},
}
require.NoError(t, activeTopology.UpdateTopology(topologyInfo))
return activeTopology
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,113 @@
package erasure_coding
import (
"context"
"net/http"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// One reachable replica + one unreachable: the reachable delete still
// succeeds, and the function surfaces an error naming the failure.
func TestDeleteOriginalVolumeSurfacesReplicaFailures(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(91842)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 918420, 0x91842042)
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
[]byte("delete-surface-content-for-issue-9184"))
_ = framework.ReadAllAndClose(t, uploadResp)
require.Equal(t, http.StatusCreated, uploadResp.StatusCode)
task := NewErasureCodingTask(
"delete-surface-fix",
clusterHarness.VolumeServerAddress(),
volumeID,
"",
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
unreachable := "127.0.0.1:1"
task.sources = []*worker_pb.TaskSource{
{
Node: clusterHarness.VolumeServerAddress(),
VolumeId: volumeID,
},
{
Node: unreachable,
VolumeId: volumeID,
},
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
err := task.deleteOriginalVolume(ctx)
require.Error(t, err, "deleteOriginalVolume must surface replica delete failures (#9184)")
require.Contains(t, err.Error(), unreachable,
"returned error should name the replica that failed: %v", err)
require.True(t,
strings.Contains(err.Error(), "failed to delete"),
"returned error should describe what failed: %v", err)
_, statusErr := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
require.Error(t, statusErr,
"reachable replica %d should have been deleted before failure was surfaced", volumeID)
}
func TestDeleteOriginalVolumeSucceedsWhenAllReplicasReachable(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(91844)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 918440, 0x91844042)
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
[]byte("delete-happy-path-content-for-issue-9184"))
_ = framework.ReadAllAndClose(t, uploadResp)
require.Equal(t, http.StatusCreated, uploadResp.StatusCode)
task := NewErasureCodingTask(
"delete-happy-path",
clusterHarness.VolumeServerAddress(),
volumeID,
"",
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
task.sources = []*worker_pb.TaskSource{
{Node: clusterHarness.VolumeServerAddress(), VolumeId: volumeID},
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
require.NoError(t, task.deleteOriginalVolume(ctx))
_, statusErr := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
require.Error(t, statusErr, "volume %d should be gone after successful delete", volumeID)
}

View File

@@ -0,0 +1,95 @@
package erasure_coding
import (
"context"
"errors"
"net/http"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
func TestReplicasPendingDelete(t *testing.T) {
replicas := []string{"a:8080", "b:8080", "c:8080"}
require.Equal(t, replicas, replicasPendingDelete(replicas, nil),
"nil swept set leaves the list unchanged")
require.ElementsMatch(t, []string{"a:8080", "c:8080"},
replicasPendingDelete(replicas, map[string]bool{"b:8080": true}),
"swept node is excluded")
require.Empty(t,
replicasPendingDelete(replicas, map[string]bool{"a:8080": true, "b:8080": true, "c:8080": true}),
"all swept yields nothing left to delete")
}
// The sweep leaves a node in place only when VolumeDelete(OnlyEmpty) reports the
// volume holds data or no longer exists. An unreachable node leaves its state
// unknown and must propagate so the encode fails rather than force-deleting it
// later over the shared .vif.
func TestIsExpectedSweepSkip(t *testing.T) {
require.True(t, isExpectedSweepSkip(errors.New("DeleteVolume 5: volume not empty")))
require.True(t, isExpectedSweepSkip(errors.New("volume 5 not found on disk")))
require.False(t, isExpectedSweepSkip(errors.New("rpc error: code = Unavailable desc = connection refused")))
require.False(t, isExpectedSweepSkip(errors.New("context deadline exceeded")))
}
// The pre-distribute sweep must delete a 0-byte stub replica (so its shared
// <collection>_<vid>.vif can't be clobbered by a later original-delete and
// can't shadow the incoming EC files) while leaving a data-bearing replica
// untouched — VolumeDelete(OnlyEmpty=true) is the guard. A data-bearing
// replica is deleted later, only after the EC shard set is verified.
func TestSweepEmptyReplicasDeletesStubKeepsData(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
server := clusterHarness.VolumeServerAddress()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// An empty allocated volume is a superblock-only stub.
const stubVol = uint32(9479)
framework.AllocateVolume(t, grpcClient, stubVol, "ec-stub")
stubTask := newSweepTaskForTest(server, stubVol, "ec-stub", dialOption)
require.NoError(t, stubTask.sweepEmptyReplicas(ctx))
require.True(t, stubTask.emptyReplicasDeleted[server], "stub server should be recorded as swept")
_, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: stubVol})
require.Error(t, err, "empty stub volume must be deleted by the sweep")
// A volume holding data must survive the sweep.
const dataVol = uint32(9480)
framework.AllocateVolume(t, grpcClient, dataVol, "ec-data")
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(dataVol, 948000, 0x9480CAFE)
upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("real-data-keep-me"))
_ = framework.ReadAllAndClose(t, upResp)
require.Equal(t, http.StatusCreated, upResp.StatusCode)
dataTask := newSweepTaskForTest(server, dataVol, "ec-data", dialOption)
require.NoError(t, dataTask.sweepEmptyReplicas(ctx), "a not-empty refusal is expected, not an error")
require.False(t, dataTask.emptyReplicasDeleted[server], "data-bearing server must not be recorded as swept")
_, err = grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: dataVol})
require.NoError(t, err, "data-bearing volume must survive the sweep")
}
func newSweepTaskForTest(server string, volumeID uint32, collection string, dialOption grpc.DialOption) *ErasureCodingTask {
task := NewErasureCodingTask("sweep-"+collection, server, volumeID, collection, dialOption)
task.sources = []*worker_pb.TaskSource{{Node: server, VolumeId: volumeID}}
return task
}

View File

@@ -0,0 +1,159 @@
package erasure_coding
import (
"context"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// End-to-end on two servers: a volume with a real replica on server A and a
// 0-byte stub replica of the same id on server B (an interrupted-encode
// leftover). After a full EC encode the cluster must end in exactly one valid
// layout — the complete shard set split across A and B, each with .ecx/.vif —
// and every wrong file must be gone: both regular .dat files (source deleted
// after verify, stub swept before distribute), no shard on the wrong server,
// and B's shared <collection>_<vid>.vif intact rather than clobbered by the
// stub's delete.
func TestEcEncodeLeavesRightFilesAndRemovesStubAndSource(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
cluster := framework.StartMultiVolumeCluster(t, matrix.P1(), 2)
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
const (
volumeID = uint32(9490)
collection = "ec-e2e"
)
addrA := serverAddress(cluster, 0)
addrB := serverAddress(cluster, 1)
connA, clientA := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
defer connA.Close()
connB, clientB := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
defer connB.Close()
// Server A: real source replica with data.
framework.AllocateVolume(t, clientA, volumeID, collection)
httpClient := framework.NewHTTPClient()
for i := 0; i < 8; i++ {
fid := framework.NewFileID(volumeID, uint64(948000+i), 0x9490CA00+uint32(i))
payload := make([]byte, 4096)
for j := range payload {
payload[j] = byte(i + 1)
}
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, payload)
_ = framework.ReadAllAndClose(t, resp)
require.Equal(t, http.StatusCreated, resp.StatusCode)
}
// Server B: empty stub replica of the same volume id.
framework.AllocateVolume(t, clientB, volumeID, collection)
dataShards := int(erasure_coding.DataShardsCount)
totalShards := int(erasure_coding.DataShardsCount + erasure_coding.ParityShardsCount)
aShards := shardRange(0, dataShards) // 0..DataShardsCount-1 on A
bShards := shardRange(dataShards, totalShards) // parity range on B
task := NewErasureCodingTask("ec-e2e", addrA, volumeID, collection, dialOption)
params := &worker_pb.TaskParams{
VolumeId: volumeID,
Collection: collection,
Sources: []*worker_pb.TaskSource{
{Node: addrA, VolumeId: volumeID},
{Node: addrB, VolumeId: volumeID},
},
Targets: []*worker_pb.TaskTarget{
{Node: addrA, ShardIds: aShards},
{Node: addrB, ShardIds: bShards},
},
TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
DataShards: erasure_coding.DataShardsCount,
ParityShards: erasure_coding.ParityShardsCount,
WorkingDir: t.TempDir(),
},
},
}
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
require.NoError(t, task.Execute(ctx, params))
dirA := filepath.Join(cluster.BaseDir(), "volume0")
dirB := filepath.Join(cluster.BaseDir(), "volume1")
base := fmt.Sprintf("%s_%d", collection, volumeID)
// Both original regular volumes are gone: A's source deleted after verify,
// B's stub swept before distribute.
requireAbsent(t, dirA, base+".dat")
requireAbsent(t, dirA, base+".idx")
requireAbsent(t, dirB, base+".dat")
requireAbsent(t, dirB, base+".idx")
// Each server holds exactly its assigned shards, plus index/info sidecars.
for _, id := range aShards {
requirePresent(t, dirA, fmt.Sprintf("%s.ec%02d", base, id))
}
for _, id := range bShards {
requireAbsent(t, dirA, fmt.Sprintf("%s.ec%02d", base, id))
}
requirePresent(t, dirA, base+".ecx")
requirePresent(t, dirA, base+".vif")
for _, id := range bShards {
requirePresent(t, dirB, fmt.Sprintf("%s.ec%02d", base, id))
}
for _, id := range aShards {
requireAbsent(t, dirB, fmt.Sprintf("%s.ec%02d", base, id))
}
requirePresent(t, dirB, base+".ecx")
// The shared .vif must survive on B: the stub was deleted before the EC
// files landed, so deleteOriginalVolume never ran removeVolumeFiles there.
requirePresent(t, dirB, base+".vif")
}
// serverAddress builds the SeaweedFS ip:httpPort.grpcPort address the worker's
// gRPC client decodes, from the multi-cluster's separate admin and grpc ports.
func serverAddress(c *framework.MultiVolumeCluster, index int) string {
_, grpcPort, err := net.SplitHostPort(c.VolumeGRPCAddress(index))
if err != nil {
panic(err)
}
return c.VolumeAdminAddress(index) + "." + grpcPort
}
func shardRange(start, end int) []uint32 {
ids := make([]uint32, 0, end-start)
for i := start; i < end; i++ {
ids = append(ids, uint32(i))
}
return ids
}
func requirePresent(t *testing.T, dir, name string) {
t.Helper()
_, err := os.Stat(filepath.Join(dir, name))
require.NoError(t, err, "expected %s to be present in %s", name, dir)
}
func requireAbsent(t *testing.T, dir, name string) {
t.Helper()
_, err := os.Stat(filepath.Join(dir, name))
require.True(t, os.IsNotExist(err), "expected %s to be absent in %s, stat err=%v", name, dir, err)
}

View File

@@ -0,0 +1,173 @@
package erasure_coding
import (
"context"
"fmt"
"net/http"
"os"
"path/filepath"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// Reproduces the issue-9478 volume-13 layout end to end on a multi-server,
// multi-disk cluster and asserts the cluster converges to exactly one valid EC
// layout. Before the encode the volume is in the stuck state: a real .dat
// source, a 0-byte stub replica left by an interrupted encode, and partial
// stale EC shards from that attempt. After the worker runs (sweep stubs ->
// clear stale shards -> distribute -> mount -> verify -> delete originals):
//
// - every assigned shard is present on its target disk, with .ecx/.vif
// alongside it (the EC volume's info file survives even when shards land on
// the source's own disk);
// - no regular .dat/.idx remains anywhere — the source is deleted after
// verify, the stub was swept before distribute;
// - the stale shards that aren't part of the new assignment are gone.
func TestEcEncodeJulorLayoutConverges(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
const disksPerServer = 2
cluster := framework.StartMultiVolumeClusterWithDisks(t, matrix.P1(), 3, disksPerServer)
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
const (
volumeID = uint32(9500)
collection = "ec-julor"
)
// S0: real source. S1: 0-byte stub + EC target. S2: pure EC target.
const (
srcServer = 0
stubServer = 1
)
addr := func(i int) string { return serverAddress(cluster, i) }
conns := make([]*grpc.ClientConn, 3)
clients := make([]volume_server_pb.VolumeServerClient, 3)
for i := 0; i < 3; i++ {
conns[i], clients[i] = framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(i))
defer conns[i].Close()
}
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
defer cancel()
// S0: real source replica with data.
framework.AllocateVolume(t, clients[srcServer], volumeID, collection)
httpClient := framework.NewHTTPClient()
for i := 0; i < 8; i++ {
fid := framework.NewFileID(volumeID, uint64(950000+i), 0x9500CA00+uint32(i))
payload := make([]byte, 4096)
for j := range payload {
payload[j] = byte(i + 1)
}
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(srcServer), fid, payload)
_ = framework.ReadAllAndClose(t, resp)
require.Equal(t, http.StatusCreated, resp.StatusCode)
}
// Partial stale EC shards from a previous interrupted encode, mounted on the
// source. cleanupStaleEcShards must clear them before re-distribute.
_, err := clients[srcServer].VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID, Collection: collection,
})
require.NoError(t, err)
staleShards := []uint32{11, 12, 13}
_, err = clients[srcServer].VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID, Collection: collection, ShardIds: staleShards,
})
require.NoError(t, err)
// S1: a 0-byte stub replica of the same volume (interrupted-encode leftover).
framework.AllocateVolume(t, clients[stubServer], volumeID, collection)
// Plan: 14 shards spread across all three servers and both disks. The source
// server (S0) receives shards too, so its regular volume coexists with EC on
// the same disk before deletion.
targets := []*worker_pb.TaskTarget{
{Node: addr(0), DiskId: 0, ShardIds: shardRange(0, 3)}, // S0 d0: 0,1,2
{Node: addr(0), DiskId: 1, ShardIds: shardRange(3, 5)}, // S0 d1: 3,4
{Node: addr(1), DiskId: 0, ShardIds: shardRange(5, 8)}, // S1 d0: 5,6,7
{Node: addr(1), DiskId: 1, ShardIds: shardRange(8, 10)}, // S1 d1: 8,9
{Node: addr(2), DiskId: 0, ShardIds: shardRange(10, 12)}, // S2 d0: 10,11
{Node: addr(2), DiskId: 1, ShardIds: shardRange(12, 14)}, // S2 d1: 12,13
}
task := NewErasureCodingTask("ec-julor", addr(srcServer), volumeID, collection, dialOption)
params := &worker_pb.TaskParams{
VolumeId: volumeID,
Collection: collection,
Sources: []*worker_pb.TaskSource{
{Node: addr(srcServer), VolumeId: volumeID}, // real source
{Node: addr(stubServer), VolumeId: volumeID}, // 0-byte stub
{Node: addr(srcServer), VolumeId: volumeID, ShardIds: staleShards}, // stale EC shards to clear
},
Targets: targets,
TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
DataShards: erasure_coding.DataShardsCount,
ParityShards: erasure_coding.ParityShardsCount,
WorkingDir: t.TempDir(),
},
},
}
require.NoError(t, task.Execute(ctx, params))
base := fmt.Sprintf("%s_%d", collection, volumeID)
// Shards are distributed per-server; the disk within a server is auto-selected
// when DiskId is 0, and on a multi-disk server shards can spread to a sibling
// disk while the .ecx/.vif index lives on one of them (the cross-disk EC
// layout). So assert per server (aggregate across its disks): each server
// holds exactly its assigned shards, its .ecx/.vif survive on at least one
// disk (the info file is not stripped, even on the source's own disk), and no
// regular .dat/.idx remains. Stale shards 11-13 mounted on the source must be
// gone — they are not in S0's assignment.
serverShards := map[int][]uint32{
0: shardRange(0, 5), // S0 (source): 0..4
1: shardRange(5, 10), // S1 (stub): 5..9
2: shardRange(10, 14), // S2 (target): 10..13
}
allShards := map[uint32]bool{}
for server, want := range serverShards {
found := map[uint32]bool{}
hasEcx, hasVif := false, false
for d := 0; d < disksPerServer; d++ {
dir := cluster.VolumeDiskDir(server, d)
requireAbsent(t, dir, base+".dat")
requireAbsent(t, dir, base+".idx")
for id := 0; id < erasure_coding.TotalShardsCount; id++ {
if ecFileExists(dir, fmt.Sprintf("%s.ec%02d", base, id)) {
found[uint32(id)] = true
allShards[uint32(id)] = true
}
}
hasEcx = hasEcx || ecFileExists(dir, base+".ecx")
hasVif = hasVif || ecFileExists(dir, base+".vif")
}
gotIDs := make([]uint32, 0, len(found))
for id := range found {
gotIDs = append(gotIDs, id)
}
require.ElementsMatch(t, want, gotIDs, "server %d should hold exactly its assigned shards", server)
require.True(t, hasEcx, "server %d must keep its .ecx", server)
require.True(t, hasVif, "server %d must keep its .vif (not stripped by the original-volume delete)", server)
}
require.Len(t, allShards, erasure_coding.TotalShardsCount, "the full shard set must be present across the cluster")
}
func ecFileExists(dir, name string) bool {
_, err := os.Stat(filepath.Join(dir, name))
return err == nil
}

View File

@@ -0,0 +1,94 @@
package erasure_coding
import (
"context"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// cleanupOrphanSourceReplicas must not delete a source replica that has come
// back writable: it may hold writes the EC shards do not contain. A still
// readonly replica is the genuine orphan and is deleted.
func TestCleanupOrphanSkipsWritableSourceReplica(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(955)
framework.AllocateVolume(t, grpcClient, volumeID, "")
clusterInfo := &types.ClusterInfo{
ActiveTopology: buildSingleNodeTopology(t, clusterHarness.VolumeServerAddress(), volumeID),
GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
}
metric := &types.VolumeHealthMetrics{VolumeID: volumeID, Collection: ""}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Writable source replica: must be left intact.
deleted, err := cleanupOrphanSourceReplicas(ctx, clusterInfo, metric, 0)
require.NoError(t, err)
require.Equal(t, 0, deleted, "a writable source replica must not be deleted")
require.True(t, volumeExists(t, grpcClient, volumeID), "writable source volume must still exist")
// Mark readonly: now it is a genuine orphan and must be deleted.
_, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: volumeID})
require.NoError(t, err)
deleted, err = cleanupOrphanSourceReplicas(ctx, clusterInfo, metric, 0)
require.NoError(t, err)
require.Equal(t, 1, deleted, "a readonly orphan source replica must be deleted")
require.False(t, volumeExists(t, grpcClient, volumeID), "readonly orphan source volume must be deleted")
}
func volumeExists(t *testing.T, client volume_server_pb.VolumeServerClient, volumeID uint32) bool {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
return err == nil
}
// buildSingleNodeTopology builds an ActiveTopology with one node (at serverAddr)
// holding the regular volume, so cleanupOrphanSourceReplicas resolves the
// replica to the live test server and its gRPC calls reach it.
func buildSingleNodeTopology(t *testing.T, serverAddr string, volumeID uint32) *topology.ActiveTopology {
t.Helper()
at := topology.NewActiveTopology(10)
topologyInfo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{{
Id: "rack1",
DataNodeInfos: []*master_pb.DataNodeInfo{{
Id: serverAddr,
DiskInfos: map[string]*master_pb.DiskInfo{
"hdd": {
DiskId: 0,
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: volumeID, Collection: "", DiskId: 0},
},
},
},
}},
}},
}},
}
require.NoError(t, at.UpdateTopology(topologyInfo))
return at
}

View File

@@ -0,0 +1,231 @@
package erasure_coding
import (
"context"
"net/http"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// Reproduces the swtest interrupted-encode scenario at the task boundary: a
// previous encode of this volume was cut mid-distribute, leaving partial EC
// shards mounted on a destination. When a fresh encode task starts, its Step 0
// preflight (ensureCleanEcStart) must clear those shards before any
// destructive step, so distribute's ReceiveFile is not refused and no orphan
// shards survive to confuse the volume-server loader or make detection refuse
// the volume. The destination is named only as a target (no source row) to
// prove the preflight reaches the full write set, not just shard-bearing
// sources.
func TestEnsureCleanEcStartClearsStaleShardsBeforeEncode(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const (
volumeID = uint32(94782)
collection = "ec-preflight-invariant"
)
framework.AllocateVolume(t, grpcClient, volumeID, collection)
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 9478200, 0x9478BEEF)
upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
[]byte("payload-for-preflight-stale-ec-cleanup"))
_ = framework.ReadAllAndClose(t, upResp)
require.Equal(t, http.StatusCreated, upResp.StatusCode)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID, Collection: collection,
})
require.NoError(t, err)
// A half-finished previous distribute left a partial shard set mounted.
staleShards := []uint32{0, 1, 2}
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID, Collection: collection,
ShardIds: staleShards,
})
require.NoError(t, err)
shardPath := makeTinyEcShardFile(t)
// Precondition for the reproduction: the mounted partial EC blocks a fresh
// ReceiveFile via the mounted-volume guard.
err = sendShardViaReceiveFile(ctx, grpcClient, volumeID, collection, 0, shardPath)
require.Error(t, err, "expected ReceiveFile to be refused while stale EC volume is mounted")
task := NewErasureCodingTask(
"preflight-invariant",
clusterHarness.VolumeServerAddress(),
volumeID,
collection,
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
// Named only as a target — no source row for this node. The preflight must
// still clear its stale shards.
task.targets = []*worker_pb.TaskTarget{
{Node: clusterHarness.VolumeServerAddress(), VolumeId: volumeID, ShardIds: []uint32{0}},
}
require.NoError(t, task.ensureCleanEcStart(ctx),
"Step 0 preflight must clear stale EC shards at task start")
_, infoErr := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{VolumeId: volumeID})
require.Error(t, infoErr, "stale EC volume must be gone after the preflight")
require.NoError(t,
sendShardViaReceiveFile(ctx, grpcClient, volumeID, collection, 0, shardPath),
"ReceiveFile must succeed once the preflight has cleared the stale shards")
}
// A post-distribute failure that has no successor to clean up after it (a
// single-attempt job, or the last of a retry series) must leave nothing behind:
// rollbackDistribute tears down the shards this attempt wrote while leaving the
// source volume intact for a future re-encode. Reproduces the swtest finding
// that a terminally-failed encode stranded orphan shards.
func TestRollbackDistributeClearsShardsAndKeepsSource(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const (
volumeID = uint32(94783)
collection = "ec-rollback-distribute"
)
framework.AllocateVolume(t, grpcClient, volumeID, collection)
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 9478300, 0x9478D00D)
upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
[]byte("payload-for-rollback-distribute"))
_ = framework.ReadAllAndClose(t, upResp)
require.Equal(t, http.StatusCreated, upResp.StatusCode)
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
defer cancel()
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID, Collection: collection,
})
require.NoError(t, err)
// Shards this attempt "distributed" and mounted on the destination.
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID, Collection: collection,
ShardIds: []uint32{0, 1, 2, 3},
})
require.NoError(t, err)
task := NewErasureCodingTask(
"rollback-distribute",
clusterHarness.VolumeServerAddress(),
volumeID,
collection,
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
task.targets = []*worker_pb.TaskTarget{
{Node: clusterHarness.VolumeServerAddress(), VolumeId: volumeID, ShardIds: []uint32{0}},
}
// Simulate the failure path after distribute began.
task.rollbackDistribute(ctx)
_, infoErr := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{VolumeId: volumeID})
require.Error(t, infoErr, "rollback must tear down the shards this attempt distributed")
// The source normal volume must survive so a future re-encode can proceed.
_, statusErr := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
require.NoError(t, statusErr, "rollback must leave the source volume intact")
}
// A malformed plan (no targets) must be rejected by the preflight before the
// source is marked readonly or copied, so a bad plan cannot leave the source
// fenced readonly with nothing to show for it. Pure precondition check: no
// cluster and no RPC.
func TestEnsureCleanEcStartRejectsMissingTargets(t *testing.T) {
task := NewErasureCodingTask(
"preflight-no-targets",
"10.0.0.1:8080",
42,
"c",
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
// No targets set.
err := task.ensureCleanEcStart(context.Background())
require.Error(t, err, "preflight must reject an encode with no shard targets")
require.True(t, strings.Contains(err.Error(), "no EC shard targets"),
"error must name the missing-targets invariant, got: %v", err)
}
// A non-empty target slice whose entries are malformed (nil, empty Node, or no
// assigned shards) must also be rejected before the source is marked readonly —
// cleanupStaleEcShards silently skips such entries, so a length check alone
// would let the encode proceed with nothing to distribute to.
func TestEnsureCleanEcStartRejectsMalformedTarget(t *testing.T) {
cases := map[string][]*worker_pb.TaskTarget{
"nil target": {nil},
"empty node": {{Node: "", VolumeId: 42, ShardIds: []uint32{0}}},
"no shards": {{Node: "10.0.0.2:8080", VolumeId: 42}},
"one good one bad": {
{Node: "10.0.0.2:8080", VolumeId: 42, ShardIds: []uint32{0}},
{Node: "", VolumeId: 42, ShardIds: []uint32{1}},
},
}
for name, targets := range cases {
t.Run(name, func(t *testing.T) {
task := NewErasureCodingTask(
"preflight-malformed-target",
"10.0.0.1:8080",
42,
"c",
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
task.targets = targets
err := task.ensureCleanEcStart(context.Background())
require.Error(t, err, "preflight must reject a malformed target")
require.True(t, strings.Contains(err.Error(), "malformed EC shard target"),
"error must name the malformed-target invariant, got: %v", err)
})
}
}
// A plan with targets but no source replica and no assigned server must be
// rejected before any destructive step.
func TestEnsureCleanEcStartRejectsMissingSource(t *testing.T) {
task := NewErasureCodingTask(
"preflight-no-source",
"", // no assigned server
42,
"c",
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
task.targets = []*worker_pb.TaskTarget{
{Node: "10.0.0.2:8080", VolumeId: 42, ShardIds: []uint32{0}},
}
// No sources and no server.
err := task.ensureCleanEcStart(context.Background())
require.Error(t, err, "preflight must reject an encode with no source replica")
require.True(t, strings.Contains(err.Error(), "no source replica"),
"error must name the missing-source invariant, got: %v", err)
}

View File

@@ -0,0 +1,363 @@
package erasure_coding
import (
"context"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// Reproduces a stuck re-encode: partial EC shards mounted on a destination
// from a previous failed encode cause ReceiveFile to refuse with the
// mounted-volume guard. cleanupStaleEcShards must clear them so the next
// ReceiveFile lands.
func TestCleanupStaleEcShardsBeforeDistribute(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const (
volumeID = uint32(9478)
collection = "ec-9478-xserver"
)
framework.AllocateVolume(t, grpcClient, volumeID, collection)
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 947800, 0x9478CAFE)
upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
[]byte("payload-for-cross-server-stale-ec-cleanup"))
_ = framework.ReadAllAndClose(t, upResp)
require.Equal(t, http.StatusCreated, upResp.StatusCode)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID, Collection: collection,
})
require.NoError(t, err)
// Partial subset mimics a half-finished previous distribute: shards
// mounted on the destination with no .dat to anchor a same-store prune.
staleShards := []uint32{0, 1, 2}
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID, Collection: collection,
ShardIds: staleShards,
})
require.NoError(t, err)
shardPath := makeTinyEcShardFile(t)
// Pre-cleanup: the mounted partial EC blocks ReceiveFile.
err = sendShardViaReceiveFile(ctx, grpcClient, volumeID, collection, 0, shardPath)
require.Error(t, err, "expected ReceiveFile to be refused while EC volume is mounted")
require.True(t,
strings.Contains(err.Error(), "is mounted") ||
strings.Contains(err.Error(), "unmount before ReceiveFile"),
"expected refusal to name the mounted-volume guard, got: %v", err)
// ShardIds set marks this as an EC-shard cleanup source: cleanup will
// target it; getReplicas must skip it.
task := NewErasureCodingTask(
"stale-ec-xserver",
clusterHarness.VolumeServerAddress(),
volumeID,
collection,
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
task.dataShards = erasure_coding.DataShardsCount
task.parityShards = erasure_coding.ParityShardsCount
task.sources = []*worker_pb.TaskSource{
{
Node: clusterHarness.VolumeServerAddress(),
VolumeId: volumeID,
ShardIds: staleShards,
},
}
require.NoError(t, task.cleanupStaleEcShards(ctx))
_, infoErr := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{VolumeId: volumeID})
require.Error(t, infoErr, "EC volume should be gone after cleanupStaleEcShards")
require.NoError(t,
sendShardViaReceiveFile(ctx, grpcClient, volumeID, collection, 0, shardPath),
"ReceiveFile must succeed after cleanup")
require.Empty(t, task.getReplicas(),
"EC-shard sources must not appear in replica delete list")
}
// The destination has more shards mounted than t.sources lists — simulates
// detection-time topology missing shards (e.g., a prior attempt's mount
// hadn't heartbeated yet, or different shards live on a sibling disk). The
// cleanup must clear FindEcVolume regardless, by issuing unmount/delete
// over the full shard range, so the next ReceiveFile lands.
func TestCleanupStaleEcShardsClearsShardsBeyondSources(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const (
volumeID = uint32(94780)
collection = "ec-9478-beyond"
)
framework.AllocateVolume(t, grpcClient, volumeID, collection)
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 9478001, 0x9478FACE)
upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
[]byte("payload-for-shards-beyond-sources"))
_ = framework.ReadAllAndClose(t, upResp)
require.Equal(t, http.StatusCreated, upResp.StatusCode)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID, Collection: collection,
})
require.NoError(t, err)
// Five shards mounted on the destination.
mountedShards := []uint32{0, 1, 2, 3, 4}
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID, Collection: collection,
ShardIds: mountedShards,
})
require.NoError(t, err)
// Detection only "saw" two of them — the rest must still get cleared.
knownToDetection := []uint32{0, 1}
task := NewErasureCodingTask(
"stale-ec-beyond-sources",
clusterHarness.VolumeServerAddress(),
volumeID,
collection,
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
task.dataShards = erasure_coding.DataShardsCount
task.parityShards = erasure_coding.ParityShardsCount
task.sources = []*worker_pb.TaskSource{
{
Node: clusterHarness.VolumeServerAddress(),
VolumeId: volumeID,
ShardIds: knownToDetection,
},
}
require.NoError(t, task.cleanupStaleEcShards(ctx))
_, infoErr := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{VolumeId: volumeID})
require.Error(t, infoErr,
"all mounted shards must be cleared even though detection only listed a subset")
shardPath := makeTinyEcShardFile(t)
require.NoError(t,
sendShardViaReceiveFile(ctx, grpcClient, volumeID, collection, 4, shardPath),
"ReceiveFile for a shard outside detection's snapshot must succeed after cleanup")
}
// Cleanup also targets fresh destinations from t.targets even when no
// EC-shard source row exists for them. This catches concurrent-attempt
// fallout where a previous worker mounted shards on a node we're now
// writing to but the topology snapshot is stale.
func TestCleanupStaleEcShardsCoversTargetsWithoutSources(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const (
volumeID = uint32(94781)
collection = "ec-9478-targets-only"
)
framework.AllocateVolume(t, grpcClient, volumeID, collection)
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 9478101, 0x947811CE)
upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
[]byte("payload-for-targets-only-cleanup"))
_ = framework.ReadAllAndClose(t, upResp)
require.Equal(t, http.StatusCreated, upResp.StatusCode)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
VolumeId: volumeID, Collection: collection,
})
require.NoError(t, err)
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
VolumeId: volumeID, Collection: collection,
ShardIds: []uint32{0, 1},
})
require.NoError(t, err)
task := NewErasureCodingTask(
"stale-ec-targets-only",
clusterHarness.VolumeServerAddress(),
volumeID,
collection,
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
task.dataShards = erasure_coding.DataShardsCount
task.parityShards = erasure_coding.ParityShardsCount
// No sources. The destination is named only as a target — the cleanup
// must still reach it.
task.targets = []*worker_pb.TaskTarget{
{Node: clusterHarness.VolumeServerAddress(), VolumeId: volumeID, ShardIds: []uint32{0}},
}
require.NoError(t, task.cleanupStaleEcShards(ctx))
_, infoErr := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{VolumeId: volumeID})
require.Error(t, infoErr,
"target-only destinations must also be cleaned even without a source row")
}
// Cleanup is a no-op when sources carry only the regular .dat replica.
func TestCleanupStaleEcShardsSkipsRegularReplicas(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(9479)
framework.AllocateVolume(t, grpcClient, volumeID, "")
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
task := NewErasureCodingTask(
"no-stale-ec",
clusterHarness.VolumeServerAddress(),
volumeID,
"",
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
task.sources = []*worker_pb.TaskSource{
{Node: clusterHarness.VolumeServerAddress(), VolumeId: volumeID},
}
require.NoError(t, task.cleanupStaleEcShards(ctx))
_, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
require.NoError(t, err, "regular volume must remain untouched")
}
// makeTinyEcShardFile writes a placeholder payload — the mounted-volume
// guard fires before any content is consumed, so the bytes don't need to
// be a real shard.
func makeTinyEcShardFile(t *testing.T) string {
t.Helper()
p := filepath.Join(t.TempDir(), "shard.bin")
require.NoError(t, os.WriteFile(p, []byte("ec-shard-placeholder"), 0o600))
return p
}
// sendShardViaReceiveFile streams a shard file through the same ReceiveFile
// gRPC the EC worker uses, returning the server's reply error verbatim.
func sendShardViaReceiveFile(
ctx context.Context,
client volume_server_pb.VolumeServerClient,
volumeID uint32,
collection string,
shardID uint32,
filePath string,
) error {
f, err := os.Open(filePath)
if err != nil {
return err
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return err
}
stream, err := client.ReceiveFile(ctx)
if err != nil {
return err
}
if err := stream.Send(&volume_server_pb.ReceiveFileRequest{
Data: &volume_server_pb.ReceiveFileRequest_Info{
Info: &volume_server_pb.ReceiveFileInfo{
VolumeId: volumeID,
Ext: erasure_coding.ToExt(int(shardID)),
Collection: collection,
IsEcVolume: true,
ShardId: shardID,
FileSize: uint64(info.Size()),
},
},
}); err != nil {
return err
}
buf := make([]byte, 32*1024)
for {
n, readErr := f.Read(buf)
if n > 0 {
if err := stream.Send(&volume_server_pb.ReceiveFileRequest{
Data: &volume_server_pb.ReceiveFileRequest_FileContent{
FileContent: buf[:n],
},
}); err != nil {
return err
}
}
if readErr == io.EOF {
break
}
if readErr != nil {
return readErr
}
}
resp, err := stream.CloseAndRecv()
if err != nil {
return err
}
if resp.Error != "" {
return &receiveFileServerError{msg: resp.Error}
}
return nil
}
type receiveFileServerError struct{ msg string }
func (e *receiveFileServerError) Error() string { return e.msg }

View File

@@ -0,0 +1,249 @@
package erasure_coding
import (
"context"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
func TestCopyVolumeFilesToWorkerUsesCurrentCompactionRevision(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(951)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
liveFID := framework.NewFileID(volumeID, 1001, 0x1111AAAA)
liveUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), liveFID, []byte("live-payload-for-ec-copy"))
_ = framework.ReadAllAndClose(t, liveUploadResp)
require.Equal(t, http.StatusCreated, liveUploadResp.StatusCode)
deletedFID := framework.NewFileID(volumeID, 1002, 0x2222BBBB)
deletedUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), deletedFID, []byte("deleted-payload-for-vacuum"))
_ = framework.ReadAllAndClose(t, deletedUploadResp)
require.Equal(t, http.StatusCreated, deletedUploadResp.StatusCode)
deleteReq, err := http.NewRequest(http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+deletedFID, nil)
require.NoError(t, err)
deleteResp := framework.DoRequest(t, httpClient, deleteReq)
_ = framework.ReadAllAndClose(t, deleteResp)
require.Equal(t, http.StatusAccepted, deleteResp.StatusCode)
compactVolumeOnce(t, grpcClient, volumeID)
task := NewErasureCodingTask(
"copy-after-compaction",
clusterHarness.VolumeServerAddress(),
volumeID,
"",
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
require.NoError(t, task.markReplicasReadonly(ctx))
fileStatus, err := task.readSourceVolumeFileStatus(ctx)
require.NoError(t, err)
require.Greater(t, fileStatus.GetCompactionRevision(), uint32(0))
localFiles, err := task.copyVolumeFilesToWorker(ctx, t.TempDir())
require.NoError(t, err)
datInfo, err := os.Stat(localFiles["dat"])
require.NoError(t, err)
require.Equal(t, int64(fileStatus.GetDatFileSize()), datInfo.Size())
idxInfo, err := os.Stat(localFiles["idx"])
require.NoError(t, err)
require.Equal(t, int64(fileStatus.GetIdxFileSize()), idxInfo.Size())
}
// markReplicasReadonly must persist the readonly mark into the .vif so a
// source-server restart cannot silently reopen the volume to writes that the
// EC shards would not contain.
func TestMarkReplicasReadonlyPersists(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(954)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 3001, 0x4444DDDD)
resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("payload-for-readonly-persist"))
_ = framework.ReadAllAndClose(t, resp)
require.Equal(t, http.StatusCreated, resp.StatusCode)
task := NewErasureCodingTask(
"ec-readonly-persist",
clusterHarness.VolumeServerAddress(),
volumeID,
"",
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
require.NoError(t, task.markReplicasReadonly(ctx))
vifPath := filepath.Join(clusterHarness.BaseDir(), "volume", fmt.Sprintf("%d.vif", volumeID))
vi, _, found, err := volume_info.MaybeLoadVolumeInfo(vifPath)
require.NoError(t, err)
require.True(t, found, "volume must have a .vif after a persisted readonly mark")
require.True(t, vi.ReadOnly, "markReplicasReadonly must persist ReadOnly=true into the .vif")
}
// The worker-local encode path must stamp the EC ratio and an encode identity
// into the .vif, or the read guard is silently off for worker-encoded volumes.
func TestGenerateEcShardsLocallyStampsEncodeIdentity(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(952)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 2001, 0x3333CCCC)
resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("payload-for-ec-encode-identity"))
_ = framework.ReadAllAndClose(t, resp)
require.Equal(t, http.StatusCreated, resp.StatusCode)
task := NewErasureCodingTask(
"ec-encode-identity",
clusterHarness.VolumeServerAddress(),
volumeID,
"",
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
require.NoError(t, task.markReplicasReadonly(ctx))
localFiles, err := task.copyVolumeFilesToWorker(ctx, t.TempDir())
require.NoError(t, err)
shardFiles, err := task.generateEcShardsLocally(localFiles, t.TempDir())
require.NoError(t, err)
vifPath := shardFiles["vif"]
require.NotEmpty(t, vifPath, "generate must produce a .vif")
vi, _, found, err := volume_info.MaybeLoadVolumeInfo(vifPath)
require.NoError(t, err)
require.True(t, found)
require.NotNil(t, vi.EcShardConfig, "worker .vif must record the EC config")
require.Greater(t, vi.EcShardConfig.DataShards, uint32(0))
require.Greater(t, vi.EcShardConfig.ParityShards, uint32(0))
require.NotZero(t, vi.EcShardConfig.EncodeTsNs, "worker-encoded .vif must carry an encode identity")
}
// A fenced run must stamp the admin-issued generation (t.encodeTsNs) verbatim into
// the distributed .vif, so the generation the worker distributes equals the one it
// later sends on the teardown RPCs and the volume-server fence compares like-for-like.
func TestGenerateEcShardsLocallyUsesAdminGeneration(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(957)
framework.AllocateVolume(t, grpcClient, volumeID, "")
httpClient := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, 2002, 0x5555EEEE)
resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("payload-for-admin-generation"))
_ = framework.ReadAllAndClose(t, resp)
require.Equal(t, http.StatusCreated, resp.StatusCode)
task := NewErasureCodingTask(
"ec-admin-generation",
clusterHarness.VolumeServerAddress(),
volumeID,
"",
grpc.WithTransportCredentials(insecure.NewCredentials()),
)
const adminGeneration int64 = 1_700_000_000_000_000_321
task.encodeTsNs = adminGeneration
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
require.NoError(t, task.markReplicasReadonly(ctx))
localFiles, err := task.copyVolumeFilesToWorker(ctx, t.TempDir())
require.NoError(t, err)
shardFiles, err := task.generateEcShardsLocally(localFiles, t.TempDir())
require.NoError(t, err)
vi, _, found, err := volume_info.MaybeLoadVolumeInfo(shardFiles["vif"])
require.NoError(t, err)
require.True(t, found)
require.NotNil(t, vi.EcShardConfig)
require.Equal(t, adminGeneration, vi.EcShardConfig.EncodeTsNs, "fenced run must stamp the admin generation verbatim, not a local timestamp")
}
func compactVolumeOnce(t *testing.T, grpcClient volume_server_pb.VolumeServerClient, volumeID uint32) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
compactStream, err := grpcClient.VacuumVolumeCompact(ctx, &volume_server_pb.VacuumVolumeCompactRequest{
VolumeId: volumeID,
})
require.NoError(t, err)
for {
_, err = compactStream.Recv()
if err == io.EOF {
break
}
require.NoError(t, err)
}
_, err = grpcClient.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{
VolumeId: volumeID,
})
require.NoError(t, err)
_, err = grpcClient.VacuumVolumeCleanup(ctx, &volume_server_pb.VacuumVolumeCleanupRequest{
VolumeId: volumeID,
})
require.NoError(t, err)
}

View File

@@ -0,0 +1,43 @@
package erasure_coding
import (
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// isStubReplica reports whether a regular replica's .dat is too small to hold
// any data — at most a bare superblock. An interrupted encode or copy can
// leave a 0-byte .dat, or write the 8-byte superblock and then fail; either
// way the file is not an encode source and must not be counted toward
// size/fullness checks.
func isStubReplica(size uint64) bool {
return size <= uint64(super_block.SuperBlockSize)
}
// selectCanonicalMetric picks the metric that drives the encode checks and
// names the encode source server. It prefers the lowest-server credible
// replica — data-bearing and not already EC — so a 0-byte stub or a leftover
// EC shard set sharing the volume id cannot become canonical and strand the
// volume (a stub trips the min-size gate; an EC metric trips the IsECVolume
// guard, hiding the volume from both orphan-source cleanup and re-encode).
// When nothing is credible there is nothing to encode, so the lowest-server
// metric is returned unchanged and the downstream gates make the skip
// decision as before.
func selectCanonicalMetric(group []*types.VolumeHealthMetrics) *types.VolumeHealthMetrics {
var lowest, credible *types.VolumeHealthMetrics
for _, m := range group {
if lowest == nil || m.Server < lowest.Server {
lowest = m
}
if m.IsECVolume || isStubReplica(m.Size) {
continue
}
if credible == nil || m.Server < credible.Server {
credible = m
}
}
if credible != nil {
return credible
}
return lowest
}

View File

@@ -0,0 +1,140 @@
package erasure_coding
import (
"context"
"fmt"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestIsStubReplica(t *testing.T) {
assert.True(t, isStubReplica(0), "0-byte .dat is a stub")
assert.True(t, isStubReplica(uint64(super_block.SuperBlockSize)-1), "below a superblock is a stub")
assert.True(t, isStubReplica(uint64(super_block.SuperBlockSize)), "a bare superblock holds no data — a stub")
assert.False(t, isStubReplica(uint64(super_block.SuperBlockSize)+1), "data beyond the superblock is a real replica")
assert.False(t, isStubReplica(200*1024*1024), "a data-bearing replica is not a stub")
}
// A 0-byte stub left by an interrupted encode often sorts to a lower server id
// than the real replica. The old lowest-server canonical pick then reported
// Size=0, tripped the min-size gate, and the volume was stranded in
// skippedTooSmall forever. selectCanonicalMetric must skip the stub and return
// the data-bearing replica.
func TestSelectCanonicalMetricPrefersCredibleOverLowServerStub(t *testing.T) {
stub := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.1:8080", Size: 0}
real := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.4:8080", Size: 200 * 1024 * 1024}
got := selectCanonicalMetric([]*types.VolumeHealthMetrics{stub, real})
require.Same(t, real, got)
}
// An EC-side metric (the partial shards from a failed encode) can also sort
// below the regular replica. Picking it would short-circuit at the
// IsECVolume guard and skip the volume, hiding it from both the orphan-source
// cleanup and the re-encode path. The credible regular replica must win.
func TestSelectCanonicalMetricSkipsECMetrics(t *testing.T) {
ecMetric := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.1:8080", Size: 50 * 1024 * 1024, IsECVolume: true}
real := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.4:8080", Size: 200 * 1024 * 1024}
got := selectCanonicalMetric([]*types.VolumeHealthMetrics{ecMetric, real})
require.Same(t, real, got)
}
// When nothing is credible there is nothing to encode. Fall back to the
// lowest-server metric so the downstream gates (min-size / IsECVolume) make
// the skip decision exactly as before — selectCanonicalMetric must not invent
// a source.
func TestSelectCanonicalMetricAllStubsFallsBackToLowestServer(t *testing.T) {
stubHi := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.4:8080", Size: 0}
stubLo := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.1:8080", Size: 0}
got := selectCanonicalMetric([]*types.VolumeHealthMetrics{stubHi, stubLo})
require.Same(t, stubLo, got)
}
// Among several credible replicas the lowest server id still wins, preserving
// the deterministic canonical choice the task-dedup logic relies on.
func TestSelectCanonicalMetricTieBreaksByServerAmongCredible(t *testing.T) {
hi := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.7:8080", Size: 200 * 1024 * 1024}
lo := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.2:8080", Size: 200 * 1024 * 1024}
got := selectCanonicalMetric([]*types.VolumeHealthMetrics{hi, lo})
require.Same(t, lo, got)
}
func TestSelectCanonicalMetricEmpty(t *testing.T) {
require.Nil(t, selectCanonicalMetric(nil))
require.Nil(t, selectCanonicalMetric([]*types.VolumeHealthMetrics{}))
}
// End-to-end regression for the stranded-volume bug: a volume whose lowest
// server holds a 0-byte stub and whose real replica is on a higher server must
// still be proposed for EC encoding, not silently dropped as too-small.
func TestDetectionEncodesDespiteLowServerStub(t *testing.T) {
const volumeID uint32 = 13
activeTopology := buildStubReplicaTopology(t, volumeID)
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
lastModified := time.Now().Add(-2 * time.Hour)
metrics := []*types.VolumeHealthMetrics{
// Stub on the lowest server: an interrupted encode left a 0-byte .dat.
{VolumeID: volumeID, Server: "127.0.0.1:8080", Size: 0, FullnessRatio: 0, LastModified: lastModified, Age: time.Since(lastModified)},
// The real replica on a higher server.
{VolumeID: volumeID, Server: "127.0.0.1:8081", Size: 200 * 1024 * 1024, FullnessRatio: 0.96, LastModified: lastModified, Age: time.Since(lastModified)},
}
results, _, err := Detection(context.Background(), metrics, clusterInfo, NewDefaultConfig(), 0)
require.NoError(t, err)
require.Len(t, results, 1, "volume with a real replica must be proposed for EC despite a low-server stub")
assert.Equal(t, types.TaskTypeErasureCoding, results[0].TaskType)
assert.Equal(t, volumeID, results[0].VolumeID)
}
// buildStubReplicaTopology builds a cluster with TotalShardsCount single-disk
// nodes (enough targets to place every shard) where node 0 holds a 0-byte stub
// replica of volumeID and node 1 holds the real replica.
func buildStubReplicaTopology(t *testing.T, volumeID uint32) *topology.ActiveTopology {
t.Helper()
activeTopology := topology.NewActiveTopology(10)
nodes := make([]*master_pb.DataNodeInfo, 0, erasure_coding.TotalShardsCount)
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
diskInfo := &master_pb.DiskInfo{
DiskId: 0,
MaxVolumeCount: 200,
}
switch i {
case 0:
diskInfo.VolumeInfos = []*master_pb.VolumeInformationMessage{{
Id: volumeID, DiskId: 0, DiskType: "hdd", Size: 0,
}}
diskInfo.VolumeCount = 1
case 1:
diskInfo.VolumeInfos = []*master_pb.VolumeInformationMessage{{
Id: volumeID, DiskId: 0, DiskType: "hdd", Size: 200 * 1024 * 1024,
}}
diskInfo.VolumeCount = 1
}
nodes = append(nodes, &master_pb.DataNodeInfo{
Id: fmt.Sprintf("127.0.0.1:%d", 8080+i),
DiskInfos: map[string]*master_pb.DiskInfo{"hdd": diskInfo},
})
}
require.NoError(t, activeTopology.UpdateTopology(&master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{{
Id: "rack1",
DataNodeInfos: nodes,
}},
}},
}))
return activeTopology
}

View File

@@ -0,0 +1,878 @@
package erasure_coding
import (
"context"
"fmt"
"path/filepath"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
ecstorage "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
"google.golang.org/grpc"
"google.golang.org/protobuf/proto"
)
func init() {
pluginworker.RegisterHandler(pluginworker.HandlerFactory{
JobType: "erasure_coding",
Category: pluginworker.CategoryHeavy,
Aliases: []string{"erasure-coding", "erasure.coding", "ec"},
Build: func(opts pluginworker.HandlerBuildOptions) (pluginworker.JobHandler, error) {
return NewErasureCodingHandler(opts.GrpcDialOption, opts.WorkingDir), nil
},
})
}
type erasureCodingWorkerConfig struct {
TaskConfig *Config
}
// ErasureCodingHandler is the plugin job handler for erasure coding.
type ErasureCodingHandler struct {
grpcDialOption grpc.DialOption
workingDir string
}
func NewErasureCodingHandler(grpcDialOption grpc.DialOption, workingDir string) *ErasureCodingHandler {
return &ErasureCodingHandler{grpcDialOption: grpcDialOption, workingDir: strings.TrimSpace(workingDir)}
}
func (h *ErasureCodingHandler) Capability() *plugin_pb.JobTypeCapability {
return &plugin_pb.JobTypeCapability{
JobType: "erasure_coding",
CanDetect: true,
CanExecute: true,
MaxDetectionConcurrency: 1,
MaxExecutionConcurrency: 1,
DisplayName: "EC Encoding",
Description: "Converts full and quiet volumes into EC shards",
Weight: 80,
}
}
func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
return &plugin_pb.JobTypeDescriptor{
JobType: "erasure_coding",
DisplayName: "EC Encoding",
Description: "Detect and execute erasure coding for suitable volumes",
Icon: "fas fa-shield-alt",
DescriptorVersion: 1,
AdminConfigForm: &plugin_pb.ConfigForm{
FormId: "erasure-coding-admin",
Title: "Erasure Coding Admin Config",
Description: "Admin-side controls for erasure coding detection scope.",
Sections: []*plugin_pb.ConfigSection{
{
SectionId: "scope",
Title: "Scope",
Description: "Optional filters applied before erasure coding detection.",
Fields: []*plugin_pb.ConfigField{
{
Name: "collection_filter",
Label: "Collection Filter",
Description: "Only erasure code volumes in matching collections. Comma-separated list of names, wildcards, or regex patterns.",
Placeholder: "all collections",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
},
},
},
},
DefaultValues: map[string]*plugin_pb.ConfigValue{
"collection_filter": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
},
},
},
WorkerConfigForm: &plugin_pb.ConfigForm{
FormId: "erasure-coding-worker",
Title: "Erasure Coding Worker Config",
Description: "Worker-side detection thresholds.",
Sections: []*plugin_pb.ConfigSection{
{
SectionId: "thresholds",
Title: "Detection Thresholds",
Description: "Controls for when erasure coding jobs should be proposed.",
Fields: []*plugin_pb.ConfigField{
{
Name: "quiet_for_seconds",
Label: "Quiet Period (s)",
Description: "Volume must remain unmodified for at least this duration.",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
Required: true,
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
},
{
Name: "fullness_ratio",
Label: "Fullness Ratio",
Description: "Minimum volume fullness ratio to trigger erasure coding.",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_DOUBLE,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
Required: true,
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0}},
MaxValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 1}},
},
{
Name: "min_size_mb",
Label: "Minimum Volume Size (MB)",
Description: "Only volumes larger than this size are considered.",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
Required: true,
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}},
},
{
Name: "preferred_tags",
Label: "Preferred Tags",
Description: "Comma-separated disk tags to prioritize for EC shard placement, ordered by preference.",
Placeholder: "fast,ssd",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
},
{
Name: "replica_placement",
Label: "Replica Placement",
Description: "EC shard placement (e.g. 020): 2nd/3rd digits cap shards per rack/node (best-effort during encode, enforced by rebalancing); the data-center digit is ignored. Empty uses the master default.",
Placeholder: "020",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
},
},
},
},
DefaultValues: map[string]*plugin_pb.ConfigValue{
"quiet_for_seconds": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 3600},
},
"fullness_ratio": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.95},
},
"min_size_mb": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 30},
},
"preferred_tags": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
},
"replica_placement": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
},
},
},
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
Enabled: true,
DetectionIntervalMinutes: 17,
DetectionTimeoutSeconds: 300,
MaxJobsPerDetection: 500,
GlobalExecutionConcurrency: 16,
PerWorkerExecutionConcurrency: 4,
RetryLimit: 1,
RetryBackoffSeconds: 30,
JobTypeMaxRuntimeSeconds: 1800,
ExecutionTimeoutSeconds: 1800,
},
WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{
"quiet_for_seconds": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 3600},
},
"fullness_ratio": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.95},
},
"min_size_mb": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 30},
},
"preferred_tags": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
},
},
}
}
func (h *ErasureCodingHandler) Detect(
ctx context.Context,
request *plugin_pb.RunDetectionRequest,
sender pluginworker.DetectionSender,
) error {
if request == nil {
return fmt.Errorf("run detection request is nil")
}
if sender == nil {
return fmt.Errorf("detection sender is nil")
}
if request.JobType != "" && request.JobType != "erasure_coding" {
return fmt.Errorf("job type %q is not handled by erasure_coding worker", request.JobType)
}
workerConfig := deriveErasureCodingWorkerConfig(request.GetWorkerConfigValues())
collectionFilter := strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "collection_filter", ""))
if collectionFilter != "" {
workerConfig.TaskConfig.CollectionFilter = collectionFilter
}
masters := make([]string, 0)
if request.ClusterContext != nil {
masters = append(masters, request.ClusterContext.MasterGrpcAddresses...)
}
metrics, activeTopology, err := h.collectVolumeMetrics(ctx, masters, collectionFilter)
if err != nil {
return err
}
clusterInfo := &workertypes.ClusterInfo{
ActiveTopology: activeTopology,
GrpcDialOption: h.grpcDialOption,
DefaultReplicaPlacement: pluginworker.FetchDefaultReplicaPlacement(ctx, masters, h.grpcDialOption),
}
maxResults := int(request.MaxResults)
if maxResults < 0 {
maxResults = 0
}
results, hasMore, err := Detection(ctx, metrics, clusterInfo, workerConfig.TaskConfig, maxResults)
if err != nil {
return err
}
// Stamp the admin-issued encode generation onto every EC proposal. DetectionSequence
// is minted once per cycle on the single admin clock, so generations are globally
// ordered even though detection runs on a rotating worker; this lets a stale worker's
// shard cleanup fence against a newer run instead of wiping it.
for _, result := range results {
if result == nil || result.TypedParams == nil {
continue
}
if ecp := result.TypedParams.GetErasureCodingParams(); ecp != nil {
ecp.EncodeTsNs = request.DetectionSequence
}
}
if traceErr := emitErasureCodingDetectionDecisionTrace(sender, metrics, workerConfig.TaskConfig, results, maxResults, hasMore); traceErr != nil {
glog.Warningf("Plugin worker failed to emit erasure_coding detection trace: %v", traceErr)
}
proposals := make([]*plugin_pb.JobProposal, 0, len(results))
for _, result := range results {
proposal, proposalErr := buildErasureCodingProposal(result, h.workingDir)
if proposalErr != nil {
glog.Warningf("Plugin worker skip invalid erasure_coding proposal: %v", proposalErr)
continue
}
proposals = append(proposals, proposal)
}
if err := sender.SendProposals(&plugin_pb.DetectionProposals{
JobType: "erasure_coding",
Proposals: proposals,
HasMore: hasMore,
}); err != nil {
return err
}
return sender.SendComplete(&plugin_pb.DetectionComplete{
JobType: "erasure_coding",
Success: true,
TotalProposals: int32(len(proposals)),
})
}
func emitErasureCodingDetectionDecisionTrace(
sender pluginworker.DetectionSender,
metrics []*workertypes.VolumeHealthMetrics,
taskConfig *Config,
results []*workertypes.TaskDetectionResult,
maxResults int,
hasMore bool,
) error {
if sender == nil || taskConfig == nil {
return nil
}
quietThreshold := time.Duration(taskConfig.QuietForSeconds) * time.Second
minSizeBytes := uint64(taskConfig.MinSizeMB) * 1024 * 1024
allowedCollections, err := wildcard.CompileCollectionMatcher(taskConfig.CollectionFilter)
if err != nil {
return err
}
volumeGroups := make(map[uint32][]*workertypes.VolumeHealthMetrics)
for _, metric := range metrics {
if metric == nil {
continue
}
volumeGroups[metric.VolumeID] = append(volumeGroups[metric.VolumeID], metric)
}
skippedAlreadyEC := 0
skippedTooSmall := 0
skippedCollectionFilter := 0
skippedQuietTime := 0
skippedFullness := 0
for _, groupMetrics := range volumeGroups {
if len(groupMetrics) == 0 {
continue
}
metric := groupMetrics[0]
for _, candidate := range groupMetrics {
if candidate != nil && candidate.Server < metric.Server {
metric = candidate
}
}
if metric == nil {
continue
}
if metric.IsECVolume {
skippedAlreadyEC++
continue
}
if metric.Size < minSizeBytes {
skippedTooSmall++
continue
}
if !allowedCollections.Matches(metric.Collection) {
skippedCollectionFilter++
continue
}
if metric.Age < quietThreshold {
skippedQuietTime++
continue
}
if metric.FullnessRatio < taskConfig.FullnessRatio {
skippedFullness++
continue
}
}
totalVolumes := len(metrics)
summarySuffix := ""
if hasMore {
summarySuffix = fmt.Sprintf(" (max_results=%d reached; remaining volumes not evaluated)", maxResults)
}
summaryMessage := ""
if len(results) == 0 {
summaryMessage = fmt.Sprintf(
"EC detection: No tasks created for %d volumes%s (skipped: %d already EC, %d too small, %d filtered, %d not quiet, %d not full)",
totalVolumes,
summarySuffix,
skippedAlreadyEC,
skippedTooSmall,
skippedCollectionFilter,
skippedQuietTime,
skippedFullness,
)
} else {
summaryMessage = fmt.Sprintf(
"EC detection: Created %d task(s)%s from %d volumes (skipped: %d already EC, %d too small, %d filtered, %d not quiet, %d not full)",
len(results),
summarySuffix,
totalVolumes,
skippedAlreadyEC,
skippedTooSmall,
skippedCollectionFilter,
skippedQuietTime,
skippedFullness,
)
}
if err := sender.SendActivity(pluginworker.BuildDetectorActivity("decision_summary", summaryMessage, map[string]*plugin_pb.ConfigValue{
"total_volumes": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(totalVolumes)},
},
"selected_tasks": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(results))},
},
"max_results": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(maxResults)},
},
"has_more": {
Kind: &plugin_pb.ConfigValue_BoolValue{BoolValue: hasMore},
},
"skipped_already_ec": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedAlreadyEC)},
},
"skipped_too_small": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedTooSmall)},
},
"skipped_filtered": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedCollectionFilter)},
},
"skipped_not_quiet": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedQuietTime)},
},
"skipped_not_full": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedFullness)},
},
"quiet_for_seconds": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.QuietForSeconds)},
},
"min_size_mb": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.MinSizeMB)},
},
"fullness_threshold_percent": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: taskConfig.FullnessRatio * 100},
},
})); err != nil {
return err
}
detailsEmitted := 0
for _, metric := range metrics {
if metric == nil || metric.IsECVolume {
continue
}
sizeMB := float64(metric.Size) / (1024 * 1024)
message := fmt.Sprintf(
"ERASURE CODING: Volume %d: size=%.1fMB (need ≥%dMB), age=%s (need ≥%s), fullness=%.1f%% (need ≥%.1f%%)",
metric.VolumeID,
sizeMB,
taskConfig.MinSizeMB,
metric.Age.Truncate(time.Minute),
quietThreshold.Truncate(time.Minute),
metric.FullnessRatio*100,
taskConfig.FullnessRatio*100,
)
if err := sender.SendActivity(pluginworker.BuildDetectorActivity("decision_volume", message, map[string]*plugin_pb.ConfigValue{
"volume_id": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(metric.VolumeID)},
},
"size_mb": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: sizeMB},
},
"required_min_size_mb": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.MinSizeMB)},
},
"age_seconds": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(metric.Age.Seconds())},
},
"required_quiet_for_seconds": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.QuietForSeconds)},
},
"fullness_percent": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: metric.FullnessRatio * 100},
},
"required_fullness_percent": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: taskConfig.FullnessRatio * 100},
},
})); err != nil {
return err
}
detailsEmitted++
if detailsEmitted >= 3 {
break
}
}
return nil
}
func (h *ErasureCodingHandler) Execute(
ctx context.Context,
request *plugin_pb.ExecuteJobRequest,
sender pluginworker.ExecutionSender,
) error {
if request == nil || request.Job == nil {
return fmt.Errorf("execute request/job is nil")
}
if sender == nil {
return fmt.Errorf("execution sender is nil")
}
if request.Job.JobType != "" && request.Job.JobType != "erasure_coding" {
return fmt.Errorf("job type %q is not handled by erasure_coding worker", request.Job.JobType)
}
params, err := decodeErasureCodingTaskParams(request.Job)
if err != nil {
return err
}
applyErasureCodingExecutionDefaults(params, request.GetClusterContext(), h.workingDir)
if len(params.Sources) == 0 || strings.TrimSpace(params.Sources[0].Node) == "" {
return fmt.Errorf("erasure coding source node is required")
}
if len(params.Targets) == 0 {
return fmt.Errorf("erasure coding targets are required")
}
task := NewErasureCodingTask(
request.Job.JobId,
params.Sources[0].Node,
params.VolumeId,
params.Collection,
h.grpcDialOption,
)
execCtx, execCancel := context.WithCancel(ctx)
defer execCancel()
task.SetProgressCallback(func(progress float64, stage string) {
message := fmt.Sprintf("erasure coding progress %.0f%%", progress)
if strings.TrimSpace(stage) != "" {
message = stage
}
if err := sender.SendProgress(&plugin_pb.JobProgressUpdate{
JobId: request.Job.JobId,
JobType: request.Job.JobType,
State: plugin_pb.JobState_JOB_STATE_RUNNING,
ProgressPercent: progress,
Stage: stage,
Message: message,
Activities: []*plugin_pb.ActivityEvent{
pluginworker.BuildExecutorActivity(stage, message),
},
}); err != nil {
execCancel()
}
})
if err := sender.SendProgress(&plugin_pb.JobProgressUpdate{
JobId: request.Job.JobId,
JobType: request.Job.JobType,
State: plugin_pb.JobState_JOB_STATE_ASSIGNED,
ProgressPercent: 0,
Stage: "assigned",
Message: "erasure coding job accepted",
Activities: []*plugin_pb.ActivityEvent{
pluginworker.BuildExecutorActivity("assigned", "erasure coding job accepted"),
},
}); err != nil {
return err
}
if err := task.Execute(execCtx, params); err != nil {
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
JobId: request.Job.JobId,
JobType: request.Job.JobType,
State: plugin_pb.JobState_JOB_STATE_FAILED,
ProgressPercent: 100,
Stage: "failed",
Message: err.Error(),
Activities: []*plugin_pb.ActivityEvent{
pluginworker.BuildExecutorActivity("failed", err.Error()),
},
})
return err
}
sourceNode := params.Sources[0].Node
resultSummary := fmt.Sprintf("erasure coding completed for volume %d across %d targets", params.VolumeId, len(params.Targets))
return sender.SendCompleted(&plugin_pb.JobCompleted{
JobId: request.Job.JobId,
JobType: request.Job.JobType,
Success: true,
Result: &plugin_pb.JobResult{
Summary: resultSummary,
OutputValues: map[string]*plugin_pb.ConfigValue{
"volume_id": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(params.VolumeId)},
},
"source_server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceNode},
},
"target_count": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(params.Targets))},
},
},
},
Activities: []*plugin_pb.ActivityEvent{
pluginworker.BuildExecutorActivity("completed", resultSummary),
},
})
}
func (h *ErasureCodingHandler) collectVolumeMetrics(
ctx context.Context,
masterAddresses []string,
collectionFilter string,
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, error) {
metrics, activeTopology, _, err := pluginworker.CollectVolumeMetricsFromMasters(ctx, masterAddresses, collectionFilter, h.grpcDialOption)
return metrics, activeTopology, err
}
func deriveErasureCodingWorkerConfig(values map[string]*plugin_pb.ConfigValue) *erasureCodingWorkerConfig {
taskConfig := NewDefaultConfig()
quietForSeconds := pluginworker.ReadIntConfig(values, "quiet_for_seconds", taskConfig.QuietForSeconds)
if quietForSeconds < 0 {
quietForSeconds = 0
}
taskConfig.QuietForSeconds = quietForSeconds
fullnessRatio := pluginworker.ReadDoubleConfig(values, "fullness_ratio", taskConfig.FullnessRatio)
if fullnessRatio < 0 {
fullnessRatio = 0
}
if fullnessRatio > 1 {
fullnessRatio = 1
}
taskConfig.FullnessRatio = fullnessRatio
minSizeMB := pluginworker.ReadIntConfig(values, "min_size_mb", taskConfig.MinSizeMB)
if minSizeMB < 1 {
minSizeMB = 1
}
taskConfig.MinSizeMB = minSizeMB
taskConfig.PreferredTags = util.NormalizeTagList(pluginworker.ReadStringListConfig(values, "preferred_tags"))
taskConfig.ReplicaPlacement = strings.TrimSpace(pluginworker.ReadStringConfig(values, "replica_placement", taskConfig.ReplicaPlacement))
return &erasureCodingWorkerConfig{
TaskConfig: taskConfig,
}
}
func buildErasureCodingProposal(
result *workertypes.TaskDetectionResult,
baseWorkingDir string,
) (*plugin_pb.JobProposal, error) {
if result == nil {
return nil, fmt.Errorf("task detection result is nil")
}
if result.TypedParams == nil {
return nil, fmt.Errorf("missing typed params for volume %d", result.VolumeID)
}
params := proto.Clone(result.TypedParams).(*worker_pb.TaskParams)
applyErasureCodingExecutionDefaults(params, nil, baseWorkingDir)
paramsPayload, err := proto.Marshal(params)
if err != nil {
return nil, fmt.Errorf("marshal task params: %w", err)
}
proposalID := strings.TrimSpace(result.TaskID)
if proposalID == "" {
proposalID = fmt.Sprintf("erasure-coding-%d-%d", result.VolumeID, time.Now().UnixNano())
}
dedupeKey := fmt.Sprintf("erasure_coding:%d", result.VolumeID)
if result.Collection != "" {
dedupeKey += ":" + result.Collection
}
sourceNode := ""
if len(params.Sources) > 0 {
sourceNode = strings.TrimSpace(params.Sources[0].Node)
}
summary := fmt.Sprintf("Erasure code volume %d", result.VolumeID)
if sourceNode != "" {
summary = fmt.Sprintf("Erasure code volume %d from %s", result.VolumeID, sourceNode)
}
// EC encoding reads the full volume, computes shards, and writes 14
// shards out to target nodes. Budget 10 min/GB (roughly 2x a plain copy)
// so the scheduler grants a deadline scaled to volume size.
volumeSizeGB := int64(result.TypedParams.VolumeSize/1024/1024/1024) + 1
estimatedRuntimeSeconds := volumeSizeGB * 10 * 60
return &plugin_pb.JobProposal{
ProposalId: proposalID,
DedupeKey: dedupeKey,
JobType: "erasure_coding",
Priority: pluginworker.MapTaskPriority(result.Priority),
Summary: summary,
Detail: strings.TrimSpace(result.Reason),
Parameters: map[string]*plugin_pb.ConfigValue{
"task_params_pb": {
Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: paramsPayload},
},
"volume_id": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(result.VolumeID)},
},
"source_server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceNode},
},
"collection": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Collection},
},
"target_count": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(params.Targets))},
},
"estimated_runtime_seconds": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: estimatedRuntimeSeconds},
},
},
Labels: map[string]string{
"task_type": "erasure_coding",
"volume_id": fmt.Sprintf("%d", result.VolumeID),
"collection": result.Collection,
"source_node": sourceNode,
"target_count": fmt.Sprintf("%d", len(params.Targets)),
},
}, nil
}
func decodeErasureCodingTaskParams(job *plugin_pb.JobSpec) (*worker_pb.TaskParams, error) {
if job == nil {
return nil, fmt.Errorf("job spec is nil")
}
if payload := pluginworker.ReadBytesConfig(job.Parameters, "task_params_pb"); len(payload) > 0 {
params := &worker_pb.TaskParams{}
if err := proto.Unmarshal(payload, params); err != nil {
return nil, fmt.Errorf("unmarshal task_params_pb: %w", err)
}
if params.TaskId == "" {
params.TaskId = job.JobId
}
return params, nil
}
volumeID := pluginworker.ReadUint32Config(job.Parameters, "volume_id", 0)
sourceNode := strings.TrimSpace(pluginworker.ReadStringConfig(job.Parameters, "source_server", ""))
if sourceNode == "" {
sourceNode = strings.TrimSpace(pluginworker.ReadStringConfig(job.Parameters, "server", ""))
}
targetServers := pluginworker.ReadStringListConfig(job.Parameters, "target_servers")
if len(targetServers) == 0 {
targetServers = pluginworker.ReadStringListConfig(job.Parameters, "targets")
}
collection := pluginworker.ReadStringConfig(job.Parameters, "collection", "")
dataShards := pluginworker.ReadInt32Config(job.Parameters, "data_shards", int32(ecstorage.DataShardsCount))
if dataShards <= 0 {
dataShards = int32(ecstorage.DataShardsCount)
}
parityShards := pluginworker.ReadInt32Config(job.Parameters, "parity_shards", int32(ecstorage.ParityShardsCount))
if parityShards <= 0 {
parityShards = int32(ecstorage.ParityShardsCount)
}
sourceDiskType := strings.TrimSpace(pluginworker.ReadStringConfig(job.Parameters, "source_disk_type", ""))
totalShards := int(dataShards + parityShards)
if volumeID == 0 {
return nil, fmt.Errorf("missing volume_id in job parameters")
}
if sourceNode == "" {
return nil, fmt.Errorf("missing source_server in job parameters")
}
if len(targetServers) == 0 {
return nil, fmt.Errorf("missing target_servers in job parameters")
}
if len(targetServers) < totalShards {
return nil, fmt.Errorf("insufficient target_servers: got %d, need at least %d", len(targetServers), totalShards)
}
shardAssignments := assignECShardIDs(totalShards, len(targetServers))
targets := make([]*worker_pb.TaskTarget, 0, len(targetServers))
for i := 0; i < len(targetServers); i++ {
targetNode := strings.TrimSpace(targetServers[i])
if targetNode == "" {
continue
}
targets = append(targets, &worker_pb.TaskTarget{
Node: targetNode,
VolumeId: volumeID,
ShardIds: shardAssignments[i],
})
}
if len(targets) < totalShards {
return nil, fmt.Errorf("insufficient non-empty target_servers after normalization: got %d, need at least %d", len(targets), totalShards)
}
return &worker_pb.TaskParams{
TaskId: job.JobId,
VolumeId: volumeID,
Collection: collection,
Sources: []*worker_pb.TaskSource{
{
Node: sourceNode,
VolumeId: volumeID,
},
},
Targets: targets,
TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
DataShards: dataShards,
ParityShards: parityShards,
SourceDiskType: sourceDiskType,
},
},
}, nil
}
func applyErasureCodingExecutionDefaults(
params *worker_pb.TaskParams,
clusterContext *plugin_pb.ClusterContext,
baseWorkingDir string,
) {
if params == nil {
return
}
ecParams := params.GetErasureCodingParams()
if ecParams == nil {
ecParams = &worker_pb.ErasureCodingTaskParams{
DataShards: ecstorage.DataShardsCount,
ParityShards: ecstorage.ParityShardsCount,
}
params.TaskParams = &worker_pb.TaskParams_ErasureCodingParams{ErasureCodingParams: ecParams}
}
if ecParams.DataShards <= 0 {
ecParams.DataShards = ecstorage.DataShardsCount
}
if ecParams.ParityShards <= 0 {
ecParams.ParityShards = ecstorage.ParityShardsCount
}
ecParams.WorkingDir = defaultErasureCodingWorkingDir(baseWorkingDir)
ecParams.CleanupSource = true
if strings.TrimSpace(ecParams.MasterClient) == "" && clusterContext != nil && len(clusterContext.MasterGrpcAddresses) > 0 {
ecParams.MasterClient = clusterContext.MasterGrpcAddresses[0]
}
totalShards := int(ecParams.DataShards + ecParams.ParityShards)
if totalShards <= 0 {
totalShards = ecstorage.TotalShardsCount
}
needsShardAssignment := false
for _, target := range params.Targets {
if target == nil || len(target.ShardIds) == 0 {
needsShardAssignment = true
break
}
}
if needsShardAssignment && len(params.Targets) > 0 {
assignments := assignECShardIDs(totalShards, len(params.Targets))
for i := 0; i < len(params.Targets); i++ {
if params.Targets[i] == nil {
continue
}
if len(params.Targets[i].ShardIds) == 0 {
params.Targets[i].ShardIds = assignments[i]
}
}
}
}
func assignECShardIDs(totalShards int, targetCount int) [][]uint32 {
if targetCount <= 0 {
return nil
}
if totalShards <= 0 {
totalShards = ecstorage.TotalShardsCount
}
assignments := make([][]uint32, targetCount)
for i := 0; i < totalShards; i++ {
targetIndex := i % targetCount
assignments[targetIndex] = append(assignments[targetIndex], uint32(i))
}
return assignments
}
func defaultErasureCodingWorkingDir(baseWorkingDir string) string {
dir := strings.TrimSpace(baseWorkingDir)
if dir == "" {
return filepath.Join(".", "erasure_coding")
}
return filepath.Join(dir, "erasure_coding")
}

View File

@@ -0,0 +1,343 @@
package erasure_coding
import (
"context"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
ecstorage "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
"google.golang.org/protobuf/proto"
)
func TestDecodeErasureCodingTaskParamsFromPayload(t *testing.T) {
expected := &worker_pb.TaskParams{
TaskId: "task-ec-1",
VolumeId: 88,
Collection: "images",
Sources: []*worker_pb.TaskSource{
{
Node: "10.0.0.1:8080",
VolumeId: 88,
},
},
Targets: []*worker_pb.TaskTarget{
{
Node: "10.0.0.2:8080",
VolumeId: 88,
ShardIds: []uint32{0, 10},
},
},
TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
DataShards: ecstorage.DataShardsCount,
ParityShards: ecstorage.ParityShardsCount,
WorkingDir: "/tmp/ec-work",
CleanupSource: true,
},
},
}
payload, err := proto.Marshal(expected)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
job := &plugin_pb.JobSpec{
JobId: "job-from-admin",
Parameters: map[string]*plugin_pb.ConfigValue{
"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
},
}
actual, err := decodeErasureCodingTaskParams(job)
if err != nil {
t.Fatalf("decodeErasureCodingTaskParams() err = %v", err)
}
if !proto.Equal(expected, actual) {
t.Fatalf("decoded params mismatch\nexpected: %+v\nactual: %+v", expected, actual)
}
}
func TestDecodeErasureCodingTaskParamsFallback(t *testing.T) {
targetServers := make([]string, 0, ecstorage.TotalShardsCount)
for i := 0; i < ecstorage.TotalShardsCount; i++ {
targetServers = append(targetServers, "10.0.0."+string(rune('a'+i))+":8080")
}
job := &plugin_pb.JobSpec{
JobId: "job-ec-2",
Parameters: map[string]*plugin_pb.ConfigValue{
"volume_id": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7},
},
"source_server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "127.0.0.1:8080"},
},
"target_servers": {
Kind: &plugin_pb.ConfigValue_StringList{
StringList: &plugin_pb.StringList{Values: targetServers},
},
},
"collection": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "videos"},
},
},
}
params, err := decodeErasureCodingTaskParams(job)
if err != nil {
t.Fatalf("decodeErasureCodingTaskParams() err = %v", err)
}
if params.TaskId != "job-ec-2" || params.VolumeId != 7 || params.Collection != "videos" {
t.Fatalf("unexpected basic params: %+v", params)
}
if len(params.Sources) != 1 || params.Sources[0].Node != "127.0.0.1:8080" {
t.Fatalf("unexpected sources: %+v", params.Sources)
}
if len(params.Targets) != ecstorage.TotalShardsCount {
t.Fatalf("unexpected target count: %d", len(params.Targets))
}
if params.GetErasureCodingParams() == nil {
t.Fatalf("expected fallback erasure coding params")
}
}
func TestDeriveErasureCodingWorkerConfig(t *testing.T) {
values := map[string]*plugin_pb.ConfigValue{
"quiet_for_seconds": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 720},
},
"fullness_ratio": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.92},
},
"min_size_mb": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 128},
},
}
cfg := deriveErasureCodingWorkerConfig(values)
if cfg.TaskConfig.QuietForSeconds != 720 {
t.Fatalf("expected quiet_for_seconds 720, got %d", cfg.TaskConfig.QuietForSeconds)
}
if cfg.TaskConfig.FullnessRatio != 0.92 {
t.Fatalf("expected fullness_ratio 0.92, got %v", cfg.TaskConfig.FullnessRatio)
}
if cfg.TaskConfig.MinSizeMB != 128 {
t.Fatalf("expected min_size_mb 128, got %d", cfg.TaskConfig.MinSizeMB)
}
}
func TestBuildErasureCodingProposal(t *testing.T) {
params := &worker_pb.TaskParams{
TaskId: "ec-task-1",
VolumeId: 99,
Collection: "c1",
Sources: []*worker_pb.TaskSource{
{
Node: "source-a:8080",
VolumeId: 99,
},
},
Targets: []*worker_pb.TaskTarget{
{
Node: "target-a:8080",
VolumeId: 99,
ShardIds: []uint32{0, 10},
},
{
Node: "target-b:8080",
VolumeId: 99,
ShardIds: []uint32{1, 11},
},
},
TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
DataShards: ecstorage.DataShardsCount,
ParityShards: ecstorage.ParityShardsCount,
},
},
}
result := &workertypes.TaskDetectionResult{
TaskID: "ec-task-1",
TaskType: workertypes.TaskTypeErasureCoding,
VolumeID: 99,
Server: "source-a",
Collection: "c1",
Priority: workertypes.TaskPriorityLow,
Reason: "quiet and full",
TypedParams: params,
}
proposal, err := buildErasureCodingProposal(result, "")
if err != nil {
t.Fatalf("buildErasureCodingProposal() err = %v", err)
}
if proposal.JobType != "erasure_coding" {
t.Fatalf("unexpected job type %q", proposal.JobType)
}
if proposal.Parameters["task_params_pb"] == nil {
t.Fatalf("expected serialized task params")
}
if proposal.Labels["source_node"] != "source-a:8080" {
t.Fatalf("unexpected source label %q", proposal.Labels["source_node"])
}
}
func TestErasureCodingHandlerRejectsUnsupportedJobType(t *testing.T) {
handler := NewErasureCodingHandler(nil, "")
err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
JobType: "vacuum",
}, noopDetectionSender{})
if err == nil {
t.Fatalf("expected detect job type mismatch error")
}
err = handler.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
Job: &plugin_pb.JobSpec{JobId: "job-1", JobType: "vacuum"},
}, noopExecutionSender{})
if err == nil {
t.Fatalf("expected execute job type mismatch error")
}
}
func TestEmitErasureCodingDetectionDecisionTraceNoTasks(t *testing.T) {
sender := &recordingDetectionSender{}
config := NewDefaultConfig()
config.QuietForSeconds = 5 * 60
config.MinSizeMB = 30
config.FullnessRatio = 0.91
metrics := []*workertypes.VolumeHealthMetrics{
{
VolumeID: 20,
Size: 0,
Age: 218*time.Hour + 41*time.Minute,
FullnessRatio: 0,
},
{
VolumeID: 27,
Size: uint64(16 * 1024 * 1024 / 10),
Age: 91*time.Hour + time.Minute,
FullnessRatio: 0.002,
},
{
VolumeID: 12,
Size: 0,
Age: 219*time.Hour + 49*time.Minute,
FullnessRatio: 0,
},
}
if err := emitErasureCodingDetectionDecisionTrace(sender, metrics, config, nil, 0, false); err != nil {
t.Fatalf("emitErasureCodingDetectionDecisionTrace error: %v", err)
}
if len(sender.events) < 4 {
t.Fatalf("expected at least 4 detection events, got %d", len(sender.events))
}
if sender.events[0].Source != plugin_pb.ActivitySource_ACTIVITY_SOURCE_DETECTOR {
t.Fatalf("expected detector source, got %v", sender.events[0].Source)
}
if !strings.Contains(sender.events[0].Message, "EC detection: No tasks created for 3 volumes") {
t.Fatalf("unexpected summary message: %q", sender.events[0].Message)
}
if !strings.Contains(sender.events[1].Message, "ERASURE CODING: Volume 20: size=0.0MB") {
t.Fatalf("unexpected first detail message: %q", sender.events[1].Message)
}
}
func TestErasureCodingDescriptorOmitsLocalExecutionFields(t *testing.T) {
descriptor := NewErasureCodingHandler(nil, "").Descriptor()
if descriptor == nil || descriptor.WorkerConfigForm == nil {
t.Fatalf("expected worker config form in descriptor")
}
if workerConfigFormHasField(descriptor.WorkerConfigForm, "working_dir") {
t.Fatalf("unexpected working_dir in erasure coding worker config form")
}
if workerConfigFormHasField(descriptor.WorkerConfigForm, "cleanup_source") {
t.Fatalf("unexpected cleanup_source in erasure coding worker config form")
}
}
func TestApplyErasureCodingExecutionDefaultsForcesLocalFields(t *testing.T) {
baseWorkingDir := "/var/lib/seaweedfs-worker"
params := &worker_pb.TaskParams{
TaskId: "ec-test",
VolumeId: 100,
TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
DataShards: ecstorage.DataShardsCount,
ParityShards: ecstorage.ParityShardsCount,
WorkingDir: "/tmp/custom-from-job",
CleanupSource: false,
},
},
}
applyErasureCodingExecutionDefaults(params, nil, baseWorkingDir)
ecParams := params.GetErasureCodingParams()
if ecParams == nil {
t.Fatalf("expected erasure coding params")
}
if ecParams.WorkingDir != defaultErasureCodingWorkingDir(baseWorkingDir) {
t.Fatalf("expected local working_dir %q, got %q", defaultErasureCodingWorkingDir(baseWorkingDir), ecParams.WorkingDir)
}
if !ecParams.CleanupSource {
t.Fatalf("expected cleanup_source true")
}
}
type noopDetectionSender struct{}
func (noopDetectionSender) SendProposals(*plugin_pb.DetectionProposals) error { return nil }
func (noopDetectionSender) SendComplete(*plugin_pb.DetectionComplete) error { return nil }
func (noopDetectionSender) SendActivity(*plugin_pb.ActivityEvent) error { return nil }
type noopExecutionSender struct{}
func (noopExecutionSender) SendProgress(*plugin_pb.JobProgressUpdate) error { return nil }
func (noopExecutionSender) SendCompleted(*plugin_pb.JobCompleted) error { return nil }
type recordingDetectionSender struct {
proposals *plugin_pb.DetectionProposals
complete *plugin_pb.DetectionComplete
events []*plugin_pb.ActivityEvent
}
func (r *recordingDetectionSender) SendProposals(proposals *plugin_pb.DetectionProposals) error {
r.proposals = proposals
return nil
}
func (r *recordingDetectionSender) SendComplete(complete *plugin_pb.DetectionComplete) error {
r.complete = complete
return nil
}
func (r *recordingDetectionSender) SendActivity(event *plugin_pb.ActivityEvent) error {
if event != nil {
r.events = append(r.events, event)
}
return nil
}
func workerConfigFormHasField(form *plugin_pb.ConfigForm, fieldName string) bool {
if form == nil {
return false
}
for _, section := range form.Sections {
if section == nil {
continue
}
for _, field := range section.Fields {
if field != nil && field.Name == fieldName {
return true
}
}
}
return false
}

View File

@@ -0,0 +1,96 @@
package erasure_coding
import (
"context"
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// Global variable to hold the task definition for configuration updates
var globalTaskDef *base.TaskDefinition
// Auto-register this task when the package is imported
func init() {
RegisterErasureCodingTask()
// Register config updater
tasks.AutoRegisterConfigUpdater(types.TaskTypeErasureCoding, UpdateConfigFromPersistence)
}
// RegisterErasureCodingTask registers the erasure coding task with the new architecture
func RegisterErasureCodingTask() {
// Create configuration instance
config := NewDefaultConfig()
// Create shared gRPC dial option using TLS configuration
dialOpt := security.LoadClientTLS(util.GetViper(), "grpc.worker")
// Create complete task definition
taskDef := &base.TaskDefinition{
Type: types.TaskTypeErasureCoding,
Name: "erasure_coding",
DisplayName: "Erasure Coding",
Description: "Applies erasure coding to volumes for data protection",
Icon: "fas fa-shield-alt text-success",
Capabilities: []string{"erasure_coding", "data_protection"},
Config: config,
ConfigSpec: GetConfigSpec(),
CreateTask: func(params *worker_pb.TaskParams) (types.Task, error) {
if params == nil {
return nil, fmt.Errorf("task parameters are required")
}
if len(params.Sources) == 0 {
return nil, fmt.Errorf("at least one source is required for erasure coding task")
}
return NewErasureCodingTask(
fmt.Sprintf("erasure_coding-%d", params.VolumeId),
params.Sources[0].Node, // Use first source node
params.VolumeId,
params.Collection,
dialOpt,
), nil
},
DetectionFunc: func(metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo, config base.TaskConfig) ([]*types.TaskDetectionResult, error) {
results, _, err := Detection(context.Background(), metrics, clusterInfo, config, 0)
return results, err
},
ScanInterval: 1 * time.Hour,
SchedulingFunc: Scheduling,
MaxConcurrent: 1,
RepeatInterval: 24 * time.Hour,
}
// Store task definition globally for configuration updates
globalTaskDef = taskDef
// Register everything with a single function call!
base.RegisterTask(taskDef)
}
// UpdateConfigFromPersistence updates the erasure coding configuration from persistence
func UpdateConfigFromPersistence(configPersistence interface{}) error {
if globalTaskDef == nil {
return fmt.Errorf("erasure coding task not registered")
}
// Load configuration from persistence
newConfig := LoadConfigFromPersistence(configPersistence)
if newConfig == nil {
return fmt.Errorf("failed to load configuration from persistence")
}
// Update the task definition's config
globalTaskDef.Config = newConfig
glog.V(1).Infof("Updated erasure coding task configuration from persistence")
return nil
}

View File

@@ -0,0 +1,40 @@
package erasure_coding
import (
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// Scheduling implements the scheduling logic for erasure coding tasks
func Scheduling(task *types.TaskInput, runningTasks []*types.TaskInput, availableWorkers []*types.WorkerData, config base.TaskConfig) bool {
ecConfig := config.(*Config)
// Check if we have available workers
if len(availableWorkers) == 0 {
return false
}
// Count running EC tasks
runningCount := 0
for _, runningTask := range runningTasks {
if runningTask.Type == types.TaskTypeErasureCoding {
runningCount++
}
}
// Check concurrency limit
if runningCount >= ecConfig.MaxConcurrent {
return false
}
// Check if any worker can handle EC tasks
for _, worker := range availableWorkers {
for _, capability := range worker.Capabilities {
if capability == types.TaskTypeErasureCoding {
return true
}
}
}
return false
}