snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
255
weed/worker/tasks/erasure_coding/config.go
Normal file
255
weed/worker/tasks/erasure_coding/config.go
Normal file
@@ -0,0 +1,255 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/config"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
)
|
||||
|
||||
// Config extends BaseConfig with erasure coding specific settings
|
||||
type Config struct {
|
||||
base.BaseConfig
|
||||
QuietForSeconds int `json:"quiet_for_seconds"`
|
||||
FullnessRatio float64 `json:"fullness_ratio"`
|
||||
CollectionFilter string `json:"collection_filter"`
|
||||
MinSizeMB int `json:"min_size_mb"`
|
||||
PreferredTags []string `json:"preferred_tags"`
|
||||
ReplicaPlacement string `json:"replica_placement"` // e.g. "020"; empty falls back to the master default replication
|
||||
}
|
||||
|
||||
// NewDefaultConfig creates a new default erasure coding configuration
|
||||
func NewDefaultConfig() *Config {
|
||||
return &Config{
|
||||
BaseConfig: base.BaseConfig{
|
||||
Enabled: true,
|
||||
ScanIntervalSeconds: 60 * 60, // 1 hour
|
||||
MaxConcurrent: 1,
|
||||
},
|
||||
QuietForSeconds: 3600, // 1 hour, matching the shell ec.encode -quietFor default
|
||||
FullnessRatio: 0.95, // 95%, matching the shell ec.encode -fullPercent default
|
||||
CollectionFilter: "",
|
||||
MinSizeMB: 30, // 30MB (more reasonable than 100MB)
|
||||
PreferredTags: nil,
|
||||
}
|
||||
}
|
||||
|
||||
// GetConfigSpec returns the configuration schema for erasure coding tasks
|
||||
func GetConfigSpec() base.ConfigSpec {
|
||||
return base.ConfigSpec{
|
||||
Fields: []*config.Field{
|
||||
{
|
||||
Name: "enabled",
|
||||
JSONName: "enabled",
|
||||
Type: config.FieldTypeBool,
|
||||
DefaultValue: true,
|
||||
Required: false,
|
||||
DisplayName: "Enable Erasure Coding Tasks",
|
||||
Description: "Whether erasure coding tasks should be automatically created",
|
||||
HelpText: "Toggle this to enable or disable automatic erasure coding task generation",
|
||||
InputType: "checkbox",
|
||||
CSSClasses: "form-check-input",
|
||||
},
|
||||
{
|
||||
Name: "scan_interval_seconds",
|
||||
JSONName: "scan_interval_seconds",
|
||||
Type: config.FieldTypeInterval,
|
||||
DefaultValue: 60 * 60,
|
||||
MinValue: 10 * 60,
|
||||
MaxValue: 24 * 60 * 60,
|
||||
Required: true,
|
||||
DisplayName: "Scan Interval",
|
||||
Description: "How often to scan for volumes needing erasure coding",
|
||||
HelpText: "The system will check for volumes that need erasure coding at this interval",
|
||||
Placeholder: "1",
|
||||
Unit: config.UnitHours,
|
||||
InputType: "interval",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "max_concurrent",
|
||||
JSONName: "max_concurrent",
|
||||
Type: config.FieldTypeInt,
|
||||
DefaultValue: 1,
|
||||
MinValue: 1,
|
||||
MaxValue: 5,
|
||||
Required: true,
|
||||
DisplayName: "Max Concurrent Tasks",
|
||||
Description: "Maximum number of erasure coding tasks that can run simultaneously",
|
||||
HelpText: "Limits the number of erasure coding operations running at the same time",
|
||||
Placeholder: "1 (default)",
|
||||
Unit: config.UnitCount,
|
||||
InputType: "number",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "quiet_for_seconds",
|
||||
JSONName: "quiet_for_seconds",
|
||||
Type: config.FieldTypeInterval,
|
||||
DefaultValue: 3600,
|
||||
MinValue: 1,
|
||||
MaxValue: 3600,
|
||||
Required: true,
|
||||
DisplayName: "Quiet Period",
|
||||
Description: "Minimum time volume must be quiet before erasure coding",
|
||||
HelpText: "Volume must not be modified for this duration before erasure coding",
|
||||
Placeholder: "60",
|
||||
Unit: config.UnitMinutes,
|
||||
InputType: "interval",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "fullness_ratio",
|
||||
JSONName: "fullness_ratio",
|
||||
Type: config.FieldTypeFloat,
|
||||
DefaultValue: 0.95,
|
||||
MinValue: 0.0001,
|
||||
MaxValue: 1.0,
|
||||
Required: true,
|
||||
DisplayName: "Fullness Ratio",
|
||||
Description: "Minimum fullness ratio to trigger erasure coding",
|
||||
HelpText: "Only volumes with this fullness ratio or higher will be erasure coded",
|
||||
Placeholder: "0.80 (80%)",
|
||||
Unit: config.UnitNone,
|
||||
InputType: "number",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "collection_filter",
|
||||
JSONName: "collection_filter",
|
||||
Type: config.FieldTypeString,
|
||||
DefaultValue: "",
|
||||
Required: false,
|
||||
DisplayName: "Collection Filter",
|
||||
Description: "Only process volumes from specific collections (comma-separated names, wildcards, or regex patterns)",
|
||||
HelpText: "Leave empty to process all collections, or list the collections to process",
|
||||
Placeholder: "pictures,videos",
|
||||
InputType: "text",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "min_size_mb",
|
||||
JSONName: "min_size_mb",
|
||||
Type: config.FieldTypeInt,
|
||||
DefaultValue: 30,
|
||||
MinValue: 1,
|
||||
MaxValue: 1000,
|
||||
Required: true,
|
||||
DisplayName: "Minimum Size (MB)",
|
||||
Description: "Minimum volume size to consider for erasure coding",
|
||||
HelpText: "Only volumes larger than this size will be considered for erasure coding",
|
||||
Placeholder: "30",
|
||||
Unit: config.UnitNone,
|
||||
InputType: "number",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "preferred_tags",
|
||||
JSONName: "preferred_tags",
|
||||
Type: config.FieldTypeString,
|
||||
DefaultValue: "",
|
||||
Required: false,
|
||||
DisplayName: "Preferred Disk Tags",
|
||||
Description: "Comma-separated disk tags to prioritize for EC shard placement",
|
||||
HelpText: "EC shards will be placed on disks with these tags first, then fall back to other disks if needed",
|
||||
Placeholder: "fast,ssd",
|
||||
InputType: "text",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "replica_placement",
|
||||
JSONName: "replica_placement",
|
||||
Type: config.FieldTypeString,
|
||||
DefaultValue: "",
|
||||
Required: false,
|
||||
DisplayName: "Replica Placement",
|
||||
Description: "EC shard replica placement constraint (e.g. 020)",
|
||||
HelpText: "Leave empty to use the master default replication. When set, the 2nd/3rd digits cap EC shards per rack and per node (best-effort during encode: relaxed rather than failing if the cluster can't satisfy them, then enforced by rebalancing). The 1st (data-center) digit is ignored for EC placement",
|
||||
Placeholder: "020",
|
||||
InputType: "text",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ToTaskPolicy converts configuration to a TaskPolicy protobuf message
|
||||
func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
|
||||
// Defensive copy of PreferredTags to prevent external mutation
|
||||
preferredTagsCopy := append([]string(nil), c.PreferredTags...)
|
||||
return &worker_pb.TaskPolicy{
|
||||
Enabled: c.Enabled,
|
||||
MaxConcurrent: int32(c.MaxConcurrent),
|
||||
RepeatIntervalSeconds: int32(c.ScanIntervalSeconds),
|
||||
CheckIntervalSeconds: int32(c.ScanIntervalSeconds),
|
||||
TaskConfig: &worker_pb.TaskPolicy_ErasureCodingConfig{
|
||||
ErasureCodingConfig: &worker_pb.ErasureCodingTaskConfig{
|
||||
FullnessRatio: float64(c.FullnessRatio),
|
||||
QuietForSeconds: int32(c.QuietForSeconds),
|
||||
MinVolumeSizeMb: int32(c.MinSizeMB),
|
||||
CollectionFilter: c.CollectionFilter,
|
||||
PreferredTags: preferredTagsCopy,
|
||||
ReplicaPlacement: c.ReplicaPlacement,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// FromTaskPolicy loads configuration from a TaskPolicy protobuf message
|
||||
func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
|
||||
if policy == nil {
|
||||
return fmt.Errorf("policy is nil")
|
||||
}
|
||||
|
||||
// Set general TaskPolicy fields
|
||||
c.Enabled = policy.Enabled
|
||||
c.MaxConcurrent = int(policy.MaxConcurrent)
|
||||
c.ScanIntervalSeconds = int(policy.RepeatIntervalSeconds) // Direct seconds-to-seconds mapping
|
||||
|
||||
// Set erasure coding-specific fields from the task config
|
||||
if ecConfig := policy.GetErasureCodingConfig(); ecConfig != nil {
|
||||
c.FullnessRatio = float64(ecConfig.FullnessRatio)
|
||||
c.QuietForSeconds = int(ecConfig.QuietForSeconds)
|
||||
c.MinSizeMB = int(ecConfig.MinVolumeSizeMb)
|
||||
c.CollectionFilter = ecConfig.CollectionFilter
|
||||
c.PreferredTags = append([]string(nil), ecConfig.PreferredTags...)
|
||||
c.ReplicaPlacement = ecConfig.ReplicaPlacement
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadConfigFromPersistence loads configuration from the persistence layer if available
|
||||
func LoadConfigFromPersistence(configPersistence interface{}) *Config {
|
||||
config := NewDefaultConfig()
|
||||
|
||||
// Try to load from persistence if available
|
||||
if persistence, ok := configPersistence.(interface {
|
||||
LoadErasureCodingTaskPolicy() (*worker_pb.TaskPolicy, error)
|
||||
}); ok {
|
||||
policy, err := persistence.LoadErasureCodingTaskPolicy()
|
||||
switch {
|
||||
case err != nil:
|
||||
glog.Warningf("Could not read the persisted erasure coding configuration, falling back to defaults: %v", err)
|
||||
case policy == nil:
|
||||
glog.V(1).Infof("No erasure coding configuration persisted yet, using defaults")
|
||||
default:
|
||||
if err := config.FromTaskPolicy(policy); err != nil {
|
||||
glog.Warningf("Could not apply the persisted erasure coding configuration, falling back to defaults: %v", err)
|
||||
} else {
|
||||
glog.V(1).Infof("Loaded erasure coding configuration from persistence")
|
||||
return config
|
||||
}
|
||||
}
|
||||
} else if configPersistence != nil {
|
||||
// A store was handed in but does not expose the accessor, so the persisted
|
||||
// settings are silently ignored - always a wiring bug, never a normal state.
|
||||
glog.Warningf("%T cannot provide the persisted erasure coding configuration: it has no LoadErasureCodingTaskPolicy() method, "+
|
||||
"so the compiled-in defaults are used and any saved erasure coding settings are ignored", configPersistence)
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Using default erasure coding configuration")
|
||||
return config
|
||||
}
|
||||
772
weed/worker/tasks/erasure_coding/detection.go
Normal file
772
weed/worker/tasks/erasure_coding/detection.go
Normal file
@@ -0,0 +1,772 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
|
||||
"github.com/seaweedfs/seaweedfs/weed/ec"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
workerutil "github.com/seaweedfs/seaweedfs/weed/worker/tasks/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
const (
|
||||
minProposalsBeforeEarlyStop = 10
|
||||
maxConsecutivePlanningFailures = 10
|
||||
)
|
||||
|
||||
// Detection implements the detection logic for erasure coding tasks.
|
||||
// It respects ctx cancellation and can stop early once maxResults is reached.
|
||||
func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo, config base.TaskConfig, maxResults int) ([]*types.TaskDetectionResult, bool, error) {
|
||||
if !config.IsEnabled() {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
if maxResults < 0 {
|
||||
maxResults = 0
|
||||
}
|
||||
|
||||
ecConfig := config.(*Config)
|
||||
var results []*types.TaskDetectionResult
|
||||
hasMore := false
|
||||
stoppedEarly := false
|
||||
now := time.Now()
|
||||
quietThreshold := time.Duration(ecConfig.QuietForSeconds) * time.Second
|
||||
minSizeBytes := uint64(ecConfig.MinSizeMB) * 1024 * 1024 // Configurable minimum
|
||||
|
||||
debugCount := 0
|
||||
skippedAlreadyEC := 0
|
||||
skippedTooSmall := 0
|
||||
skippedCollectionFilter := 0
|
||||
skippedQuietTime := 0
|
||||
skippedFullness := 0
|
||||
skippedRemote := 0
|
||||
skippedTooFewNodes := 0
|
||||
consecutivePlanningFailures := 0
|
||||
|
||||
// EC shard replica placement: explicit config wins, else the master default.
|
||||
var replicaPlacement *super_block.ReplicaPlacement
|
||||
if clusterInfo != nil {
|
||||
replicaPlacement = super_block.ResolveReplicaPlacement(ecConfig.ReplicaPlacement, clusterInfo.DefaultReplicaPlacement)
|
||||
}
|
||||
// EC placement honors only the rack/node digits; the data-center digit can't
|
||||
// express a useful per-DC EC shard cap (it maxes at 2). Warn once per cycle so a
|
||||
// 1xx/2xx setting isn't silently ineffective.
|
||||
if replicaPlacement != nil && replicaPlacement.DiffDataCenterCount > 0 {
|
||||
glog.Warningf("EC Detection: replica placement data-center digit (%d) is ignored for EC; only rack/node digits are honored", replicaPlacement.DiffDataCenterCount)
|
||||
}
|
||||
|
||||
allowedCollections, err := wildcard.CompileCollectionMatcher(ecConfig.CollectionFilter)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
// Cluster node count for the min-node safety gate (mirrors the shell ec.encode
|
||||
// guard that refuses to encode when nodes < parity shards, so shards cannot be
|
||||
// spread for fault tolerance).
|
||||
clusterNodeCount := countTopologyNodes(clusterInfo.ActiveTopology)
|
||||
|
||||
// Group metrics by VolumeID to handle replicas and select canonical server
|
||||
volumeGroups := make(map[uint32][]*types.VolumeHealthMetrics)
|
||||
for _, metric := range metrics {
|
||||
if ctx != nil {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return results, hasMore, err
|
||||
}
|
||||
}
|
||||
volumeGroups[metric.VolumeID] = append(volumeGroups[metric.VolumeID], metric)
|
||||
}
|
||||
|
||||
groupKeys := make([]uint32, 0, len(volumeGroups))
|
||||
for volumeID := range volumeGroups {
|
||||
groupKeys = append(groupKeys, volumeID)
|
||||
}
|
||||
sort.Slice(groupKeys, func(i, j int) bool { return groupKeys[i] < groupKeys[j] })
|
||||
|
||||
// Build the EC placement snapshot once per detection cycle. planECDestinations
|
||||
// reserves the shards it assigns directly into it, so volumes planned later in
|
||||
// this cycle see the reduced capacity. Rebuilding it per volume re-walked the
|
||||
// whole topology (and ActiveTopology's growing pending-task set) every time,
|
||||
// which is O(volumes × topology) and times out on large clusters. The node
|
||||
// address map is precomputed for the same reason (ResolveServerAddress rebuilds
|
||||
// the full node map on every call).
|
||||
var ecSnapshot *ecbalancer.Topology
|
||||
var nodeAddresses map[string]string
|
||||
if clusterInfo != nil && clusterInfo.ActiveTopology != nil {
|
||||
ecSnapshot = ecbalancer.FromActiveTopology(clusterInfo.ActiveTopology, erasure_coding.DataShardsCount)
|
||||
nodeAddresses = buildNodeAddressMap(clusterInfo.ActiveTopology)
|
||||
}
|
||||
|
||||
// Iterate over groups to check criteria and creation tasks
|
||||
for idx, volumeID := range groupKeys {
|
||||
if ctx != nil {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return results, hasMore, err
|
||||
}
|
||||
}
|
||||
if maxResults > 0 && len(results) >= maxResults {
|
||||
if idx+1 < len(groupKeys) {
|
||||
hasMore = true
|
||||
}
|
||||
stoppedEarly = true
|
||||
break
|
||||
}
|
||||
|
||||
groupMetrics := volumeGroups[volumeID]
|
||||
|
||||
// Prefer the lowest-server credible replica so a 0-byte stub or a
|
||||
// leftover EC shard set on a lower server can't become canonical and
|
||||
// strand the volume in skippedTooSmall / skippedAlreadyEC.
|
||||
metric := selectCanonicalMetric(groupMetrics)
|
||||
|
||||
// Skip if already EC volume
|
||||
if metric.IsECVolume {
|
||||
skippedAlreadyEC++
|
||||
continue
|
||||
}
|
||||
|
||||
// Handle the "stuck source" state from #9448: a previous encode
|
||||
// succeeded but the post-encode source-delete left a regular replica
|
||||
// behind, so the master heartbeats BOTH the replica AND its EC shards.
|
||||
// metric.IsECVolume above is set only for the EC-side metric path, so
|
||||
// the canonical metric we picked is the regular replica with
|
||||
// IsECVolume=false. Re-proposing an encode in that state collides with
|
||||
// the mounted shards on the targets ("ec volume %d is mounted; refusing
|
||||
// overwrite") and the detector re-queues forever.
|
||||
//
|
||||
// We only act when the EC shard set is COMPLETE — fewer than
|
||||
// totalShards present means the existing recovery branch below
|
||||
// (around the `existingECShards` block) should keep its chance to
|
||||
// fold the partial shards into the new task. Counting walks
|
||||
// EcIndexBits to handle a single info entry carrying multiple shards.
|
||||
if clusterInfo.ActiveTopology != nil {
|
||||
shardCount := countExistingEcShardsForVolume(clusterInfo.ActiveTopology, metric.VolumeID, metric.Collection)
|
||||
totalShards := erasure_coding.DataShardsCount + erasure_coding.ParityShardsCount
|
||||
if shardCount >= totalShards {
|
||||
glog.Warningf("EC Detection: Volume %d has all %d EC shards in topology; "+
|
||||
"source replica on %s is orphaned (#9448).",
|
||||
metric.VolumeID, totalShards, metric.Server)
|
||||
if clusterInfo.GrpcDialOption != nil {
|
||||
deleted, cleanupErr := cleanupOrphanSourceReplicas(ctx, clusterInfo, metric, totalShards)
|
||||
switch {
|
||||
case cleanupErr != nil:
|
||||
// Don't fall through to a re-encode — that would just
|
||||
// collide with the mounted shards again. Surface the
|
||||
// failure and wait for the next cycle; the source is
|
||||
// still safe.
|
||||
glog.Warningf("EC Detection: failed to auto-clean orphaned source for volume %d: %v", metric.VolumeID, cleanupErr)
|
||||
case deleted > 0:
|
||||
glog.Infof("EC Detection: auto-cleaned %d orphaned source replica(s) for volume %d after verifying all %d EC shards present", deleted, metric.VolumeID, totalShards)
|
||||
default:
|
||||
glog.V(1).Infof("EC Detection: no orphaned regular replicas found in topology for volume %d (collection %q)", metric.VolumeID, metric.Collection)
|
||||
}
|
||||
} else {
|
||||
glog.Warningf("EC Detection: no gRPC dial option available to auto-clean orphaned source for volume %d; "+
|
||||
"to clean up by hand, send a targeted VolumeDelete RPC to %s only — DO NOT use the cluster-wide `volume.delete` shell command, which would also delete the EC shards.",
|
||||
metric.VolumeID, metric.Server)
|
||||
}
|
||||
skippedAlreadyEC++
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Check minimum size requirement
|
||||
if metric.Size < minSizeBytes {
|
||||
skippedTooSmall++
|
||||
continue
|
||||
}
|
||||
|
||||
// Check collection filter if specified
|
||||
if !allowedCollections.Matches(metric.Collection) {
|
||||
skippedCollectionFilter++
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip remote/tiered volumes: encoding them would lose the tiering. The
|
||||
// shell ec.encode excludes remote volumes for the same reason.
|
||||
if metric.HasRemoteCopy {
|
||||
skippedRemote++
|
||||
continue
|
||||
}
|
||||
|
||||
// Min-node safety gate: don't encode when the cluster has fewer nodes than
|
||||
// this collection's parity shards — shards could not be spread to tolerate
|
||||
// failures. Mirrors the shell ec.encode guard (node count < parity shards).
|
||||
if clusterNodeCount > 0 && clusterNodeCount < erasure_coding.ParityShardsCount {
|
||||
skippedTooFewNodes++
|
||||
continue
|
||||
}
|
||||
|
||||
// Check quiet duration and fullness criteria
|
||||
if metric.Age >= quietThreshold && metric.FullnessRatio >= ecConfig.FullnessRatio {
|
||||
if ctx != nil {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return results, hasMore, err
|
||||
}
|
||||
}
|
||||
glog.Infof("EC Detection: Volume %d meets all criteria, attempting to create task", metric.VolumeID)
|
||||
|
||||
// Generate task ID for ActiveTopology integration
|
||||
taskID := fmt.Sprintf("ec_vol_%d_%d", metric.VolumeID, now.Unix())
|
||||
|
||||
result := &types.TaskDetectionResult{
|
||||
TaskID: taskID, // Link to ActiveTopology pending task
|
||||
TaskType: types.TaskTypeErasureCoding,
|
||||
VolumeID: metric.VolumeID,
|
||||
Server: metric.Server,
|
||||
Collection: metric.Collection,
|
||||
Priority: types.TaskPriorityLow, // EC is not urgent
|
||||
Reason: fmt.Sprintf("Volume meets EC criteria: quiet for %.1fs (>%ds), fullness=%.1f%% (>%.1f%%), size=%.1fMB (>%dMB)",
|
||||
metric.Age.Seconds(), ecConfig.QuietForSeconds, metric.FullnessRatio*100, ecConfig.FullnessRatio*100,
|
||||
float64(metric.Size)/(1024*1024), ecConfig.MinSizeMB),
|
||||
ScheduleAt: now,
|
||||
}
|
||||
|
||||
// Plan EC destinations if ActiveTopology is available
|
||||
if clusterInfo.ActiveTopology != nil {
|
||||
// Check if ANY task already exists in ActiveTopology for this volume
|
||||
if clusterInfo.ActiveTopology.HasAnyTask(metric.VolumeID) {
|
||||
glog.V(2).Infof("EC Detection: Skipping volume %d, task already exists in ActiveTopology", metric.VolumeID)
|
||||
continue
|
||||
}
|
||||
|
||||
glog.Infof("EC Detection: ActiveTopology available, planning destinations for volume %d", metric.VolumeID)
|
||||
dataShards := erasure_coding.DataShardsCount
|
||||
parityShards := erasure_coding.ParityShardsCount
|
||||
multiPlan, shardsPerPlan, err := planECDestinations(ecSnapshot, nodeAddresses, metric, ecConfig, replicaPlacement, dataShards, parityShards)
|
||||
if err != nil {
|
||||
glog.V(2).Infof("Failed to plan EC destinations for volume %d: %v", metric.VolumeID, err)
|
||||
consecutivePlanningFailures++
|
||||
if len(results) >= minProposalsBeforeEarlyStop && consecutivePlanningFailures >= maxConsecutivePlanningFailures {
|
||||
glog.Warningf("EC Detection: stopping early after %d consecutive placement failures with %d proposals already planned", consecutivePlanningFailures, len(results))
|
||||
hasMore = true
|
||||
stoppedEarly = true
|
||||
break
|
||||
}
|
||||
continue // Skip this volume if destination planning fails
|
||||
}
|
||||
consecutivePlanningFailures = 0
|
||||
glog.Infof("EC Detection: Successfully planned %d destinations for volume %d", len(multiPlan.Plans), metric.VolumeID)
|
||||
|
||||
// Calculate expected shard size for EC operation
|
||||
// Each data shard will be approximately volumeSize / dataShards
|
||||
expectedShardSize := uint64(metric.Size) / uint64(dataShards)
|
||||
|
||||
// Add pending EC shard task to ActiveTopology for capacity management
|
||||
|
||||
// Extract shard destinations from multiPlan
|
||||
var shardDestinations []string
|
||||
var shardDiskIDs []uint32
|
||||
for _, plan := range multiPlan.Plans {
|
||||
shardDestinations = append(shardDestinations, plan.TargetNode)
|
||||
shardDiskIDs = append(shardDiskIDs, plan.TargetDisk)
|
||||
}
|
||||
|
||||
// Find all volume replica locations (server + disk) from topology
|
||||
glog.Infof("EC Detection: Looking for replica locations for volume %d", metric.VolumeID)
|
||||
replicaLocations := findVolumeReplicaLocations(clusterInfo.ActiveTopology, metric.VolumeID, metric.Collection)
|
||||
if len(replicaLocations) == 0 {
|
||||
glog.Warningf("No replica locations found for volume %d, skipping EC", metric.VolumeID)
|
||||
continue
|
||||
}
|
||||
glog.Infof("EC Detection: Found %d replica locations for volume %d", len(replicaLocations), metric.VolumeID)
|
||||
|
||||
// Find existing EC shards from previous failed attempts
|
||||
existingECShards := findExistingECShards(clusterInfo.ActiveTopology, metric.VolumeID, metric.Collection)
|
||||
|
||||
// Combine volume replicas and existing EC shards for cleanup
|
||||
var sources []topology.TaskSourceSpec
|
||||
|
||||
// Add volume replicas (will free volume slots)
|
||||
for _, replica := range replicaLocations {
|
||||
sources = append(sources, topology.TaskSourceSpec{
|
||||
ServerID: replica.ServerID,
|
||||
DiskID: replica.DiskID,
|
||||
DataCenter: replica.DataCenter,
|
||||
Rack: replica.Rack,
|
||||
CleanupType: topology.CleanupVolumeReplica,
|
||||
})
|
||||
}
|
||||
|
||||
// Add existing EC shards (will free shard slots)
|
||||
duplicateCheck := make(map[string]bool)
|
||||
for _, replica := range replicaLocations {
|
||||
key := fmt.Sprintf("%s:%d", replica.ServerID, replica.DiskID)
|
||||
duplicateCheck[key] = true
|
||||
}
|
||||
|
||||
for _, shard := range existingECShards {
|
||||
key := fmt.Sprintf("%s:%d", shard.ServerID, shard.DiskID)
|
||||
if !duplicateCheck[key] { // Avoid duplicates if EC shards are on same disk as volume replicas
|
||||
shardIds := append([]uint32(nil), shard.ShardIds...)
|
||||
// Free exactly the shards on this disk. Without an explicit
|
||||
// impact the cleanup falls back to CalculateECShardCleanupImpact,
|
||||
// which credits TotalShardsCount (14) regardless of ratio.
|
||||
cleanupImpact := topology.StorageSlotChange{ShardSlots: -int32(len(shardIds))}
|
||||
sources = append(sources, topology.TaskSourceSpec{
|
||||
ServerID: shard.ServerID,
|
||||
DiskID: shard.DiskID,
|
||||
DataCenter: shard.DataCenter,
|
||||
Rack: shard.Rack,
|
||||
CleanupType: topology.CleanupECShards,
|
||||
ShardIds: shardIds,
|
||||
StorageImpact: &cleanupImpact,
|
||||
})
|
||||
duplicateCheck[key] = true
|
||||
}
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Found %d volume replicas and %d existing EC shards for volume %d (total %d cleanup sources)",
|
||||
len(replicaLocations), len(existingECShards), metric.VolumeID, len(sources))
|
||||
|
||||
// Convert shard destinations to TaskDestinationSpec. A destination may
|
||||
// hold several shards (small clusters), so reserve capacity for the
|
||||
// actual per-disk shard count that Place assigned (shardsPerPlan),
|
||||
// which is exactly what createECTargets writes.
|
||||
destinations := make([]topology.TaskDestinationSpec, len(shardDestinations))
|
||||
for i, dest := range shardDestinations {
|
||||
shardCount := len(shardsPerPlan[i])
|
||||
shardImpact := topology.CalculateECShardStorageImpact(int32(shardCount), int64(expectedShardSize))
|
||||
destSize := int64(expectedShardSize) * int64(shardCount)
|
||||
destinations[i] = topology.TaskDestinationSpec{
|
||||
ServerID: dest,
|
||||
DiskID: shardDiskIDs[i],
|
||||
StorageImpact: &shardImpact,
|
||||
EstimatedSize: &destSize,
|
||||
}
|
||||
}
|
||||
|
||||
// Convert sources before mutating topology
|
||||
sourcesProto, err := convertTaskSourcesToProtobuf(sources, metric.VolumeID, clusterInfo.ActiveTopology)
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to convert sources for EC task on volume %d: %v, skipping", metric.VolumeID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
err = clusterInfo.ActiveTopology.AddPendingTask(topology.TaskSpec{
|
||||
TaskID: taskID,
|
||||
TaskType: topology.TaskTypeErasureCoding,
|
||||
VolumeID: metric.VolumeID,
|
||||
VolumeSize: int64(metric.Size),
|
||||
Sources: sources,
|
||||
Destinations: destinations,
|
||||
})
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to add pending EC shard task to ActiveTopology for volume %d: %v", metric.VolumeID, err)
|
||||
continue // Skip this volume if topology task addition fails
|
||||
}
|
||||
|
||||
// Cross-volume in-cycle capacity is tracked by ActiveTopology via the
|
||||
// pending task above, which the next volume's FromActiveTopology snapshot
|
||||
// reflects; no separate planner reservation is needed.
|
||||
|
||||
glog.V(2).Infof("Added pending EC shard task %s to ActiveTopology for volume %d with %d cleanup sources and %d shard destinations",
|
||||
taskID, metric.VolumeID, len(sources), len(multiPlan.Plans))
|
||||
|
||||
// Create unified sources and targets for EC task
|
||||
result.TypedParams = &worker_pb.TaskParams{
|
||||
TaskId: taskID, // Link to ActiveTopology pending task
|
||||
VolumeId: metric.VolumeID,
|
||||
Collection: metric.Collection,
|
||||
VolumeSize: metric.Size, // Store original volume size for tracking changes
|
||||
|
||||
// Unified sources - all sources that will be processed/cleaned up
|
||||
Sources: sourcesProto,
|
||||
|
||||
// Unified targets - all EC shard destinations
|
||||
Targets: createECTargets(multiPlan, shardsPerPlan),
|
||||
|
||||
TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
|
||||
ErasureCodingParams: createECTaskParams(dataShards, parityShards, metric.DiskType),
|
||||
},
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Planned EC destinations for volume %d: %d shards across %d racks, %d DCs",
|
||||
metric.VolumeID, len(multiPlan.Plans), multiPlan.SuccessfulRack, multiPlan.SuccessfulDCs)
|
||||
} else {
|
||||
glog.Warningf("No ActiveTopology available for destination planning in EC detection")
|
||||
continue // Skip this volume if no topology available
|
||||
}
|
||||
|
||||
glog.Infof("EC Detection: Successfully created EC task for volume %d, adding to results", metric.VolumeID)
|
||||
results = append(results, result)
|
||||
} else {
|
||||
// Count debug reasons
|
||||
if metric.Age < quietThreshold {
|
||||
skippedQuietTime++
|
||||
}
|
||||
if metric.FullnessRatio < ecConfig.FullnessRatio {
|
||||
skippedFullness++
|
||||
}
|
||||
|
||||
if debugCount < 5 { // Limit to avoid spam
|
||||
// Logic moved outside
|
||||
}
|
||||
debugCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Log debug summary if no tasks were created
|
||||
if len(results) == 0 && len(metrics) > 0 && !stoppedEarly {
|
||||
totalVolumes := len(metrics)
|
||||
glog.V(1).Infof("EC detection: No tasks created for %d volumes (skipped: %d already EC, %d too small, %d filtered, %d not quiet, %d not full, %d remote, %d too few nodes)",
|
||||
totalVolumes, skippedAlreadyEC, skippedTooSmall, skippedCollectionFilter, skippedQuietTime, skippedFullness, skippedRemote, skippedTooFewNodes)
|
||||
|
||||
// Show details for first few volumes
|
||||
for i, metric := range metrics {
|
||||
if i >= 3 || metric.IsECVolume { // Limit to first 3 non-EC volumes
|
||||
continue
|
||||
}
|
||||
sizeMB := float64(metric.Size) / (1024 * 1024)
|
||||
glog.V(1).Infof("ERASURE CODING: Volume %d: size=%.1fMB (need ≥%dMB), age=%s (need ≥%s), fullness=%.1f%% (need ≥%.1f%%)",
|
||||
metric.VolumeID, sizeMB, ecConfig.MinSizeMB, metric.Age.Truncate(time.Minute), quietThreshold.Truncate(time.Minute),
|
||||
metric.FullnessRatio*100, ecConfig.FullnessRatio*100)
|
||||
}
|
||||
}
|
||||
|
||||
return results, hasMore, nil
|
||||
}
|
||||
|
||||
// countTopologyNodes counts volume-server nodes in the active topology, used by
|
||||
// the min-node safety gate.
|
||||
func countTopologyNodes(at *topology.ActiveTopology) int {
|
||||
if at == nil {
|
||||
return 0
|
||||
}
|
||||
topo := at.GetTopologyInfo()
|
||||
if topo == nil {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, dc := range topo.DataCenterInfos {
|
||||
for _, rack := range dc.RackInfos {
|
||||
n += len(rack.DataNodeInfos)
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// buildNodeAddressMap resolves every node's server address once per detection
|
||||
// cycle. ResolveServerAddress rebuilds the full node map on each call, so
|
||||
// resolving per shard destination is O(destinations × topology); callers build
|
||||
// this map once and look up addresses from it.
|
||||
func buildNodeAddressMap(at *topology.ActiveTopology) map[string]string {
|
||||
if at == nil {
|
||||
return nil
|
||||
}
|
||||
allNodes := at.GetAllNodes()
|
||||
m := make(map[string]string, len(allNodes))
|
||||
for id, n := range allNodes {
|
||||
m[id] = string(pb.NewServerAddressFromDataNode(n))
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// planECDestinations places all shards of the volume via the shared ecbalancer
|
||||
// policy and returns the per-disk destination plans plus, parallel to them, the
|
||||
// shard ids ecbalancer.Place assigned to each disk (so createECTargets and the
|
||||
// capacity reservations use the real assignment, not a round-robin guess).
|
||||
//
|
||||
// snap is the cycle-wide placement snapshot, built once by the caller and reused
|
||||
// across volumes: Place reserves the shards it assigns into it, so later volumes
|
||||
// in the same detection cycle see the reduced capacity. Rebuilding it per volume
|
||||
// is O(volumes × topology) and times out on large clusters.
|
||||
//
|
||||
// Encode is lenient (PlaceDurabilityFirst): it relaxes caps/anti-affinity/RP as
|
||||
// needed rather than fail, and prefers the source disk type but spills if that
|
||||
// type can't hold every shard. rp is the resolved replica placement (may be nil).
|
||||
func planECDestinations(snap *ecbalancer.Topology, nodeAddresses map[string]string, metric *types.VolumeHealthMetrics, ecConfig *Config, rp *super_block.ReplicaPlacement, dataShards, parityShards int) (*topology.MultiDestinationPlan, [][]uint32, error) {
|
||||
if snap == nil {
|
||||
return nil, nil, fmt.Errorf("EC placement snapshot not available")
|
||||
}
|
||||
if dataShards <= 0 || parityShards <= 0 {
|
||||
return nil, nil, fmt.Errorf("invalid EC ratio: dataShards=%d parityShards=%d", dataShards, parityShards)
|
||||
}
|
||||
totalShards := dataShards + parityShards
|
||||
// Survive losing one disk: each disk holds at most parityShards shards,
|
||||
// so we need at least ceil(totalShards / parityShards) disks.
|
||||
minTotalDisks := (totalShards + parityShards - 1) / parityShards
|
||||
expectedShardSize := uint64(metric.Size) / uint64(dataShards)
|
||||
|
||||
// Encode is greenfield: any EC shards already present for this volume are stale
|
||||
// leftovers from a prior failed attempt, which the task deletes
|
||||
// (cleanupStaleEcShards) before distributing the new shards. Release them so they
|
||||
// don't occupy capacity or skew anti-affinity / per-disk caps during planning.
|
||||
snap.ReleaseVolumeShards(metric.Collection, metric.VolumeID)
|
||||
need := make([]int, totalShards)
|
||||
for i := range need {
|
||||
need[i] = i
|
||||
}
|
||||
res, err := snap.Place(metric.VolumeID, metric.Collection, need, ecbalancer.Constraints{
|
||||
DiskType: metric.DiskType,
|
||||
DiskTypePolicy: ecbalancer.DiskTypePrefer,
|
||||
PreferredTags: ecConfig.PreferredTags,
|
||||
ReplicaPlacement: rp,
|
||||
Ratio: func(string) (int, int) { return dataShards, parityShards },
|
||||
}, ecbalancer.PlaceDurabilityFirst)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if res.SpilledToOtherDiskType {
|
||||
glog.Warningf("EC volume %d: placed shards outside preferred disk type %q", metric.VolumeID, metric.DiskType)
|
||||
}
|
||||
if res.SpilledOutsidePreferredTags {
|
||||
glog.Warningf("EC volume %d: placed shards outside preferred tags %v", metric.VolumeID, ecConfig.PreferredTags)
|
||||
}
|
||||
if len(res.Relaxed) > 0 {
|
||||
// Encode is best-effort (PlaceDurabilityFirst): it relaxes these constraints
|
||||
// rather than defer when the cluster can't satisfy them. Surface it so a tight
|
||||
// replica placement isn't silently weakened; rebalancing tightens the spread.
|
||||
glog.Warningf("EC volume %d: placed with relaxed constraints %v; replica placement not fully satisfied (rebalancing will adjust)", metric.VolumeID, res.Relaxed)
|
||||
}
|
||||
|
||||
// Group the per-shard destinations into one plan per (node,disk), iterating
|
||||
// shard ids in order for determinism.
|
||||
type diskGroup struct {
|
||||
node, rack, dc string
|
||||
diskID uint32
|
||||
shards []uint32
|
||||
}
|
||||
type diskKey struct {
|
||||
node string
|
||||
diskID uint32
|
||||
}
|
||||
groups := make(map[diskKey]*diskGroup, totalShards)
|
||||
order := make([]diskKey, 0, totalShards)
|
||||
for sid := 0; sid < totalShards; sid++ {
|
||||
d, ok := res.Destinations[sid]
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("EC volume %d: shard %d was not placed", metric.VolumeID, sid)
|
||||
}
|
||||
key := diskKey{node: d.Node, diskID: d.DiskID}
|
||||
g := groups[key]
|
||||
if g == nil {
|
||||
g = &diskGroup{node: d.Node, rack: d.Rack, dc: d.DataCenter, diskID: d.DiskID}
|
||||
groups[key] = g
|
||||
order = append(order, key)
|
||||
}
|
||||
g.shards = append(g.shards, uint32(sid))
|
||||
}
|
||||
if len(order) < minTotalDisks {
|
||||
return nil, nil, fmt.Errorf("placed onto %d disks, but EC %d+%d needs at least %d so no disk holds more than %d shards",
|
||||
len(order), dataShards, parityShards, minTotalDisks, parityShards)
|
||||
}
|
||||
|
||||
var plans []*topology.DestinationPlan
|
||||
shardsPerPlan := make([][]uint32, 0, len(order))
|
||||
rackCount := make(map[string]int)
|
||||
dcCount := make(map[string]int)
|
||||
for _, key := range order {
|
||||
g := groups[key]
|
||||
targetAddress, ok := nodeAddresses[g.node]
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("failed to resolve address for target server %s", g.node)
|
||||
}
|
||||
plans = append(plans, &topology.DestinationPlan{
|
||||
TargetNode: g.node,
|
||||
TargetAddress: targetAddress,
|
||||
TargetDisk: g.diskID,
|
||||
TargetRack: g.rack,
|
||||
TargetDC: g.dc,
|
||||
ExpectedSize: expectedShardSize,
|
||||
})
|
||||
shardsPerPlan = append(shardsPerPlan, g.shards)
|
||||
rackCount[fmt.Sprintf("%s:%s", g.dc, g.rack)]++
|
||||
dcCount[g.dc]++
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Planned EC destinations for volume %d (size=%d bytes): expected shard size=%d bytes, %d shards across %d disks, %d racks, %d DCs",
|
||||
metric.VolumeID, metric.Size, expectedShardSize, totalShards, len(plans), len(rackCount), len(dcCount))
|
||||
|
||||
return &topology.MultiDestinationPlan{
|
||||
Plans: plans,
|
||||
TotalShards: totalShards,
|
||||
SuccessfulRack: len(rackCount),
|
||||
SuccessfulDCs: len(dcCount),
|
||||
}, shardsPerPlan, nil
|
||||
}
|
||||
|
||||
// createECTargets builds TaskTargets from the per-disk plans and the shard ids
|
||||
// ecbalancer.Place assigned to each (shardsPerPlan is parallel to multiPlan.Plans).
|
||||
func createECTargets(multiPlan *topology.MultiDestinationPlan, shardsPerPlan [][]uint32) []*worker_pb.TaskTarget {
|
||||
targets := make([]*worker_pb.TaskTarget, 0, len(multiPlan.Plans))
|
||||
for i, plan := range multiPlan.Plans {
|
||||
shardIDs := shardsPerPlan[i]
|
||||
targets = append(targets, &worker_pb.TaskTarget{
|
||||
Node: plan.TargetAddress,
|
||||
DiskId: plan.TargetDisk,
|
||||
Rack: plan.TargetRack,
|
||||
DataCenter: plan.TargetDC,
|
||||
ShardIds: shardIDs,
|
||||
EstimatedSize: plan.ExpectedSize,
|
||||
})
|
||||
glog.V(2).Infof("EC planning: target %s disk %d assigned shards %v", plan.TargetNode, plan.TargetDisk, shardIDs)
|
||||
}
|
||||
return targets
|
||||
}
|
||||
|
||||
// convertTaskSourcesToProtobuf converts topology.TaskSourceSpec to worker_pb.TaskSource
|
||||
func convertTaskSourcesToProtobuf(sources []topology.TaskSourceSpec, volumeID uint32, activeTopology *topology.ActiveTopology) ([]*worker_pb.TaskSource, error) {
|
||||
var protobufSources []*worker_pb.TaskSource
|
||||
|
||||
for _, source := range sources {
|
||||
serverAddress, err := workerutil.ResolveServerAddress(source.ServerID, activeTopology)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve address for source server %s: %v", source.ServerID, err)
|
||||
}
|
||||
|
||||
pbSource := &worker_pb.TaskSource{
|
||||
Node: serverAddress,
|
||||
DiskId: source.DiskID,
|
||||
DataCenter: source.DataCenter,
|
||||
Rack: source.Rack,
|
||||
}
|
||||
|
||||
// Convert storage impact to estimated size
|
||||
if source.EstimatedSize != nil {
|
||||
pbSource.EstimatedSize = uint64(*source.EstimatedSize)
|
||||
}
|
||||
|
||||
// Populated ShardIds is the wire-level marker that flags an
|
||||
// EC-shard cleanup source; the worker routes it through
|
||||
// cleanupStaleEcShards and skips it in getReplicas.
|
||||
switch source.CleanupType {
|
||||
case topology.CleanupVolumeReplica:
|
||||
pbSource.VolumeId = volumeID
|
||||
case topology.CleanupECShards:
|
||||
pbSource.VolumeId = volumeID
|
||||
pbSource.ShardIds = append([]uint32(nil), source.ShardIds...)
|
||||
}
|
||||
|
||||
protobufSources = append(protobufSources, pbSource)
|
||||
}
|
||||
|
||||
return protobufSources, nil
|
||||
}
|
||||
|
||||
// createECTaskParams creates clean EC task parameters (destinations now in unified targets).
|
||||
// sourceDiskType is forwarded to VolumeEcShardsMount so the resulting EC volume
|
||||
// reports under the source's disk type rather than the target location's (#9423).
|
||||
func createECTaskParams(dataShards, parityShards int, sourceDiskType string) *worker_pb.ErasureCodingTaskParams {
|
||||
return &worker_pb.ErasureCodingTaskParams{
|
||||
DataShards: int32(dataShards),
|
||||
ParityShards: int32(parityShards),
|
||||
SourceDiskType: sourceDiskType,
|
||||
}
|
||||
}
|
||||
|
||||
// findVolumeReplicaLocations finds all replica locations (server + disk) for the specified volume
|
||||
// Uses O(1) indexed lookup for optimal performance on large clusters.
|
||||
func findVolumeReplicaLocations(activeTopology *topology.ActiveTopology, volumeID uint32, collection string) []topology.VolumeReplica {
|
||||
if activeTopology == nil {
|
||||
return nil
|
||||
}
|
||||
return activeTopology.GetVolumeLocations(volumeID, collection)
|
||||
}
|
||||
|
||||
// findExistingECShards finds existing EC shards for a volume (from previous failed EC attempts)
|
||||
// Uses O(1) indexed lookup for optimal performance on large clusters.
|
||||
func findExistingECShards(activeTopology *topology.ActiveTopology, volumeID uint32, collection string) []topology.VolumeReplica {
|
||||
if activeTopology == nil {
|
||||
return nil
|
||||
}
|
||||
return activeTopology.GetECShardLocations(volumeID, collection)
|
||||
}
|
||||
|
||||
// cleanupOrphanSourceReplicas deletes any regular volume replicas still
|
||||
// present in the topology for (volumeID, collection) after re-verifying that
|
||||
// the full EC shard set is intact. Caller must hold expectedShards equal to
|
||||
// the configured totalShards count. Issues VolumeDelete RPC to each replica
|
||||
// server's address — that endpoint only touches the regular volume on the
|
||||
// targeted server, never EC shards (those live in a separate store path).
|
||||
// The cluster-wide `volume.delete` shell command is what would have nuked
|
||||
// the EC shards too; the targeted RPC used here is safe by construction.
|
||||
// Returns the count of replicas successfully deleted plus any error.
|
||||
func cleanupOrphanSourceReplicas(ctx context.Context, clusterInfo *types.ClusterInfo, metric *types.VolumeHealthMetrics, expectedShards int) (int, error) {
|
||||
if clusterInfo == nil || clusterInfo.ActiveTopology == nil {
|
||||
return 0, fmt.Errorf("active topology unavailable")
|
||||
}
|
||||
if clusterInfo.GrpcDialOption == nil {
|
||||
return 0, fmt.Errorf("grpc dial option unavailable")
|
||||
}
|
||||
|
||||
// Re-verify shard completeness right before acting. Defensive: detection
|
||||
// processes many volumes sequentially and the topology snapshot we built
|
||||
// at start-of-detection could have lost shards in between (a volume
|
||||
// server going down between iterations). Refusing to delete the source
|
||||
// when we can no longer prove the shards are complete is the safer
|
||||
// failure mode — the source replica is the only complete copy.
|
||||
actualShards := countExistingEcShardsForVolume(clusterInfo.ActiveTopology, metric.VolumeID, metric.Collection)
|
||||
if actualShards < expectedShards {
|
||||
return 0, fmt.Errorf("EC shard set shrank between detection and cleanup (%d < %d); refusing to delete source replica", actualShards, expectedShards)
|
||||
}
|
||||
|
||||
replicas := findVolumeReplicaLocations(clusterInfo.ActiveTopology, metric.VolumeID, metric.Collection)
|
||||
if len(replicas) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
deleted := 0
|
||||
var deleteErrors []string
|
||||
for _, replica := range replicas {
|
||||
serverAddress := replica.ServerID
|
||||
var isReadOnly bool
|
||||
err := operation.WithVolumeServerClient(false, pb.ServerAddress(serverAddress), clusterInfo.GrpcDialOption,
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
// Re-probe before deleting: only a still-readonly source replica is
|
||||
// safe to remove. One that came back writable may have accepted
|
||||
// writes the EC shards do not contain, so deleting it loses data.
|
||||
status, statusErr := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: metric.VolumeID})
|
||||
if statusErr != nil {
|
||||
return statusErr
|
||||
}
|
||||
isReadOnly = status.GetIsReadOnly()
|
||||
if !isReadOnly {
|
||||
return nil
|
||||
}
|
||||
_, deleteErr := client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
|
||||
VolumeId: metric.VolumeID,
|
||||
OnlyEmpty: false,
|
||||
})
|
||||
return deleteErr
|
||||
})
|
||||
if err != nil {
|
||||
deleteErrors = append(deleteErrors, fmt.Sprintf("server %s: %v", serverAddress, err))
|
||||
continue
|
||||
}
|
||||
if !isReadOnly {
|
||||
glog.Warningf("EC Detection: source replica for volume %d on %s is writable; not deleting (may hold writes the EC shards lack)", metric.VolumeID, serverAddress)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
glog.V(1).Infof("EC Detection: deleted orphan regular replica for volume %d on %s", metric.VolumeID, serverAddress)
|
||||
}
|
||||
|
||||
if len(deleteErrors) > 0 {
|
||||
return deleted, fmt.Errorf("%d of %d replica delete(s) failed: %s", len(deleteErrors), len(replicas), strings.Join(deleteErrors, "; "))
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// countExistingEcShardsForVolume counts (volumeID, collection) EC shards in the
|
||||
// topology, counting only the single largest encode generation; see
|
||||
// ec.CountExistingEcShardsForVolume for the generation semantics.
|
||||
func countExistingEcShardsForVolume(activeTopology *topology.ActiveTopology, volumeID uint32, collection string) int {
|
||||
if activeTopology == nil {
|
||||
return 0
|
||||
}
|
||||
return ec.CountExistingEcShardsForVolume(activeTopology.GetTopologyInfo(), volumeID, collection)
|
||||
}
|
||||
93
weed/worker/tasks/erasure_coding/detection_disk_type_test.go
Normal file
93
weed/worker/tasks/erasure_coding/detection_disk_type_test.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestPlanECDestinationsPrefersSourceDiskType_FullCluster covers the
|
||||
// #9423 happy path through the worker plumbing: every node carries both
|
||||
// an HDD and an SSD disk, so the cluster has enough SSD slots to fully
|
||||
// satisfy a 10+4 placement. With metric.DiskType="ssd", every selected
|
||||
// destination must land on an SSD disk.
|
||||
func TestPlanECDestinationsPrefersSourceDiskType_FullCluster(t *testing.T) {
|
||||
// 14 nodes × 2 disks per node (hdd + ssd) — enough SSD slots alone
|
||||
// for a 10+4 layout with one-shard-per-(server,disk) diversity.
|
||||
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd", "ssd"}, 100, 0, "")
|
||||
|
||||
metric := &types.VolumeHealthMetrics{
|
||||
VolumeID: 1,
|
||||
Server: "10.0.0.1:8080",
|
||||
Size: 100 * 1024 * 1024,
|
||||
Collection: "",
|
||||
DiskType: "ssd", // the property being plumbed end-to-end
|
||||
}
|
||||
|
||||
snap := ecbalancer.FromActiveTopology(activeTopology, erasure_coding.DataShardsCount)
|
||||
nodeAddresses := buildNodeAddressMap(activeTopology)
|
||||
plan, _, err := planECDestinations(snap, nodeAddresses, metric, NewDefaultConfig(), nil, erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, plan.Plans, erasure_coding.TotalShardsCount)
|
||||
|
||||
// buildActiveTopology assigns DiskID by index into the diskTypes
|
||||
// slice, so SSD is DiskID 1. Every plan entry must point at an SSD
|
||||
// disk — anything else means metric.DiskType was dropped on the way
|
||||
// to the placement layer.
|
||||
for _, p := range plan.Plans {
|
||||
assert.Equalf(t, uint32(1), p.TargetDisk,
|
||||
"target %s diskID = %d, want SSD disk (DiskID=1)", p.TargetNode, p.TargetDisk)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPlanECDestinationsSpillsToOtherDiskType_WhenPreferredScarce pins
|
||||
// the prefer-with-spillover policy: only one node in the cluster has an
|
||||
// SSD disk; the rest are HDD-only. A 10+4 placement must still succeed
|
||||
// (no hard fail), filling the one SSD slot first and the remaining 13
|
||||
// from HDD nodes. Without the spillover behavior this would either
|
||||
// error out or starve the placement.
|
||||
func TestPlanECDestinationsSpillsToOtherDiskType_WhenPreferredScarce(t *testing.T) {
|
||||
// Start with every node carrying both HDD and SSD, then strip SSD
|
||||
// from all but the first node so the SSD pool is too small alone.
|
||||
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd", "ssd"}, 100, 0, "")
|
||||
topo := activeTopology.GetTopologyInfo()
|
||||
for _, dc := range topo.DataCenterInfos {
|
||||
for _, rack := range dc.RackInfos {
|
||||
for i, node := range rack.DataNodeInfos {
|
||||
if i == 0 {
|
||||
continue // keep the first node's SSD
|
||||
}
|
||||
delete(node.DiskInfos, "ssd")
|
||||
}
|
||||
}
|
||||
}
|
||||
require.NoError(t, activeTopology.UpdateTopology(topo))
|
||||
|
||||
metric := &types.VolumeHealthMetrics{
|
||||
VolumeID: 2,
|
||||
Server: "10.0.0.1:8080",
|
||||
Size: 100 * 1024 * 1024,
|
||||
Collection: "",
|
||||
DiskType: "ssd",
|
||||
}
|
||||
|
||||
snap := ecbalancer.FromActiveTopology(activeTopology, erasure_coding.DataShardsCount)
|
||||
nodeAddresses := buildNodeAddressMap(activeTopology)
|
||||
plan, _, err := planECDestinations(snap, nodeAddresses, metric, NewDefaultConfig(), nil, erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, plan.Plans, erasure_coding.TotalShardsCount)
|
||||
|
||||
// Exactly one SSD target (the only SSD node), the rest on HDD.
|
||||
ssdHits := 0
|
||||
for _, p := range plan.Plans {
|
||||
if p.TargetDisk == 1 {
|
||||
ssdHits++
|
||||
}
|
||||
}
|
||||
assert.Equalf(t, 1, ssdHits,
|
||||
"expected exactly 1 SSD placement (the only SSD node in the cluster), got %d", ssdHits)
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type ecShardSet struct {
|
||||
bits uint32
|
||||
encodeTsNs int64
|
||||
}
|
||||
|
||||
func ecBits(shardIds ...int) uint32 {
|
||||
var b uint32
|
||||
for _, id := range shardIds {
|
||||
b |= uint32(1) << uint(id)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// buildEcTopology builds an ActiveTopology where each shard set lives on its own
|
||||
// node, so countExistingEcShardsForVolume can be exercised across encode
|
||||
// generations spread over different disks.
|
||||
func buildEcTopology(t *testing.T, vid uint32, collection string, sets []ecShardSet) *topology.ActiveTopology {
|
||||
t.Helper()
|
||||
at := topology.NewActiveTopology(10)
|
||||
nodes := make([]*master_pb.DataNodeInfo, 0, len(sets))
|
||||
for i, set := range sets {
|
||||
nodes = append(nodes, &master_pb.DataNodeInfo{
|
||||
Id: fmt.Sprintf("10.0.0.%d:8080", i+1),
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"hdd": {
|
||||
DiskId: 0,
|
||||
MaxVolumeCount: 100,
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
|
||||
{Id: vid, Collection: collection, EcIndexBits: set.bits, EncodeTsNs: set.encodeTsNs},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
topologyInfo := &master_pb.TopologyInfo{
|
||||
DataCenterInfos: []*master_pb.DataCenterInfo{{
|
||||
Id: "dc1",
|
||||
RackInfos: []*master_pb.RackInfo{{Id: "rack1", DataNodeInfos: nodes}},
|
||||
}},
|
||||
}
|
||||
require.NoError(t, at.UpdateTopology(topologyInfo))
|
||||
return at
|
||||
}
|
||||
|
||||
// Two interrupted encode runs whose shard sets overlap in count must NOT be
|
||||
// unioned into a false-complete set. countExistingEcShardsForVolume returns the
|
||||
// largest single generation's shard count, not the cross-run union.
|
||||
func TestCountExistingEcShards_DoesNotUnionAcrossGenerations(t *testing.T) {
|
||||
const vid = uint32(42)
|
||||
|
||||
// run A holds shards 0-6 (7 shards, encode_ts 1); run B holds shards 7-13
|
||||
// (7 shards, encode_ts 2). The union would be all 14; the per-generation max
|
||||
// is 7, so the orphan-source delete trigger (>= 14) must not fire.
|
||||
at := buildEcTopology(t, vid, "", []ecShardSet{
|
||||
{bits: ecBits(0, 1, 2, 3, 4, 5, 6), encodeTsNs: 1},
|
||||
{bits: ecBits(7, 8, 9, 10, 11, 12, 13), encodeTsNs: 2},
|
||||
})
|
||||
require.Equal(t, 7, countExistingEcShardsForVolume(at, vid, ""))
|
||||
}
|
||||
|
||||
// A single complete generation still counts as complete, so genuine stuck-source
|
||||
// cleanup keeps working.
|
||||
func TestCountExistingEcShards_SingleCompleteGeneration(t *testing.T) {
|
||||
const vid = uint32(43)
|
||||
at := buildEcTopology(t, vid, "", []ecShardSet{
|
||||
{bits: ecBits(0, 1, 2, 3, 4, 5, 6), encodeTsNs: 5},
|
||||
{bits: ecBits(7, 8, 9, 10, 11, 12, 13), encodeTsNs: 5},
|
||||
})
|
||||
require.Equal(t, 14, countExistingEcShardsForVolume(at, vid, ""))
|
||||
}
|
||||
|
||||
// Shards from pre-upgrade servers report encode_ts_ns==0; a complete 0-generation
|
||||
// set is still recognized as complete (its own bucket).
|
||||
func TestCountExistingEcShards_PreUpgradeZeroGeneration(t *testing.T) {
|
||||
const vid = uint32(44)
|
||||
at := buildEcTopology(t, vid, "", []ecShardSet{
|
||||
{bits: ecBits(0, 1, 2, 3, 4, 5, 6), encodeTsNs: 0},
|
||||
{bits: ecBits(7, 8, 9, 10, 11, 12, 13), encodeTsNs: 0},
|
||||
})
|
||||
require.Equal(t, 14, countExistingEcShardsForVolume(at, vid, ""))
|
||||
}
|
||||
499
weed/worker/tasks/erasure_coding/detection_test.go
Normal file
499
weed/worker/tasks/erasure_coding/detection_test.go
Normal file
@@ -0,0 +1,499 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPlanECDestinationsUsesPlanner(t *testing.T) {
|
||||
activeTopology := buildActiveTopology(t, 7, []string{"hdd", "ssd"}, 100, 0, "")
|
||||
|
||||
metric := &types.VolumeHealthMetrics{
|
||||
VolumeID: 1,
|
||||
Server: "10.0.0.1:8080",
|
||||
Size: 100 * 1024 * 1024,
|
||||
Collection: "",
|
||||
}
|
||||
|
||||
snap := ecbalancer.FromActiveTopology(activeTopology, erasure_coding.DataShardsCount)
|
||||
nodeAddresses := buildNodeAddressMap(activeTopology)
|
||||
plan, shardsPerPlan, err := planECDestinations(snap, nodeAddresses, metric, NewDefaultConfig(), nil, erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, plan)
|
||||
requireAllShardsPlaced(t, plan, shardsPerPlan)
|
||||
}
|
||||
|
||||
// TestDetectionSkipsWhenECShardsAlreadyExist guards against issue #9448: a
|
||||
// regular replica that survived a previous successful EC encode (source
|
||||
// delete didn't clean it up for some reason) gets re-proposed for encoding,
|
||||
// the new encode collides with the already-mounted shards on the targets
|
||||
// ("ec volume %d is mounted; refusing overwrite"), and detection loops
|
||||
// forever on the same volume. Detection must see the existing shards and
|
||||
// skip the volume so an admin can clean it up out-of-band.
|
||||
//
|
||||
// The guard fires ONLY when the EC shard set is complete (count >=
|
||||
// totalShards), so a partially-distributed previous attempt still falls
|
||||
// through to the existing recovery branch in the encode path.
|
||||
func TestDetectionSkipsWhenECShardsAlreadyExist(t *testing.T) {
|
||||
const volumeID uint32 = 42
|
||||
activeTopology := buildStuckSourceTopology(t, volumeID, erasure_coding.TotalShardsCount)
|
||||
|
||||
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
|
||||
metrics := buildStuckSourceMetrics(volumeID, "127.0.0.1:8080")
|
||||
|
||||
results, hasMore, err := Detection(context.Background(), metrics, clusterInfo, NewDefaultConfig(), 0)
|
||||
require.NoError(t, err)
|
||||
require.False(t, hasMore)
|
||||
require.Empty(t, results, "stuck source replica with all EC shards present must not yield a new encoding proposal")
|
||||
}
|
||||
|
||||
// TestDetectionAllowsRegularReplicaWhenShardsPartial covers the partial-EC
|
||||
// branch of the #9448 guard: when fewer than totalShards exist, the volume
|
||||
// is allowed to flow through to the normal encoding path so the existing
|
||||
// recovery branch (the `existingECShards` block in the encode arm) can fold
|
||||
// the partial shards into the new task. A bug here would either (a) skip
|
||||
// the volume entirely or (b) emit a proposal that later collides on the
|
||||
// mounted shards.
|
||||
func TestDetectionAllowsRegularReplicaWhenShardsPartial(t *testing.T) {
|
||||
const volumeID uint32 = 43
|
||||
activeTopology := buildStuckSourceTopology(t, volumeID, erasure_coding.DataShardsCount-1)
|
||||
|
||||
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
|
||||
metrics := buildStuckSourceMetrics(volumeID, "127.0.0.1:8080")
|
||||
|
||||
results, _, err := Detection(context.Background(), metrics, clusterInfo, NewDefaultConfig(), 0)
|
||||
require.NoError(t, err)
|
||||
// Partial shards are not a "stuck source" — the encode arm must keep
|
||||
// its chance to either propose a fresh task that folds the partial
|
||||
// shards into cleanup, or fail planning on the constrained topology.
|
||||
// We don't require len(results) > 0 because the constrained topology
|
||||
// (one disk per node, the orphaned shards already taking slots) can
|
||||
// legitimately fail destination planning. The assertion that matters
|
||||
// is: the #9448 guard did NOT silently swallow the volume into a
|
||||
// skippedAlreadyEC counter, and any emitted result is still an EC
|
||||
// task and not a no-op.
|
||||
for _, r := range results {
|
||||
require.Equal(t, types.TaskTypeErasureCoding, r.TaskType, "any emitted result should still be an EC task, not a no-op")
|
||||
}
|
||||
}
|
||||
|
||||
// buildStuckSourceTopology constructs a topology that mimics the #9448 stuck
|
||||
// state: a regular volume replica on node 0 plus `presentShardCount` EC
|
||||
// shards distributed across nodes 0..presentShardCount-1.
|
||||
func buildStuckSourceTopology(t *testing.T, volumeID uint32, presentShardCount int) *topology.ActiveTopology {
|
||||
t.Helper()
|
||||
require.LessOrEqual(t, presentShardCount, erasure_coding.TotalShardsCount)
|
||||
activeTopology := topology.NewActiveTopology(10)
|
||||
nodes := make([]*master_pb.DataNodeInfo, 0, erasure_coding.TotalShardsCount)
|
||||
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
|
||||
nodeID := fmt.Sprintf("127.0.0.1:%d", 8080+i)
|
||||
diskInfo := &master_pb.DiskInfo{
|
||||
DiskId: 0,
|
||||
VolumeCount: 1,
|
||||
MaxVolumeCount: 100,
|
||||
}
|
||||
if i < presentShardCount {
|
||||
diskInfo.EcShardInfos = []*master_pb.VolumeEcShardInformationMessage{{
|
||||
Id: volumeID,
|
||||
Collection: "",
|
||||
EcIndexBits: uint32(1) << uint(i),
|
||||
DiskId: 0,
|
||||
}}
|
||||
}
|
||||
if i == 0 {
|
||||
diskInfo.VolumeInfos = []*master_pb.VolumeInformationMessage{{
|
||||
Id: volumeID,
|
||||
DiskId: 0,
|
||||
DiskType: "hdd",
|
||||
Size: 200 * 1024 * 1024,
|
||||
}}
|
||||
}
|
||||
nodes = append(nodes, &master_pb.DataNodeInfo{
|
||||
Id: nodeID,
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{"hdd": diskInfo},
|
||||
})
|
||||
}
|
||||
require.NoError(t, activeTopology.UpdateTopology(&master_pb.TopologyInfo{
|
||||
DataCenterInfos: []*master_pb.DataCenterInfo{{
|
||||
Id: "dc1",
|
||||
RackInfos: []*master_pb.RackInfo{{
|
||||
Id: "rack1",
|
||||
DataNodeInfos: nodes,
|
||||
}},
|
||||
}},
|
||||
}))
|
||||
return activeTopology
|
||||
}
|
||||
|
||||
// buildStuckSourceMetrics returns a metric that already satisfies the EC
|
||||
// criteria (Age, FullnessRatio, Size), with `Age` derived from `LastModified`
|
||||
// so the two fields stay consistent for any reader.
|
||||
func buildStuckSourceMetrics(volumeID uint32, server string) []*types.VolumeHealthMetrics {
|
||||
lastModified := time.Now().Add(-2 * time.Hour)
|
||||
return []*types.VolumeHealthMetrics{{
|
||||
VolumeID: volumeID,
|
||||
Server: server,
|
||||
Size: 200 * 1024 * 1024,
|
||||
Collection: "",
|
||||
FullnessRatio: 0.96,
|
||||
LastModified: lastModified,
|
||||
Age: time.Since(lastModified),
|
||||
}}
|
||||
}
|
||||
|
||||
// TestCountExistingEcShardsForVolume verifies that the helper walks the
|
||||
// EcIndexBits bitmap (not just len(EcShardInfos)) so it correctly counts
|
||||
// distinct shard ids even when a single info entry on one disk carries
|
||||
// multiple shards.
|
||||
func TestCountExistingEcShardsForVolume(t *testing.T) {
|
||||
const volumeID uint32 = 99
|
||||
activeTopology := topology.NewActiveTopology(10)
|
||||
require.NoError(t, activeTopology.UpdateTopology(&master_pb.TopologyInfo{
|
||||
DataCenterInfos: []*master_pb.DataCenterInfo{{
|
||||
Id: "dc1",
|
||||
RackInfos: []*master_pb.RackInfo{{
|
||||
Id: "rack1",
|
||||
DataNodeInfos: []*master_pb.DataNodeInfo{
|
||||
{
|
||||
Id: "127.0.0.1:8080",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"hdd": {
|
||||
DiskId: 0,
|
||||
MaxVolumeCount: 100,
|
||||
// One info entry, three shards present (ids 0, 2, 5).
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{{
|
||||
Id: volumeID,
|
||||
Collection: "",
|
||||
EcIndexBits: (uint32(1) << 0) | (uint32(1) << 2) | (uint32(1) << 5),
|
||||
DiskId: 0,
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Id: "127.0.0.1:8081",
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"hdd": {
|
||||
DiskId: 0,
|
||||
MaxVolumeCount: 100,
|
||||
// One info entry, one shard (id 3) — overlaps with neither.
|
||||
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{{
|
||||
Id: volumeID,
|
||||
Collection: "",
|
||||
EcIndexBits: uint32(1) << 3,
|
||||
DiskId: 0,
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
}},
|
||||
}))
|
||||
|
||||
assert.Equal(t, 4, countExistingEcShardsForVolume(activeTopology, volumeID, ""))
|
||||
assert.Equal(t, 0, countExistingEcShardsForVolume(activeTopology, volumeID, "other-collection"))
|
||||
assert.Equal(t, 0, countExistingEcShardsForVolume(nil, volumeID, ""))
|
||||
}
|
||||
|
||||
func TestDetectionContextCancellation(t *testing.T) {
|
||||
activeTopology := buildActiveTopology(t, 5, []string{"hdd", "ssd"}, 50, 0, "")
|
||||
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
|
||||
metrics := buildVolumeMetricsForIDs(50)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
_, _, err := Detection(ctx, metrics, clusterInfo, NewDefaultConfig(), 0)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
}
|
||||
|
||||
// The admin UI collection filter takes a list, and the two mode sentinels must
|
||||
// not be read as a collection name.
|
||||
func TestDetectionCollectionFilter(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
filter string
|
||||
want int
|
||||
}{
|
||||
{filter: "", want: 3},
|
||||
{filter: "ALL_COLLECTIONS", want: 3},
|
||||
{filter: "collection-a", want: 3},
|
||||
{filter: "collection-a,collection-b", want: 3},
|
||||
{filter: " collection-b , collection-a ", want: 3},
|
||||
{filter: "collection-*", want: 3},
|
||||
{filter: "collection-b,collection-c", want: 0},
|
||||
{filter: "collection-ab", want: 0},
|
||||
} {
|
||||
t.Run(tc.filter, func(t *testing.T) {
|
||||
// A fresh topology per case: planned destinations reserve capacity.
|
||||
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd"}, 20, 0, "collection-a")
|
||||
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
|
||||
|
||||
metrics := buildVolumeMetricsForIDs(3)
|
||||
for _, metric := range metrics {
|
||||
metric.Collection = "collection-a"
|
||||
}
|
||||
|
||||
config := NewDefaultConfig()
|
||||
config.CollectionFilter = tc.filter
|
||||
results, _, err := Detection(context.Background(), metrics, clusterInfo, config, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, results, tc.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectionMaxResultsHonorsLimit(t *testing.T) {
|
||||
// One node per shard so each shard gets its own disk (#9369).
|
||||
activeTopology := buildActiveTopology(t, erasure_coding.TotalShardsCount, []string{"hdd"}, 20, 0, "")
|
||||
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
|
||||
metrics := buildVolumeMetricsForIDs(3)
|
||||
|
||||
results, hasMore, err := Detection(context.Background(), metrics, clusterInfo, NewDefaultConfig(), 1)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, results, 1)
|
||||
assert.True(t, hasMore)
|
||||
}
|
||||
|
||||
// #9369: 7 servers × 2 physical HDDs must yield 14 distinct (server, disk_id)
|
||||
// destinations, not 7 destinations doubled up on the same disk.
|
||||
func TestPlanECDestinationsSpreadsAcrossPhysicalDisks(t *testing.T) {
|
||||
const numServers = 7
|
||||
const disksPerServer = 2
|
||||
|
||||
activeTopology := topology.NewActiveTopology(10)
|
||||
nodes := make([]*master_pb.DataNodeInfo, 0, numServers)
|
||||
for i := 1; i <= numServers; i++ {
|
||||
volumeInfos := make([]*master_pb.VolumeInformationMessage, 0, disksPerServer)
|
||||
for d := uint32(0); d < disksPerServer; d++ {
|
||||
volumeInfos = append(volumeInfos, &master_pb.VolumeInformationMessage{
|
||||
Id: uint32(i*100 + int(d)),
|
||||
DiskId: d,
|
||||
DiskType: "hdd",
|
||||
})
|
||||
}
|
||||
nodes = append(nodes, &master_pb.DataNodeInfo{
|
||||
Id: fmt.Sprintf("127.0.0.1:%d", 8080+i),
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"hdd": {
|
||||
DiskId: 0,
|
||||
VolumeCount: int64(disksPerServer),
|
||||
MaxVolumeCount: 200,
|
||||
VolumeInfos: volumeInfos,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
require.NoError(t, activeTopology.UpdateTopology(&master_pb.TopologyInfo{
|
||||
DataCenterInfos: []*master_pb.DataCenterInfo{{
|
||||
Id: "dc1",
|
||||
RackInfos: []*master_pb.RackInfo{{
|
||||
Id: "rack1",
|
||||
DataNodeInfos: nodes,
|
||||
}},
|
||||
}},
|
||||
}))
|
||||
|
||||
metric := &types.VolumeHealthMetrics{
|
||||
VolumeID: 42,
|
||||
Server: "127.0.0.1:8081",
|
||||
Size: 100 * 1024 * 1024,
|
||||
Collection: "",
|
||||
}
|
||||
|
||||
snap := ecbalancer.FromActiveTopology(activeTopology, erasure_coding.DataShardsCount)
|
||||
nodeAddresses := buildNodeAddressMap(activeTopology)
|
||||
plan, shardsPerPlan, err := planECDestinations(snap, nodeAddresses, metric, NewDefaultConfig(), nil, erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, plan)
|
||||
requireAllShardsPlaced(t, plan, shardsPerPlan)
|
||||
}
|
||||
|
||||
func TestPlanECDestinationsFailsWithInsufficientCapacity(t *testing.T) {
|
||||
activeTopology := buildActiveTopology(t, 1, []string{"hdd"}, 1, 1, "")
|
||||
|
||||
metric := &types.VolumeHealthMetrics{
|
||||
VolumeID: 2,
|
||||
Server: "10.0.0.1:8080",
|
||||
Size: 10 * 1024 * 1024,
|
||||
Collection: "",
|
||||
}
|
||||
|
||||
snap := ecbalancer.FromActiveTopology(activeTopology, erasure_coding.DataShardsCount)
|
||||
nodeAddresses := buildNodeAddressMap(activeTopology)
|
||||
_, _, err := planECDestinations(snap, nodeAddresses, metric, NewDefaultConfig(), nil, erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
// #9586: with fewer single-disk servers than total shards, EC must still plan
|
||||
// by packing several shards onto a disk (ec.encode's "4,4,3,3" fallback) rather
|
||||
// than refusing. The reporter has 8 single-disk servers across 3 racks and a
|
||||
// 10+4 scheme — 8 disks for 14 shards. minTotalDisks (ceil(14/4)=4) keeps any
|
||||
// disk under parityShards shards, so durability holds.
|
||||
func TestPlanECDestinationsPacksWhenFewerDisksThanShards(t *testing.T) {
|
||||
const numServers = 8
|
||||
// rack3 holds 4 servers, rack1 and rack2 hold 2 each, mirroring the report.
|
||||
racks := []string{"rack3", "rack3", "rack3", "rack3", "rack1", "rack1", "rack2", "rack2"}
|
||||
|
||||
activeTopology := topology.NewActiveTopology(10)
|
||||
rackNodes := make(map[string][]*master_pb.DataNodeInfo)
|
||||
for i := 0; i < numServers; i++ {
|
||||
nodeID := fmt.Sprintf("192.168.1.%d:%d", 143+i/3, 8080+i)
|
||||
rackNodes[racks[i]] = append(rackNodes[racks[i]], &master_pb.DataNodeInfo{
|
||||
Id: nodeID,
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"hdd": {
|
||||
DiskId: 0,
|
||||
VolumeCount: 1,
|
||||
MaxVolumeCount: 200,
|
||||
VolumeInfos: []*master_pb.VolumeInformationMessage{{
|
||||
Id: uint32(i + 1),
|
||||
DiskId: 0,
|
||||
DiskType: "hdd",
|
||||
}},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
rackInfos := make([]*master_pb.RackInfo, 0, len(rackNodes))
|
||||
for _, rackID := range []string{"rack1", "rack2", "rack3"} {
|
||||
rackInfos = append(rackInfos, &master_pb.RackInfo{Id: rackID, DataNodeInfos: rackNodes[rackID]})
|
||||
}
|
||||
require.NoError(t, activeTopology.UpdateTopology(&master_pb.TopologyInfo{
|
||||
DataCenterInfos: []*master_pb.DataCenterInfo{{Id: "dc1", RackInfos: rackInfos}},
|
||||
}))
|
||||
|
||||
metric := &types.VolumeHealthMetrics{
|
||||
VolumeID: 4569,
|
||||
Server: "192.168.1.145:8081",
|
||||
Size: 100 * 1024 * 1024,
|
||||
Collection: "",
|
||||
}
|
||||
|
||||
snap := ecbalancer.FromActiveTopology(activeTopology, erasure_coding.DataShardsCount)
|
||||
nodeAddresses := buildNodeAddressMap(activeTopology)
|
||||
plan, shardsPerPlan, err := planECDestinations(snap, nodeAddresses, metric, NewDefaultConfig(), nil, erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, plan)
|
||||
// Packed onto the available disks: more than one shard per disk but never more
|
||||
// than the 8 disks, and at least the durability floor of distinct disks.
|
||||
require.LessOrEqual(t, len(plan.Plans), numServers)
|
||||
require.GreaterOrEqual(t, len(plan.Plans), (erasure_coding.TotalShardsCount+erasure_coding.ParityShardsCount-1)/erasure_coding.ParityShardsCount)
|
||||
|
||||
// createECTargets must cover all 14 shards exactly once, packing onto the
|
||||
// available disks without any disk exceeding parityShards shards.
|
||||
targets := createECTargets(plan, shardsPerPlan)
|
||||
require.Equal(t, len(plan.Plans), len(targets))
|
||||
|
||||
seenShards := make(map[uint32]bool)
|
||||
for _, target := range targets {
|
||||
require.LessOrEqual(t, len(target.ShardIds), erasure_coding.ParityShardsCount,
|
||||
"no disk may hold more than parityShards shards, else losing it loses the volume")
|
||||
for _, shardId := range target.ShardIds {
|
||||
require.False(t, seenShards[shardId], "shard %d assigned to more than one target", shardId)
|
||||
seenShards[shardId] = true
|
||||
}
|
||||
}
|
||||
require.Len(t, seenShards, erasure_coding.TotalShardsCount, "every shard must be placed exactly once")
|
||||
}
|
||||
|
||||
// requireAllShardsPlaced asserts every EC shard landed exactly once, on a distinct
|
||||
// (node,disk) target, with no disk holding more than parityShards shards (so losing
|
||||
// any one disk cannot lose the volume). shardsPerPlan is parallel to plan.Plans.
|
||||
func requireAllShardsPlaced(t *testing.T, plan *topology.MultiDestinationPlan, shardsPerPlan [][]uint32) {
|
||||
t.Helper()
|
||||
require.Equal(t, len(plan.Plans), len(shardsPerPlan), "one shard list per plan entry")
|
||||
keys := make(map[string]bool, len(plan.Plans))
|
||||
seen := make(map[uint32]bool)
|
||||
for i, p := range plan.Plans {
|
||||
key := fmt.Sprintf("%s:%d", p.TargetNode, p.TargetDisk)
|
||||
require.False(t, keys[key], "duplicate (node,disk) target %s", key)
|
||||
keys[key] = true
|
||||
require.LessOrEqual(t, len(shardsPerPlan[i]), erasure_coding.ParityShardsCount,
|
||||
"disk %s holds %d shards, over parityShards", key, len(shardsPerPlan[i]))
|
||||
for _, s := range shardsPerPlan[i] {
|
||||
require.False(t, seen[s], "shard %d placed more than once", s)
|
||||
seen[s] = true
|
||||
}
|
||||
}
|
||||
require.Len(t, seen, erasure_coding.TotalShardsCount, "every shard must be placed exactly once")
|
||||
}
|
||||
|
||||
func buildVolumeMetricsForIDs(count int) []*types.VolumeHealthMetrics {
|
||||
metrics := make([]*types.VolumeHealthMetrics, 0, count)
|
||||
now := time.Now()
|
||||
for id := 1; id <= count; id++ {
|
||||
metrics = append(metrics, &types.VolumeHealthMetrics{
|
||||
VolumeID: uint32(id),
|
||||
Server: "10.0.0.1:8080",
|
||||
Size: 200 * 1024 * 1024,
|
||||
Collection: "",
|
||||
FullnessRatio: 0.96,
|
||||
LastModified: now.Add(-2 * time.Hour),
|
||||
Age: 2 * time.Hour,
|
||||
})
|
||||
}
|
||||
return metrics
|
||||
}
|
||||
|
||||
func buildActiveTopology(t *testing.T, nodeCount int, diskTypes []string, maxVolumeCount, usedVolumeCount int64, collection string) *topology.ActiveTopology {
|
||||
t.Helper()
|
||||
activeTopology := topology.NewActiveTopology(10)
|
||||
|
||||
nodes := make([]*master_pb.DataNodeInfo, 0, nodeCount)
|
||||
for i := 1; i <= nodeCount; i++ {
|
||||
diskInfos := make(map[string]*master_pb.DiskInfo)
|
||||
for diskIndex, diskType := range diskTypes {
|
||||
used := usedVolumeCount
|
||||
if used > maxVolumeCount {
|
||||
used = maxVolumeCount
|
||||
}
|
||||
volumeInfos := make([]*master_pb.VolumeInformationMessage, 0, 200)
|
||||
for vid := 1; vid <= 200; vid++ {
|
||||
volumeInfos = append(volumeInfos, &master_pb.VolumeInformationMessage{
|
||||
Id: uint32(vid),
|
||||
Collection: collection,
|
||||
DiskId: uint32(diskIndex),
|
||||
})
|
||||
}
|
||||
diskInfos[diskType] = &master_pb.DiskInfo{
|
||||
DiskId: uint32(diskIndex),
|
||||
VolumeCount: used,
|
||||
MaxVolumeCount: maxVolumeCount,
|
||||
VolumeInfos: volumeInfos,
|
||||
}
|
||||
}
|
||||
|
||||
nodes = append(nodes, &master_pb.DataNodeInfo{
|
||||
Id: fmt.Sprintf("10.0.0.%d:8080", i),
|
||||
DiskInfos: diskInfos,
|
||||
})
|
||||
}
|
||||
|
||||
topologyInfo := &master_pb.TopologyInfo{
|
||||
DataCenterInfos: []*master_pb.DataCenterInfo{
|
||||
{
|
||||
Id: "dc1",
|
||||
RackInfos: []*master_pb.RackInfo{
|
||||
{
|
||||
Id: "rack1",
|
||||
DataNodeInfos: nodes,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
require.NoError(t, activeTopology.UpdateTopology(topologyInfo))
|
||||
return activeTopology
|
||||
}
|
||||
1192
weed/worker/tasks/erasure_coding/ec_task.go
Normal file
1192
weed/worker/tasks/erasure_coding/ec_task.go
Normal file
File diff suppressed because it is too large
Load Diff
113
weed/worker/tasks/erasure_coding/ec_task_delete_swallow_test.go
Normal file
113
weed/worker/tasks/erasure_coding/ec_task_delete_swallow_test.go
Normal file
@@ -0,0 +1,113 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// One reachable replica + one unreachable: the reachable delete still
|
||||
// succeeds, and the function surfaces an error naming the failure.
|
||||
func TestDeleteOriginalVolumeSurfacesReplicaFailures(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(91842)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 918420, 0x91842042)
|
||||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
|
||||
[]byte("delete-surface-content-for-issue-9184"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
require.Equal(t, http.StatusCreated, uploadResp.StatusCode)
|
||||
|
||||
task := NewErasureCodingTask(
|
||||
"delete-surface-fix",
|
||||
clusterHarness.VolumeServerAddress(),
|
||||
volumeID,
|
||||
"",
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
|
||||
unreachable := "127.0.0.1:1"
|
||||
task.sources = []*worker_pb.TaskSource{
|
||||
{
|
||||
Node: clusterHarness.VolumeServerAddress(),
|
||||
VolumeId: volumeID,
|
||||
},
|
||||
{
|
||||
Node: unreachable,
|
||||
VolumeId: volumeID,
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := task.deleteOriginalVolume(ctx)
|
||||
require.Error(t, err, "deleteOriginalVolume must surface replica delete failures (#9184)")
|
||||
require.Contains(t, err.Error(), unreachable,
|
||||
"returned error should name the replica that failed: %v", err)
|
||||
require.True(t,
|
||||
strings.Contains(err.Error(), "failed to delete"),
|
||||
"returned error should describe what failed: %v", err)
|
||||
|
||||
_, statusErr := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
|
||||
require.Error(t, statusErr,
|
||||
"reachable replica %d should have been deleted before failure was surfaced", volumeID)
|
||||
}
|
||||
|
||||
func TestDeleteOriginalVolumeSucceedsWhenAllReplicasReachable(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(91844)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 918440, 0x91844042)
|
||||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
|
||||
[]byte("delete-happy-path-content-for-issue-9184"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
require.Equal(t, http.StatusCreated, uploadResp.StatusCode)
|
||||
|
||||
task := NewErasureCodingTask(
|
||||
"delete-happy-path",
|
||||
clusterHarness.VolumeServerAddress(),
|
||||
volumeID,
|
||||
"",
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
task.sources = []*worker_pb.TaskSource{
|
||||
{Node: clusterHarness.VolumeServerAddress(), VolumeId: volumeID},
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
require.NoError(t, task.deleteOriginalVolume(ctx))
|
||||
|
||||
_, statusErr := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
|
||||
require.Error(t, statusErr, "volume %d should be gone after successful delete", volumeID)
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
func TestReplicasPendingDelete(t *testing.T) {
|
||||
replicas := []string{"a:8080", "b:8080", "c:8080"}
|
||||
|
||||
require.Equal(t, replicas, replicasPendingDelete(replicas, nil),
|
||||
"nil swept set leaves the list unchanged")
|
||||
require.ElementsMatch(t, []string{"a:8080", "c:8080"},
|
||||
replicasPendingDelete(replicas, map[string]bool{"b:8080": true}),
|
||||
"swept node is excluded")
|
||||
require.Empty(t,
|
||||
replicasPendingDelete(replicas, map[string]bool{"a:8080": true, "b:8080": true, "c:8080": true}),
|
||||
"all swept yields nothing left to delete")
|
||||
}
|
||||
|
||||
// The sweep leaves a node in place only when VolumeDelete(OnlyEmpty) reports the
|
||||
// volume holds data or no longer exists. An unreachable node leaves its state
|
||||
// unknown and must propagate so the encode fails rather than force-deleting it
|
||||
// later over the shared .vif.
|
||||
func TestIsExpectedSweepSkip(t *testing.T) {
|
||||
require.True(t, isExpectedSweepSkip(errors.New("DeleteVolume 5: volume not empty")))
|
||||
require.True(t, isExpectedSweepSkip(errors.New("volume 5 not found on disk")))
|
||||
require.False(t, isExpectedSweepSkip(errors.New("rpc error: code = Unavailable desc = connection refused")))
|
||||
require.False(t, isExpectedSweepSkip(errors.New("context deadline exceeded")))
|
||||
}
|
||||
|
||||
// The pre-distribute sweep must delete a 0-byte stub replica (so its shared
|
||||
// <collection>_<vid>.vif can't be clobbered by a later original-delete and
|
||||
// can't shadow the incoming EC files) while leaving a data-bearing replica
|
||||
// untouched — VolumeDelete(OnlyEmpty=true) is the guard. A data-bearing
|
||||
// replica is deleted later, only after the EC shard set is verified.
|
||||
func TestSweepEmptyReplicasDeletesStubKeepsData(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
server := clusterHarness.VolumeServerAddress()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// An empty allocated volume is a superblock-only stub.
|
||||
const stubVol = uint32(9479)
|
||||
framework.AllocateVolume(t, grpcClient, stubVol, "ec-stub")
|
||||
|
||||
stubTask := newSweepTaskForTest(server, stubVol, "ec-stub", dialOption)
|
||||
require.NoError(t, stubTask.sweepEmptyReplicas(ctx))
|
||||
|
||||
require.True(t, stubTask.emptyReplicasDeleted[server], "stub server should be recorded as swept")
|
||||
_, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: stubVol})
|
||||
require.Error(t, err, "empty stub volume must be deleted by the sweep")
|
||||
|
||||
// A volume holding data must survive the sweep.
|
||||
const dataVol = uint32(9480)
|
||||
framework.AllocateVolume(t, grpcClient, dataVol, "ec-data")
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(dataVol, 948000, 0x9480CAFE)
|
||||
upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("real-data-keep-me"))
|
||||
_ = framework.ReadAllAndClose(t, upResp)
|
||||
require.Equal(t, http.StatusCreated, upResp.StatusCode)
|
||||
|
||||
dataTask := newSweepTaskForTest(server, dataVol, "ec-data", dialOption)
|
||||
require.NoError(t, dataTask.sweepEmptyReplicas(ctx), "a not-empty refusal is expected, not an error")
|
||||
|
||||
require.False(t, dataTask.emptyReplicasDeleted[server], "data-bearing server must not be recorded as swept")
|
||||
_, err = grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: dataVol})
|
||||
require.NoError(t, err, "data-bearing volume must survive the sweep")
|
||||
}
|
||||
|
||||
func newSweepTaskForTest(server string, volumeID uint32, collection string, dialOption grpc.DialOption) *ErasureCodingTask {
|
||||
task := NewErasureCodingTask("sweep-"+collection, server, volumeID, collection, dialOption)
|
||||
task.sources = []*worker_pb.TaskSource{{Node: server, VolumeId: volumeID}}
|
||||
return task
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// End-to-end on two servers: a volume with a real replica on server A and a
|
||||
// 0-byte stub replica of the same id on server B (an interrupted-encode
|
||||
// leftover). After a full EC encode the cluster must end in exactly one valid
|
||||
// layout — the complete shard set split across A and B, each with .ecx/.vif —
|
||||
// and every wrong file must be gone: both regular .dat files (source deleted
|
||||
// after verify, stub swept before distribute), no shard on the wrong server,
|
||||
// and B's shared <collection>_<vid>.vif intact rather than clobbered by the
|
||||
// stub's delete.
|
||||
func TestEcEncodeLeavesRightFilesAndRemovesStubAndSource(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := framework.StartMultiVolumeCluster(t, matrix.P1(), 2)
|
||||
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
|
||||
const (
|
||||
volumeID = uint32(9490)
|
||||
collection = "ec-e2e"
|
||||
)
|
||||
addrA := serverAddress(cluster, 0)
|
||||
addrB := serverAddress(cluster, 1)
|
||||
|
||||
connA, clientA := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(0))
|
||||
defer connA.Close()
|
||||
connB, clientB := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(1))
|
||||
defer connB.Close()
|
||||
|
||||
// Server A: real source replica with data.
|
||||
framework.AllocateVolume(t, clientA, volumeID, collection)
|
||||
httpClient := framework.NewHTTPClient()
|
||||
for i := 0; i < 8; i++ {
|
||||
fid := framework.NewFileID(volumeID, uint64(948000+i), 0x9490CA00+uint32(i))
|
||||
payload := make([]byte, 4096)
|
||||
for j := range payload {
|
||||
payload[j] = byte(i + 1)
|
||||
}
|
||||
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(0), fid, payload)
|
||||
_ = framework.ReadAllAndClose(t, resp)
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
}
|
||||
|
||||
// Server B: empty stub replica of the same volume id.
|
||||
framework.AllocateVolume(t, clientB, volumeID, collection)
|
||||
|
||||
dataShards := int(erasure_coding.DataShardsCount)
|
||||
totalShards := int(erasure_coding.DataShardsCount + erasure_coding.ParityShardsCount)
|
||||
aShards := shardRange(0, dataShards) // 0..DataShardsCount-1 on A
|
||||
bShards := shardRange(dataShards, totalShards) // parity range on B
|
||||
|
||||
task := NewErasureCodingTask("ec-e2e", addrA, volumeID, collection, dialOption)
|
||||
params := &worker_pb.TaskParams{
|
||||
VolumeId: volumeID,
|
||||
Collection: collection,
|
||||
Sources: []*worker_pb.TaskSource{
|
||||
{Node: addrA, VolumeId: volumeID},
|
||||
{Node: addrB, VolumeId: volumeID},
|
||||
},
|
||||
Targets: []*worker_pb.TaskTarget{
|
||||
{Node: addrA, ShardIds: aShards},
|
||||
{Node: addrB, ShardIds: bShards},
|
||||
},
|
||||
TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
|
||||
ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
|
||||
DataShards: erasure_coding.DataShardsCount,
|
||||
ParityShards: erasure_coding.ParityShardsCount,
|
||||
WorkingDir: t.TempDir(),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
require.NoError(t, task.Execute(ctx, params))
|
||||
|
||||
dirA := filepath.Join(cluster.BaseDir(), "volume0")
|
||||
dirB := filepath.Join(cluster.BaseDir(), "volume1")
|
||||
base := fmt.Sprintf("%s_%d", collection, volumeID)
|
||||
|
||||
// Both original regular volumes are gone: A's source deleted after verify,
|
||||
// B's stub swept before distribute.
|
||||
requireAbsent(t, dirA, base+".dat")
|
||||
requireAbsent(t, dirA, base+".idx")
|
||||
requireAbsent(t, dirB, base+".dat")
|
||||
requireAbsent(t, dirB, base+".idx")
|
||||
|
||||
// Each server holds exactly its assigned shards, plus index/info sidecars.
|
||||
for _, id := range aShards {
|
||||
requirePresent(t, dirA, fmt.Sprintf("%s.ec%02d", base, id))
|
||||
}
|
||||
for _, id := range bShards {
|
||||
requireAbsent(t, dirA, fmt.Sprintf("%s.ec%02d", base, id))
|
||||
}
|
||||
requirePresent(t, dirA, base+".ecx")
|
||||
requirePresent(t, dirA, base+".vif")
|
||||
|
||||
for _, id := range bShards {
|
||||
requirePresent(t, dirB, fmt.Sprintf("%s.ec%02d", base, id))
|
||||
}
|
||||
for _, id := range aShards {
|
||||
requireAbsent(t, dirB, fmt.Sprintf("%s.ec%02d", base, id))
|
||||
}
|
||||
requirePresent(t, dirB, base+".ecx")
|
||||
// The shared .vif must survive on B: the stub was deleted before the EC
|
||||
// files landed, so deleteOriginalVolume never ran removeVolumeFiles there.
|
||||
requirePresent(t, dirB, base+".vif")
|
||||
}
|
||||
|
||||
// serverAddress builds the SeaweedFS ip:httpPort.grpcPort address the worker's
|
||||
// gRPC client decodes, from the multi-cluster's separate admin and grpc ports.
|
||||
func serverAddress(c *framework.MultiVolumeCluster, index int) string {
|
||||
_, grpcPort, err := net.SplitHostPort(c.VolumeGRPCAddress(index))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return c.VolumeAdminAddress(index) + "." + grpcPort
|
||||
}
|
||||
|
||||
func shardRange(start, end int) []uint32 {
|
||||
ids := make([]uint32, 0, end-start)
|
||||
for i := start; i < end; i++ {
|
||||
ids = append(ids, uint32(i))
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func requirePresent(t *testing.T, dir, name string) {
|
||||
t.Helper()
|
||||
_, err := os.Stat(filepath.Join(dir, name))
|
||||
require.NoError(t, err, "expected %s to be present in %s", name, dir)
|
||||
}
|
||||
|
||||
func requireAbsent(t *testing.T, dir, name string) {
|
||||
t.Helper()
|
||||
_, err := os.Stat(filepath.Join(dir, name))
|
||||
require.True(t, os.IsNotExist(err), "expected %s to be absent in %s, stat err=%v", name, dir, err)
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// Reproduces the issue-9478 volume-13 layout end to end on a multi-server,
|
||||
// multi-disk cluster and asserts the cluster converges to exactly one valid EC
|
||||
// layout. Before the encode the volume is in the stuck state: a real .dat
|
||||
// source, a 0-byte stub replica left by an interrupted encode, and partial
|
||||
// stale EC shards from that attempt. After the worker runs (sweep stubs ->
|
||||
// clear stale shards -> distribute -> mount -> verify -> delete originals):
|
||||
//
|
||||
// - every assigned shard is present on its target disk, with .ecx/.vif
|
||||
// alongside it (the EC volume's info file survives even when shards land on
|
||||
// the source's own disk);
|
||||
// - no regular .dat/.idx remains anywhere — the source is deleted after
|
||||
// verify, the stub was swept before distribute;
|
||||
// - the stale shards that aren't part of the new assignment are gone.
|
||||
func TestEcEncodeJulorLayoutConverges(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
const disksPerServer = 2
|
||||
cluster := framework.StartMultiVolumeClusterWithDisks(t, matrix.P1(), 3, disksPerServer)
|
||||
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
|
||||
const (
|
||||
volumeID = uint32(9500)
|
||||
collection = "ec-julor"
|
||||
)
|
||||
// S0: real source. S1: 0-byte stub + EC target. S2: pure EC target.
|
||||
const (
|
||||
srcServer = 0
|
||||
stubServer = 1
|
||||
)
|
||||
addr := func(i int) string { return serverAddress(cluster, i) }
|
||||
|
||||
conns := make([]*grpc.ClientConn, 3)
|
||||
clients := make([]volume_server_pb.VolumeServerClient, 3)
|
||||
for i := 0; i < 3; i++ {
|
||||
conns[i], clients[i] = framework.DialVolumeServer(t, cluster.VolumeGRPCAddress(i))
|
||||
defer conns[i].Close()
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// S0: real source replica with data.
|
||||
framework.AllocateVolume(t, clients[srcServer], volumeID, collection)
|
||||
httpClient := framework.NewHTTPClient()
|
||||
for i := 0; i < 8; i++ {
|
||||
fid := framework.NewFileID(volumeID, uint64(950000+i), 0x9500CA00+uint32(i))
|
||||
payload := make([]byte, 4096)
|
||||
for j := range payload {
|
||||
payload[j] = byte(i + 1)
|
||||
}
|
||||
resp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(srcServer), fid, payload)
|
||||
_ = framework.ReadAllAndClose(t, resp)
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
}
|
||||
|
||||
// Partial stale EC shards from a previous interrupted encode, mounted on the
|
||||
// source. cleanupStaleEcShards must clear them before re-distribute.
|
||||
_, err := clients[srcServer].VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID, Collection: collection,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
staleShards := []uint32{11, 12, 13}
|
||||
_, err = clients[srcServer].VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID, Collection: collection, ShardIds: staleShards,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// S1: a 0-byte stub replica of the same volume (interrupted-encode leftover).
|
||||
framework.AllocateVolume(t, clients[stubServer], volumeID, collection)
|
||||
|
||||
// Plan: 14 shards spread across all three servers and both disks. The source
|
||||
// server (S0) receives shards too, so its regular volume coexists with EC on
|
||||
// the same disk before deletion.
|
||||
targets := []*worker_pb.TaskTarget{
|
||||
{Node: addr(0), DiskId: 0, ShardIds: shardRange(0, 3)}, // S0 d0: 0,1,2
|
||||
{Node: addr(0), DiskId: 1, ShardIds: shardRange(3, 5)}, // S0 d1: 3,4
|
||||
{Node: addr(1), DiskId: 0, ShardIds: shardRange(5, 8)}, // S1 d0: 5,6,7
|
||||
{Node: addr(1), DiskId: 1, ShardIds: shardRange(8, 10)}, // S1 d1: 8,9
|
||||
{Node: addr(2), DiskId: 0, ShardIds: shardRange(10, 12)}, // S2 d0: 10,11
|
||||
{Node: addr(2), DiskId: 1, ShardIds: shardRange(12, 14)}, // S2 d1: 12,13
|
||||
}
|
||||
|
||||
task := NewErasureCodingTask("ec-julor", addr(srcServer), volumeID, collection, dialOption)
|
||||
params := &worker_pb.TaskParams{
|
||||
VolumeId: volumeID,
|
||||
Collection: collection,
|
||||
Sources: []*worker_pb.TaskSource{
|
||||
{Node: addr(srcServer), VolumeId: volumeID}, // real source
|
||||
{Node: addr(stubServer), VolumeId: volumeID}, // 0-byte stub
|
||||
{Node: addr(srcServer), VolumeId: volumeID, ShardIds: staleShards}, // stale EC shards to clear
|
||||
},
|
||||
Targets: targets,
|
||||
TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
|
||||
ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
|
||||
DataShards: erasure_coding.DataShardsCount,
|
||||
ParityShards: erasure_coding.ParityShardsCount,
|
||||
WorkingDir: t.TempDir(),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
require.NoError(t, task.Execute(ctx, params))
|
||||
|
||||
base := fmt.Sprintf("%s_%d", collection, volumeID)
|
||||
|
||||
// Shards are distributed per-server; the disk within a server is auto-selected
|
||||
// when DiskId is 0, and on a multi-disk server shards can spread to a sibling
|
||||
// disk while the .ecx/.vif index lives on one of them (the cross-disk EC
|
||||
// layout). So assert per server (aggregate across its disks): each server
|
||||
// holds exactly its assigned shards, its .ecx/.vif survive on at least one
|
||||
// disk (the info file is not stripped, even on the source's own disk), and no
|
||||
// regular .dat/.idx remains. Stale shards 11-13 mounted on the source must be
|
||||
// gone — they are not in S0's assignment.
|
||||
serverShards := map[int][]uint32{
|
||||
0: shardRange(0, 5), // S0 (source): 0..4
|
||||
1: shardRange(5, 10), // S1 (stub): 5..9
|
||||
2: shardRange(10, 14), // S2 (target): 10..13
|
||||
}
|
||||
allShards := map[uint32]bool{}
|
||||
for server, want := range serverShards {
|
||||
found := map[uint32]bool{}
|
||||
hasEcx, hasVif := false, false
|
||||
for d := 0; d < disksPerServer; d++ {
|
||||
dir := cluster.VolumeDiskDir(server, d)
|
||||
requireAbsent(t, dir, base+".dat")
|
||||
requireAbsent(t, dir, base+".idx")
|
||||
for id := 0; id < erasure_coding.TotalShardsCount; id++ {
|
||||
if ecFileExists(dir, fmt.Sprintf("%s.ec%02d", base, id)) {
|
||||
found[uint32(id)] = true
|
||||
allShards[uint32(id)] = true
|
||||
}
|
||||
}
|
||||
hasEcx = hasEcx || ecFileExists(dir, base+".ecx")
|
||||
hasVif = hasVif || ecFileExists(dir, base+".vif")
|
||||
}
|
||||
gotIDs := make([]uint32, 0, len(found))
|
||||
for id := range found {
|
||||
gotIDs = append(gotIDs, id)
|
||||
}
|
||||
require.ElementsMatch(t, want, gotIDs, "server %d should hold exactly its assigned shards", server)
|
||||
require.True(t, hasEcx, "server %d must keep its .ecx", server)
|
||||
require.True(t, hasVif, "server %d must keep its .vif (not stripped by the original-volume delete)", server)
|
||||
}
|
||||
require.Len(t, allShards, erasure_coding.TotalShardsCount, "the full shard set must be present across the cluster")
|
||||
}
|
||||
|
||||
func ecFileExists(dir, name string) bool {
|
||||
_, err := os.Stat(filepath.Join(dir, name))
|
||||
return err == nil
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// cleanupOrphanSourceReplicas must not delete a source replica that has come
|
||||
// back writable: it may hold writes the EC shards do not contain. A still
|
||||
// readonly replica is the genuine orphan and is deleted.
|
||||
func TestCleanupOrphanSkipsWritableSourceReplica(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(955)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
clusterInfo := &types.ClusterInfo{
|
||||
ActiveTopology: buildSingleNodeTopology(t, clusterHarness.VolumeServerAddress(), volumeID),
|
||||
GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
}
|
||||
metric := &types.VolumeHealthMetrics{VolumeID: volumeID, Collection: ""}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// Writable source replica: must be left intact.
|
||||
deleted, err := cleanupOrphanSourceReplicas(ctx, clusterInfo, metric, 0)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, deleted, "a writable source replica must not be deleted")
|
||||
require.True(t, volumeExists(t, grpcClient, volumeID), "writable source volume must still exist")
|
||||
|
||||
// Mark readonly: now it is a genuine orphan and must be deleted.
|
||||
_, err = grpcClient.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{VolumeId: volumeID})
|
||||
require.NoError(t, err)
|
||||
|
||||
deleted, err = cleanupOrphanSourceReplicas(ctx, clusterInfo, metric, 0)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, deleted, "a readonly orphan source replica must be deleted")
|
||||
require.False(t, volumeExists(t, grpcClient, volumeID), "readonly orphan source volume must be deleted")
|
||||
}
|
||||
|
||||
func volumeExists(t *testing.T, client volume_server_pb.VolumeServerClient, volumeID uint32) bool {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, err := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// buildSingleNodeTopology builds an ActiveTopology with one node (at serverAddr)
|
||||
// holding the regular volume, so cleanupOrphanSourceReplicas resolves the
|
||||
// replica to the live test server and its gRPC calls reach it.
|
||||
func buildSingleNodeTopology(t *testing.T, serverAddr string, volumeID uint32) *topology.ActiveTopology {
|
||||
t.Helper()
|
||||
at := topology.NewActiveTopology(10)
|
||||
topologyInfo := &master_pb.TopologyInfo{
|
||||
DataCenterInfos: []*master_pb.DataCenterInfo{{
|
||||
Id: "dc1",
|
||||
RackInfos: []*master_pb.RackInfo{{
|
||||
Id: "rack1",
|
||||
DataNodeInfos: []*master_pb.DataNodeInfo{{
|
||||
Id: serverAddr,
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{
|
||||
"hdd": {
|
||||
DiskId: 0,
|
||||
VolumeInfos: []*master_pb.VolumeInformationMessage{
|
||||
{Id: volumeID, Collection: "", DiskId: 0},
|
||||
},
|
||||
},
|
||||
},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
}
|
||||
require.NoError(t, at.UpdateTopology(topologyInfo))
|
||||
return at
|
||||
}
|
||||
231
weed/worker/tasks/erasure_coding/ec_task_preflight_test.go
Normal file
231
weed/worker/tasks/erasure_coding/ec_task_preflight_test.go
Normal file
@@ -0,0 +1,231 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// Reproduces the swtest interrupted-encode scenario at the task boundary: a
|
||||
// previous encode of this volume was cut mid-distribute, leaving partial EC
|
||||
// shards mounted on a destination. When a fresh encode task starts, its Step 0
|
||||
// preflight (ensureCleanEcStart) must clear those shards before any
|
||||
// destructive step, so distribute's ReceiveFile is not refused and no orphan
|
||||
// shards survive to confuse the volume-server loader or make detection refuse
|
||||
// the volume. The destination is named only as a target (no source row) to
|
||||
// prove the preflight reaches the full write set, not just shard-bearing
|
||||
// sources.
|
||||
func TestEnsureCleanEcStartClearsStaleShardsBeforeEncode(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const (
|
||||
volumeID = uint32(94782)
|
||||
collection = "ec-preflight-invariant"
|
||||
)
|
||||
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, collection)
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 9478200, 0x9478BEEF)
|
||||
upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
|
||||
[]byte("payload-for-preflight-stale-ec-cleanup"))
|
||||
_ = framework.ReadAllAndClose(t, upResp)
|
||||
require.Equal(t, http.StatusCreated, upResp.StatusCode)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID, Collection: collection,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// A half-finished previous distribute left a partial shard set mounted.
|
||||
staleShards := []uint32{0, 1, 2}
|
||||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID, Collection: collection,
|
||||
ShardIds: staleShards,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
shardPath := makeTinyEcShardFile(t)
|
||||
|
||||
// Precondition for the reproduction: the mounted partial EC blocks a fresh
|
||||
// ReceiveFile via the mounted-volume guard.
|
||||
err = sendShardViaReceiveFile(ctx, grpcClient, volumeID, collection, 0, shardPath)
|
||||
require.Error(t, err, "expected ReceiveFile to be refused while stale EC volume is mounted")
|
||||
|
||||
task := NewErasureCodingTask(
|
||||
"preflight-invariant",
|
||||
clusterHarness.VolumeServerAddress(),
|
||||
volumeID,
|
||||
collection,
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
// Named only as a target — no source row for this node. The preflight must
|
||||
// still clear its stale shards.
|
||||
task.targets = []*worker_pb.TaskTarget{
|
||||
{Node: clusterHarness.VolumeServerAddress(), VolumeId: volumeID, ShardIds: []uint32{0}},
|
||||
}
|
||||
|
||||
require.NoError(t, task.ensureCleanEcStart(ctx),
|
||||
"Step 0 preflight must clear stale EC shards at task start")
|
||||
|
||||
_, infoErr := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{VolumeId: volumeID})
|
||||
require.Error(t, infoErr, "stale EC volume must be gone after the preflight")
|
||||
|
||||
require.NoError(t,
|
||||
sendShardViaReceiveFile(ctx, grpcClient, volumeID, collection, 0, shardPath),
|
||||
"ReceiveFile must succeed once the preflight has cleared the stale shards")
|
||||
}
|
||||
|
||||
// A post-distribute failure that has no successor to clean up after it (a
|
||||
// single-attempt job, or the last of a retry series) must leave nothing behind:
|
||||
// rollbackDistribute tears down the shards this attempt wrote while leaving the
|
||||
// source volume intact for a future re-encode. Reproduces the swtest finding
|
||||
// that a terminally-failed encode stranded orphan shards.
|
||||
func TestRollbackDistributeClearsShardsAndKeepsSource(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const (
|
||||
volumeID = uint32(94783)
|
||||
collection = "ec-rollback-distribute"
|
||||
)
|
||||
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, collection)
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 9478300, 0x9478D00D)
|
||||
upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
|
||||
[]byte("payload-for-rollback-distribute"))
|
||||
_ = framework.ReadAllAndClose(t, upResp)
|
||||
require.Equal(t, http.StatusCreated, upResp.StatusCode)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID, Collection: collection,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
// Shards this attempt "distributed" and mounted on the destination.
|
||||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID, Collection: collection,
|
||||
ShardIds: []uint32{0, 1, 2, 3},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
task := NewErasureCodingTask(
|
||||
"rollback-distribute",
|
||||
clusterHarness.VolumeServerAddress(),
|
||||
volumeID,
|
||||
collection,
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
task.targets = []*worker_pb.TaskTarget{
|
||||
{Node: clusterHarness.VolumeServerAddress(), VolumeId: volumeID, ShardIds: []uint32{0}},
|
||||
}
|
||||
|
||||
// Simulate the failure path after distribute began.
|
||||
task.rollbackDistribute(ctx)
|
||||
|
||||
_, infoErr := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{VolumeId: volumeID})
|
||||
require.Error(t, infoErr, "rollback must tear down the shards this attempt distributed")
|
||||
|
||||
// The source normal volume must survive so a future re-encode can proceed.
|
||||
_, statusErr := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
|
||||
require.NoError(t, statusErr, "rollback must leave the source volume intact")
|
||||
}
|
||||
|
||||
// A malformed plan (no targets) must be rejected by the preflight before the
|
||||
// source is marked readonly or copied, so a bad plan cannot leave the source
|
||||
// fenced readonly with nothing to show for it. Pure precondition check: no
|
||||
// cluster and no RPC.
|
||||
func TestEnsureCleanEcStartRejectsMissingTargets(t *testing.T) {
|
||||
task := NewErasureCodingTask(
|
||||
"preflight-no-targets",
|
||||
"10.0.0.1:8080",
|
||||
42,
|
||||
"c",
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
// No targets set.
|
||||
err := task.ensureCleanEcStart(context.Background())
|
||||
require.Error(t, err, "preflight must reject an encode with no shard targets")
|
||||
require.True(t, strings.Contains(err.Error(), "no EC shard targets"),
|
||||
"error must name the missing-targets invariant, got: %v", err)
|
||||
}
|
||||
|
||||
// A non-empty target slice whose entries are malformed (nil, empty Node, or no
|
||||
// assigned shards) must also be rejected before the source is marked readonly —
|
||||
// cleanupStaleEcShards silently skips such entries, so a length check alone
|
||||
// would let the encode proceed with nothing to distribute to.
|
||||
func TestEnsureCleanEcStartRejectsMalformedTarget(t *testing.T) {
|
||||
cases := map[string][]*worker_pb.TaskTarget{
|
||||
"nil target": {nil},
|
||||
"empty node": {{Node: "", VolumeId: 42, ShardIds: []uint32{0}}},
|
||||
"no shards": {{Node: "10.0.0.2:8080", VolumeId: 42}},
|
||||
"one good one bad": {
|
||||
{Node: "10.0.0.2:8080", VolumeId: 42, ShardIds: []uint32{0}},
|
||||
{Node: "", VolumeId: 42, ShardIds: []uint32{1}},
|
||||
},
|
||||
}
|
||||
for name, targets := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
task := NewErasureCodingTask(
|
||||
"preflight-malformed-target",
|
||||
"10.0.0.1:8080",
|
||||
42,
|
||||
"c",
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
task.targets = targets
|
||||
err := task.ensureCleanEcStart(context.Background())
|
||||
require.Error(t, err, "preflight must reject a malformed target")
|
||||
require.True(t, strings.Contains(err.Error(), "malformed EC shard target"),
|
||||
"error must name the malformed-target invariant, got: %v", err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A plan with targets but no source replica and no assigned server must be
|
||||
// rejected before any destructive step.
|
||||
func TestEnsureCleanEcStartRejectsMissingSource(t *testing.T) {
|
||||
task := NewErasureCodingTask(
|
||||
"preflight-no-source",
|
||||
"", // no assigned server
|
||||
42,
|
||||
"c",
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
task.targets = []*worker_pb.TaskTarget{
|
||||
{Node: "10.0.0.2:8080", VolumeId: 42, ShardIds: []uint32{0}},
|
||||
}
|
||||
// No sources and no server.
|
||||
err := task.ensureCleanEcStart(context.Background())
|
||||
require.Error(t, err, "preflight must reject an encode with no source replica")
|
||||
require.True(t, strings.Contains(err.Error(), "no source replica"),
|
||||
"error must name the missing-source invariant, got: %v", err)
|
||||
}
|
||||
@@ -0,0 +1,363 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
// Reproduces a stuck re-encode: partial EC shards mounted on a destination
|
||||
// from a previous failed encode cause ReceiveFile to refuse with the
|
||||
// mounted-volume guard. cleanupStaleEcShards must clear them so the next
|
||||
// ReceiveFile lands.
|
||||
func TestCleanupStaleEcShardsBeforeDistribute(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const (
|
||||
volumeID = uint32(9478)
|
||||
collection = "ec-9478-xserver"
|
||||
)
|
||||
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, collection)
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 947800, 0x9478CAFE)
|
||||
upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
|
||||
[]byte("payload-for-cross-server-stale-ec-cleanup"))
|
||||
_ = framework.ReadAllAndClose(t, upResp)
|
||||
require.Equal(t, http.StatusCreated, upResp.StatusCode)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID, Collection: collection,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Partial subset mimics a half-finished previous distribute: shards
|
||||
// mounted on the destination with no .dat to anchor a same-store prune.
|
||||
staleShards := []uint32{0, 1, 2}
|
||||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID, Collection: collection,
|
||||
ShardIds: staleShards,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
shardPath := makeTinyEcShardFile(t)
|
||||
|
||||
// Pre-cleanup: the mounted partial EC blocks ReceiveFile.
|
||||
err = sendShardViaReceiveFile(ctx, grpcClient, volumeID, collection, 0, shardPath)
|
||||
require.Error(t, err, "expected ReceiveFile to be refused while EC volume is mounted")
|
||||
require.True(t,
|
||||
strings.Contains(err.Error(), "is mounted") ||
|
||||
strings.Contains(err.Error(), "unmount before ReceiveFile"),
|
||||
"expected refusal to name the mounted-volume guard, got: %v", err)
|
||||
|
||||
// ShardIds set marks this as an EC-shard cleanup source: cleanup will
|
||||
// target it; getReplicas must skip it.
|
||||
task := NewErasureCodingTask(
|
||||
"stale-ec-xserver",
|
||||
clusterHarness.VolumeServerAddress(),
|
||||
volumeID,
|
||||
collection,
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
task.dataShards = erasure_coding.DataShardsCount
|
||||
task.parityShards = erasure_coding.ParityShardsCount
|
||||
task.sources = []*worker_pb.TaskSource{
|
||||
{
|
||||
Node: clusterHarness.VolumeServerAddress(),
|
||||
VolumeId: volumeID,
|
||||
ShardIds: staleShards,
|
||||
},
|
||||
}
|
||||
|
||||
require.NoError(t, task.cleanupStaleEcShards(ctx))
|
||||
|
||||
_, infoErr := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{VolumeId: volumeID})
|
||||
require.Error(t, infoErr, "EC volume should be gone after cleanupStaleEcShards")
|
||||
|
||||
require.NoError(t,
|
||||
sendShardViaReceiveFile(ctx, grpcClient, volumeID, collection, 0, shardPath),
|
||||
"ReceiveFile must succeed after cleanup")
|
||||
|
||||
require.Empty(t, task.getReplicas(),
|
||||
"EC-shard sources must not appear in replica delete list")
|
||||
}
|
||||
|
||||
// The destination has more shards mounted than t.sources lists — simulates
|
||||
// detection-time topology missing shards (e.g., a prior attempt's mount
|
||||
// hadn't heartbeated yet, or different shards live on a sibling disk). The
|
||||
// cleanup must clear FindEcVolume regardless, by issuing unmount/delete
|
||||
// over the full shard range, so the next ReceiveFile lands.
|
||||
func TestCleanupStaleEcShardsClearsShardsBeyondSources(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const (
|
||||
volumeID = uint32(94780)
|
||||
collection = "ec-9478-beyond"
|
||||
)
|
||||
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, collection)
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 9478001, 0x9478FACE)
|
||||
upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
|
||||
[]byte("payload-for-shards-beyond-sources"))
|
||||
_ = framework.ReadAllAndClose(t, upResp)
|
||||
require.Equal(t, http.StatusCreated, upResp.StatusCode)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID, Collection: collection,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Five shards mounted on the destination.
|
||||
mountedShards := []uint32{0, 1, 2, 3, 4}
|
||||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID, Collection: collection,
|
||||
ShardIds: mountedShards,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Detection only "saw" two of them — the rest must still get cleared.
|
||||
knownToDetection := []uint32{0, 1}
|
||||
|
||||
task := NewErasureCodingTask(
|
||||
"stale-ec-beyond-sources",
|
||||
clusterHarness.VolumeServerAddress(),
|
||||
volumeID,
|
||||
collection,
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
task.dataShards = erasure_coding.DataShardsCount
|
||||
task.parityShards = erasure_coding.ParityShardsCount
|
||||
task.sources = []*worker_pb.TaskSource{
|
||||
{
|
||||
Node: clusterHarness.VolumeServerAddress(),
|
||||
VolumeId: volumeID,
|
||||
ShardIds: knownToDetection,
|
||||
},
|
||||
}
|
||||
|
||||
require.NoError(t, task.cleanupStaleEcShards(ctx))
|
||||
|
||||
_, infoErr := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{VolumeId: volumeID})
|
||||
require.Error(t, infoErr,
|
||||
"all mounted shards must be cleared even though detection only listed a subset")
|
||||
|
||||
shardPath := makeTinyEcShardFile(t)
|
||||
require.NoError(t,
|
||||
sendShardViaReceiveFile(ctx, grpcClient, volumeID, collection, 4, shardPath),
|
||||
"ReceiveFile for a shard outside detection's snapshot must succeed after cleanup")
|
||||
}
|
||||
|
||||
// Cleanup also targets fresh destinations from t.targets even when no
|
||||
// EC-shard source row exists for them. This catches concurrent-attempt
|
||||
// fallout where a previous worker mounted shards on a node we're now
|
||||
// writing to but the topology snapshot is stale.
|
||||
func TestCleanupStaleEcShardsCoversTargetsWithoutSources(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const (
|
||||
volumeID = uint32(94781)
|
||||
collection = "ec-9478-targets-only"
|
||||
)
|
||||
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, collection)
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 9478101, 0x947811CE)
|
||||
upResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid,
|
||||
[]byte("payload-for-targets-only-cleanup"))
|
||||
_ = framework.ReadAllAndClose(t, upResp)
|
||||
require.Equal(t, http.StatusCreated, upResp.StatusCode)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID, Collection: collection,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID, Collection: collection,
|
||||
ShardIds: []uint32{0, 1},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
task := NewErasureCodingTask(
|
||||
"stale-ec-targets-only",
|
||||
clusterHarness.VolumeServerAddress(),
|
||||
volumeID,
|
||||
collection,
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
task.dataShards = erasure_coding.DataShardsCount
|
||||
task.parityShards = erasure_coding.ParityShardsCount
|
||||
// No sources. The destination is named only as a target — the cleanup
|
||||
// must still reach it.
|
||||
task.targets = []*worker_pb.TaskTarget{
|
||||
{Node: clusterHarness.VolumeServerAddress(), VolumeId: volumeID, ShardIds: []uint32{0}},
|
||||
}
|
||||
|
||||
require.NoError(t, task.cleanupStaleEcShards(ctx))
|
||||
|
||||
_, infoErr := grpcClient.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{VolumeId: volumeID})
|
||||
require.Error(t, infoErr,
|
||||
"target-only destinations must also be cleaned even without a source row")
|
||||
}
|
||||
|
||||
// Cleanup is a no-op when sources carry only the regular .dat replica.
|
||||
func TestCleanupStaleEcShardsSkipsRegularReplicas(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(9479)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
task := NewErasureCodingTask(
|
||||
"no-stale-ec",
|
||||
clusterHarness.VolumeServerAddress(),
|
||||
volumeID,
|
||||
"",
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
task.sources = []*worker_pb.TaskSource{
|
||||
{Node: clusterHarness.VolumeServerAddress(), VolumeId: volumeID},
|
||||
}
|
||||
|
||||
require.NoError(t, task.cleanupStaleEcShards(ctx))
|
||||
|
||||
_, err := grpcClient.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{VolumeId: volumeID})
|
||||
require.NoError(t, err, "regular volume must remain untouched")
|
||||
}
|
||||
|
||||
// makeTinyEcShardFile writes a placeholder payload — the mounted-volume
|
||||
// guard fires before any content is consumed, so the bytes don't need to
|
||||
// be a real shard.
|
||||
func makeTinyEcShardFile(t *testing.T) string {
|
||||
t.Helper()
|
||||
p := filepath.Join(t.TempDir(), "shard.bin")
|
||||
require.NoError(t, os.WriteFile(p, []byte("ec-shard-placeholder"), 0o600))
|
||||
return p
|
||||
}
|
||||
|
||||
// sendShardViaReceiveFile streams a shard file through the same ReceiveFile
|
||||
// gRPC the EC worker uses, returning the server's reply error verbatim.
|
||||
func sendShardViaReceiveFile(
|
||||
ctx context.Context,
|
||||
client volume_server_pb.VolumeServerClient,
|
||||
volumeID uint32,
|
||||
collection string,
|
||||
shardID uint32,
|
||||
filePath string,
|
||||
) error {
|
||||
f, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stream, err := client.ReceiveFile(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := stream.Send(&volume_server_pb.ReceiveFileRequest{
|
||||
Data: &volume_server_pb.ReceiveFileRequest_Info{
|
||||
Info: &volume_server_pb.ReceiveFileInfo{
|
||||
VolumeId: volumeID,
|
||||
Ext: erasure_coding.ToExt(int(shardID)),
|
||||
Collection: collection,
|
||||
IsEcVolume: true,
|
||||
ShardId: shardID,
|
||||
FileSize: uint64(info.Size()),
|
||||
},
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
buf := make([]byte, 32*1024)
|
||||
for {
|
||||
n, readErr := f.Read(buf)
|
||||
if n > 0 {
|
||||
if err := stream.Send(&volume_server_pb.ReceiveFileRequest{
|
||||
Data: &volume_server_pb.ReceiveFileRequest_FileContent{
|
||||
FileContent: buf[:n],
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if readErr == io.EOF {
|
||||
break
|
||||
}
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := stream.CloseAndRecv()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.Error != "" {
|
||||
return &receiveFileServerError{msg: resp.Error}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type receiveFileServerError struct{ msg string }
|
||||
|
||||
func (e *receiveFileServerError) Error() string { return e.msg }
|
||||
249
weed/worker/tasks/erasure_coding/ec_task_test.go
Normal file
249
weed/worker/tasks/erasure_coding/ec_task_test.go
Normal file
@@ -0,0 +1,249 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
func TestCopyVolumeFilesToWorkerUsesCurrentCompactionRevision(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(951)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
|
||||
liveFID := framework.NewFileID(volumeID, 1001, 0x1111AAAA)
|
||||
liveUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), liveFID, []byte("live-payload-for-ec-copy"))
|
||||
_ = framework.ReadAllAndClose(t, liveUploadResp)
|
||||
require.Equal(t, http.StatusCreated, liveUploadResp.StatusCode)
|
||||
|
||||
deletedFID := framework.NewFileID(volumeID, 1002, 0x2222BBBB)
|
||||
deletedUploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), deletedFID, []byte("deleted-payload-for-vacuum"))
|
||||
_ = framework.ReadAllAndClose(t, deletedUploadResp)
|
||||
require.Equal(t, http.StatusCreated, deletedUploadResp.StatusCode)
|
||||
|
||||
deleteReq, err := http.NewRequest(http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+deletedFID, nil)
|
||||
require.NoError(t, err)
|
||||
deleteResp := framework.DoRequest(t, httpClient, deleteReq)
|
||||
_ = framework.ReadAllAndClose(t, deleteResp)
|
||||
require.Equal(t, http.StatusAccepted, deleteResp.StatusCode)
|
||||
|
||||
compactVolumeOnce(t, grpcClient, volumeID)
|
||||
|
||||
task := NewErasureCodingTask(
|
||||
"copy-after-compaction",
|
||||
clusterHarness.VolumeServerAddress(),
|
||||
volumeID,
|
||||
"",
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
require.NoError(t, task.markReplicasReadonly(ctx))
|
||||
|
||||
fileStatus, err := task.readSourceVolumeFileStatus(ctx)
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, fileStatus.GetCompactionRevision(), uint32(0))
|
||||
|
||||
localFiles, err := task.copyVolumeFilesToWorker(ctx, t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
datInfo, err := os.Stat(localFiles["dat"])
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(fileStatus.GetDatFileSize()), datInfo.Size())
|
||||
|
||||
idxInfo, err := os.Stat(localFiles["idx"])
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(fileStatus.GetIdxFileSize()), idxInfo.Size())
|
||||
}
|
||||
|
||||
// markReplicasReadonly must persist the readonly mark into the .vif so a
|
||||
// source-server restart cannot silently reopen the volume to writes that the
|
||||
// EC shards would not contain.
|
||||
func TestMarkReplicasReadonlyPersists(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(954)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 3001, 0x4444DDDD)
|
||||
resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("payload-for-readonly-persist"))
|
||||
_ = framework.ReadAllAndClose(t, resp)
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
|
||||
task := NewErasureCodingTask(
|
||||
"ec-readonly-persist",
|
||||
clusterHarness.VolumeServerAddress(),
|
||||
volumeID,
|
||||
"",
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
require.NoError(t, task.markReplicasReadonly(ctx))
|
||||
|
||||
vifPath := filepath.Join(clusterHarness.BaseDir(), "volume", fmt.Sprintf("%d.vif", volumeID))
|
||||
vi, _, found, err := volume_info.MaybeLoadVolumeInfo(vifPath)
|
||||
require.NoError(t, err)
|
||||
require.True(t, found, "volume must have a .vif after a persisted readonly mark")
|
||||
require.True(t, vi.ReadOnly, "markReplicasReadonly must persist ReadOnly=true into the .vif")
|
||||
}
|
||||
|
||||
// The worker-local encode path must stamp the EC ratio and an encode identity
|
||||
// into the .vif, or the read guard is silently off for worker-encoded volumes.
|
||||
func TestGenerateEcShardsLocallyStampsEncodeIdentity(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(952)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 2001, 0x3333CCCC)
|
||||
resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("payload-for-ec-encode-identity"))
|
||||
_ = framework.ReadAllAndClose(t, resp)
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
|
||||
task := NewErasureCodingTask(
|
||||
"ec-encode-identity",
|
||||
clusterHarness.VolumeServerAddress(),
|
||||
volumeID,
|
||||
"",
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
require.NoError(t, task.markReplicasReadonly(ctx))
|
||||
localFiles, err := task.copyVolumeFilesToWorker(ctx, t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
shardFiles, err := task.generateEcShardsLocally(localFiles, t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
vifPath := shardFiles["vif"]
|
||||
require.NotEmpty(t, vifPath, "generate must produce a .vif")
|
||||
vi, _, found, err := volume_info.MaybeLoadVolumeInfo(vifPath)
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
require.NotNil(t, vi.EcShardConfig, "worker .vif must record the EC config")
|
||||
require.Greater(t, vi.EcShardConfig.DataShards, uint32(0))
|
||||
require.Greater(t, vi.EcShardConfig.ParityShards, uint32(0))
|
||||
require.NotZero(t, vi.EcShardConfig.EncodeTsNs, "worker-encoded .vif must carry an encode identity")
|
||||
}
|
||||
|
||||
// A fenced run must stamp the admin-issued generation (t.encodeTsNs) verbatim into
|
||||
// the distributed .vif, so the generation the worker distributes equals the one it
|
||||
// later sends on the teardown RPCs and the volume-server fence compares like-for-like.
|
||||
func TestGenerateEcShardsLocallyUsesAdminGeneration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(957)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 2002, 0x5555EEEE)
|
||||
resp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("payload-for-admin-generation"))
|
||||
_ = framework.ReadAllAndClose(t, resp)
|
||||
require.Equal(t, http.StatusCreated, resp.StatusCode)
|
||||
|
||||
task := NewErasureCodingTask(
|
||||
"ec-admin-generation",
|
||||
clusterHarness.VolumeServerAddress(),
|
||||
volumeID,
|
||||
"",
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
)
|
||||
const adminGeneration int64 = 1_700_000_000_000_000_321
|
||||
task.encodeTsNs = adminGeneration
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
require.NoError(t, task.markReplicasReadonly(ctx))
|
||||
localFiles, err := task.copyVolumeFilesToWorker(ctx, t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
shardFiles, err := task.generateEcShardsLocally(localFiles, t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
vi, _, found, err := volume_info.MaybeLoadVolumeInfo(shardFiles["vif"])
|
||||
require.NoError(t, err)
|
||||
require.True(t, found)
|
||||
require.NotNil(t, vi.EcShardConfig)
|
||||
require.Equal(t, adminGeneration, vi.EcShardConfig.EncodeTsNs, "fenced run must stamp the admin generation verbatim, not a local timestamp")
|
||||
}
|
||||
|
||||
func compactVolumeOnce(t *testing.T, grpcClient volume_server_pb.VolumeServerClient, volumeID uint32) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
compactStream, err := grpcClient.VacuumVolumeCompact(ctx, &volume_server_pb.VacuumVolumeCompactRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
for {
|
||||
_, err = compactStream.Recv()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
_, err = grpcClient.VacuumVolumeCommit(ctx, &volume_server_pb.VacuumVolumeCommitRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = grpcClient.VacuumVolumeCleanup(ctx, &volume_server_pb.VacuumVolumeCleanupRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
43
weed/worker/tasks/erasure_coding/planning.go
Normal file
43
weed/worker/tasks/erasure_coding/planning.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// isStubReplica reports whether a regular replica's .dat is too small to hold
|
||||
// any data — at most a bare superblock. An interrupted encode or copy can
|
||||
// leave a 0-byte .dat, or write the 8-byte superblock and then fail; either
|
||||
// way the file is not an encode source and must not be counted toward
|
||||
// size/fullness checks.
|
||||
func isStubReplica(size uint64) bool {
|
||||
return size <= uint64(super_block.SuperBlockSize)
|
||||
}
|
||||
|
||||
// selectCanonicalMetric picks the metric that drives the encode checks and
|
||||
// names the encode source server. It prefers the lowest-server credible
|
||||
// replica — data-bearing and not already EC — so a 0-byte stub or a leftover
|
||||
// EC shard set sharing the volume id cannot become canonical and strand the
|
||||
// volume (a stub trips the min-size gate; an EC metric trips the IsECVolume
|
||||
// guard, hiding the volume from both orphan-source cleanup and re-encode).
|
||||
// When nothing is credible there is nothing to encode, so the lowest-server
|
||||
// metric is returned unchanged and the downstream gates make the skip
|
||||
// decision as before.
|
||||
func selectCanonicalMetric(group []*types.VolumeHealthMetrics) *types.VolumeHealthMetrics {
|
||||
var lowest, credible *types.VolumeHealthMetrics
|
||||
for _, m := range group {
|
||||
if lowest == nil || m.Server < lowest.Server {
|
||||
lowest = m
|
||||
}
|
||||
if m.IsECVolume || isStubReplica(m.Size) {
|
||||
continue
|
||||
}
|
||||
if credible == nil || m.Server < credible.Server {
|
||||
credible = m
|
||||
}
|
||||
}
|
||||
if credible != nil {
|
||||
return credible
|
||||
}
|
||||
return lowest
|
||||
}
|
||||
140
weed/worker/tasks/erasure_coding/planning_test.go
Normal file
140
weed/worker/tasks/erasure_coding/planning_test.go
Normal file
@@ -0,0 +1,140 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestIsStubReplica(t *testing.T) {
|
||||
assert.True(t, isStubReplica(0), "0-byte .dat is a stub")
|
||||
assert.True(t, isStubReplica(uint64(super_block.SuperBlockSize)-1), "below a superblock is a stub")
|
||||
assert.True(t, isStubReplica(uint64(super_block.SuperBlockSize)), "a bare superblock holds no data — a stub")
|
||||
assert.False(t, isStubReplica(uint64(super_block.SuperBlockSize)+1), "data beyond the superblock is a real replica")
|
||||
assert.False(t, isStubReplica(200*1024*1024), "a data-bearing replica is not a stub")
|
||||
}
|
||||
|
||||
// A 0-byte stub left by an interrupted encode often sorts to a lower server id
|
||||
// than the real replica. The old lowest-server canonical pick then reported
|
||||
// Size=0, tripped the min-size gate, and the volume was stranded in
|
||||
// skippedTooSmall forever. selectCanonicalMetric must skip the stub and return
|
||||
// the data-bearing replica.
|
||||
func TestSelectCanonicalMetricPrefersCredibleOverLowServerStub(t *testing.T) {
|
||||
stub := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.1:8080", Size: 0}
|
||||
real := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.4:8080", Size: 200 * 1024 * 1024}
|
||||
|
||||
got := selectCanonicalMetric([]*types.VolumeHealthMetrics{stub, real})
|
||||
require.Same(t, real, got)
|
||||
}
|
||||
|
||||
// An EC-side metric (the partial shards from a failed encode) can also sort
|
||||
// below the regular replica. Picking it would short-circuit at the
|
||||
// IsECVolume guard and skip the volume, hiding it from both the orphan-source
|
||||
// cleanup and the re-encode path. The credible regular replica must win.
|
||||
func TestSelectCanonicalMetricSkipsECMetrics(t *testing.T) {
|
||||
ecMetric := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.1:8080", Size: 50 * 1024 * 1024, IsECVolume: true}
|
||||
real := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.4:8080", Size: 200 * 1024 * 1024}
|
||||
|
||||
got := selectCanonicalMetric([]*types.VolumeHealthMetrics{ecMetric, real})
|
||||
require.Same(t, real, got)
|
||||
}
|
||||
|
||||
// When nothing is credible there is nothing to encode. Fall back to the
|
||||
// lowest-server metric so the downstream gates (min-size / IsECVolume) make
|
||||
// the skip decision exactly as before — selectCanonicalMetric must not invent
|
||||
// a source.
|
||||
func TestSelectCanonicalMetricAllStubsFallsBackToLowestServer(t *testing.T) {
|
||||
stubHi := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.4:8080", Size: 0}
|
||||
stubLo := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.1:8080", Size: 0}
|
||||
|
||||
got := selectCanonicalMetric([]*types.VolumeHealthMetrics{stubHi, stubLo})
|
||||
require.Same(t, stubLo, got)
|
||||
}
|
||||
|
||||
// Among several credible replicas the lowest server id still wins, preserving
|
||||
// the deterministic canonical choice the task-dedup logic relies on.
|
||||
func TestSelectCanonicalMetricTieBreaksByServerAmongCredible(t *testing.T) {
|
||||
hi := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.7:8080", Size: 200 * 1024 * 1024}
|
||||
lo := &types.VolumeHealthMetrics{VolumeID: 13, Server: "10.0.0.2:8080", Size: 200 * 1024 * 1024}
|
||||
|
||||
got := selectCanonicalMetric([]*types.VolumeHealthMetrics{hi, lo})
|
||||
require.Same(t, lo, got)
|
||||
}
|
||||
|
||||
func TestSelectCanonicalMetricEmpty(t *testing.T) {
|
||||
require.Nil(t, selectCanonicalMetric(nil))
|
||||
require.Nil(t, selectCanonicalMetric([]*types.VolumeHealthMetrics{}))
|
||||
}
|
||||
|
||||
// End-to-end regression for the stranded-volume bug: a volume whose lowest
|
||||
// server holds a 0-byte stub and whose real replica is on a higher server must
|
||||
// still be proposed for EC encoding, not silently dropped as too-small.
|
||||
func TestDetectionEncodesDespiteLowServerStub(t *testing.T) {
|
||||
const volumeID uint32 = 13
|
||||
activeTopology := buildStubReplicaTopology(t, volumeID)
|
||||
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
|
||||
|
||||
lastModified := time.Now().Add(-2 * time.Hour)
|
||||
metrics := []*types.VolumeHealthMetrics{
|
||||
// Stub on the lowest server: an interrupted encode left a 0-byte .dat.
|
||||
{VolumeID: volumeID, Server: "127.0.0.1:8080", Size: 0, FullnessRatio: 0, LastModified: lastModified, Age: time.Since(lastModified)},
|
||||
// The real replica on a higher server.
|
||||
{VolumeID: volumeID, Server: "127.0.0.1:8081", Size: 200 * 1024 * 1024, FullnessRatio: 0.96, LastModified: lastModified, Age: time.Since(lastModified)},
|
||||
}
|
||||
|
||||
results, _, err := Detection(context.Background(), metrics, clusterInfo, NewDefaultConfig(), 0)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, results, 1, "volume with a real replica must be proposed for EC despite a low-server stub")
|
||||
assert.Equal(t, types.TaskTypeErasureCoding, results[0].TaskType)
|
||||
assert.Equal(t, volumeID, results[0].VolumeID)
|
||||
}
|
||||
|
||||
// buildStubReplicaTopology builds a cluster with TotalShardsCount single-disk
|
||||
// nodes (enough targets to place every shard) where node 0 holds a 0-byte stub
|
||||
// replica of volumeID and node 1 holds the real replica.
|
||||
func buildStubReplicaTopology(t *testing.T, volumeID uint32) *topology.ActiveTopology {
|
||||
t.Helper()
|
||||
activeTopology := topology.NewActiveTopology(10)
|
||||
nodes := make([]*master_pb.DataNodeInfo, 0, erasure_coding.TotalShardsCount)
|
||||
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
|
||||
diskInfo := &master_pb.DiskInfo{
|
||||
DiskId: 0,
|
||||
MaxVolumeCount: 200,
|
||||
}
|
||||
switch i {
|
||||
case 0:
|
||||
diskInfo.VolumeInfos = []*master_pb.VolumeInformationMessage{{
|
||||
Id: volumeID, DiskId: 0, DiskType: "hdd", Size: 0,
|
||||
}}
|
||||
diskInfo.VolumeCount = 1
|
||||
case 1:
|
||||
diskInfo.VolumeInfos = []*master_pb.VolumeInformationMessage{{
|
||||
Id: volumeID, DiskId: 0, DiskType: "hdd", Size: 200 * 1024 * 1024,
|
||||
}}
|
||||
diskInfo.VolumeCount = 1
|
||||
}
|
||||
nodes = append(nodes, &master_pb.DataNodeInfo{
|
||||
Id: fmt.Sprintf("127.0.0.1:%d", 8080+i),
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{"hdd": diskInfo},
|
||||
})
|
||||
}
|
||||
require.NoError(t, activeTopology.UpdateTopology(&master_pb.TopologyInfo{
|
||||
DataCenterInfos: []*master_pb.DataCenterInfo{{
|
||||
Id: "dc1",
|
||||
RackInfos: []*master_pb.RackInfo{{
|
||||
Id: "rack1",
|
||||
DataNodeInfos: nodes,
|
||||
}},
|
||||
}},
|
||||
}))
|
||||
return activeTopology
|
||||
}
|
||||
878
weed/worker/tasks/erasure_coding/plugin_handler.go
Normal file
878
weed/worker/tasks/erasure_coding/plugin_handler.go
Normal file
@@ -0,0 +1,878 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
ecstorage "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
|
||||
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func init() {
|
||||
pluginworker.RegisterHandler(pluginworker.HandlerFactory{
|
||||
JobType: "erasure_coding",
|
||||
Category: pluginworker.CategoryHeavy,
|
||||
Aliases: []string{"erasure-coding", "erasure.coding", "ec"},
|
||||
Build: func(opts pluginworker.HandlerBuildOptions) (pluginworker.JobHandler, error) {
|
||||
return NewErasureCodingHandler(opts.GrpcDialOption, opts.WorkingDir), nil
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
type erasureCodingWorkerConfig struct {
|
||||
TaskConfig *Config
|
||||
}
|
||||
|
||||
// ErasureCodingHandler is the plugin job handler for erasure coding.
|
||||
type ErasureCodingHandler struct {
|
||||
grpcDialOption grpc.DialOption
|
||||
workingDir string
|
||||
}
|
||||
|
||||
func NewErasureCodingHandler(grpcDialOption grpc.DialOption, workingDir string) *ErasureCodingHandler {
|
||||
return &ErasureCodingHandler{grpcDialOption: grpcDialOption, workingDir: strings.TrimSpace(workingDir)}
|
||||
}
|
||||
|
||||
func (h *ErasureCodingHandler) Capability() *plugin_pb.JobTypeCapability {
|
||||
return &plugin_pb.JobTypeCapability{
|
||||
JobType: "erasure_coding",
|
||||
CanDetect: true,
|
||||
CanExecute: true,
|
||||
MaxDetectionConcurrency: 1,
|
||||
MaxExecutionConcurrency: 1,
|
||||
DisplayName: "EC Encoding",
|
||||
Description: "Converts full and quiet volumes into EC shards",
|
||||
Weight: 80,
|
||||
}
|
||||
}
|
||||
|
||||
func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
|
||||
return &plugin_pb.JobTypeDescriptor{
|
||||
JobType: "erasure_coding",
|
||||
DisplayName: "EC Encoding",
|
||||
Description: "Detect and execute erasure coding for suitable volumes",
|
||||
Icon: "fas fa-shield-alt",
|
||||
DescriptorVersion: 1,
|
||||
AdminConfigForm: &plugin_pb.ConfigForm{
|
||||
FormId: "erasure-coding-admin",
|
||||
Title: "Erasure Coding Admin Config",
|
||||
Description: "Admin-side controls for erasure coding detection scope.",
|
||||
Sections: []*plugin_pb.ConfigSection{
|
||||
{
|
||||
SectionId: "scope",
|
||||
Title: "Scope",
|
||||
Description: "Optional filters applied before erasure coding detection.",
|
||||
Fields: []*plugin_pb.ConfigField{
|
||||
{
|
||||
Name: "collection_filter",
|
||||
Label: "Collection Filter",
|
||||
Description: "Only erasure code volumes in matching collections. Comma-separated list of names, wildcards, or regex patterns.",
|
||||
Placeholder: "all collections",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
DefaultValues: map[string]*plugin_pb.ConfigValue{
|
||||
"collection_filter": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
|
||||
},
|
||||
},
|
||||
},
|
||||
WorkerConfigForm: &plugin_pb.ConfigForm{
|
||||
FormId: "erasure-coding-worker",
|
||||
Title: "Erasure Coding Worker Config",
|
||||
Description: "Worker-side detection thresholds.",
|
||||
Sections: []*plugin_pb.ConfigSection{
|
||||
{
|
||||
SectionId: "thresholds",
|
||||
Title: "Detection Thresholds",
|
||||
Description: "Controls for when erasure coding jobs should be proposed.",
|
||||
Fields: []*plugin_pb.ConfigField{
|
||||
{
|
||||
Name: "quiet_for_seconds",
|
||||
Label: "Quiet Period (s)",
|
||||
Description: "Volume must remain unmodified for at least this duration.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
Required: true,
|
||||
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
},
|
||||
{
|
||||
Name: "fullness_ratio",
|
||||
Label: "Fullness Ratio",
|
||||
Description: "Minimum volume fullness ratio to trigger erasure coding.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_DOUBLE,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
Required: true,
|
||||
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0}},
|
||||
MaxValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 1}},
|
||||
},
|
||||
{
|
||||
Name: "min_size_mb",
|
||||
Label: "Minimum Volume Size (MB)",
|
||||
Description: "Only volumes larger than this size are considered.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
Required: true,
|
||||
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}},
|
||||
},
|
||||
{
|
||||
Name: "preferred_tags",
|
||||
Label: "Preferred Tags",
|
||||
Description: "Comma-separated disk tags to prioritize for EC shard placement, ordered by preference.",
|
||||
Placeholder: "fast,ssd",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
|
||||
},
|
||||
{
|
||||
Name: "replica_placement",
|
||||
Label: "Replica Placement",
|
||||
Description: "EC shard placement (e.g. 020): 2nd/3rd digits cap shards per rack/node (best-effort during encode, enforced by rebalancing); the data-center digit is ignored. Empty uses the master default.",
|
||||
Placeholder: "020",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
DefaultValues: map[string]*plugin_pb.ConfigValue{
|
||||
"quiet_for_seconds": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 3600},
|
||||
},
|
||||
"fullness_ratio": {
|
||||
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.95},
|
||||
},
|
||||
"min_size_mb": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 30},
|
||||
},
|
||||
"preferred_tags": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
|
||||
},
|
||||
"replica_placement": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
|
||||
},
|
||||
},
|
||||
},
|
||||
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
|
||||
Enabled: true,
|
||||
DetectionIntervalMinutes: 17,
|
||||
DetectionTimeoutSeconds: 300,
|
||||
MaxJobsPerDetection: 500,
|
||||
GlobalExecutionConcurrency: 16,
|
||||
PerWorkerExecutionConcurrency: 4,
|
||||
RetryLimit: 1,
|
||||
RetryBackoffSeconds: 30,
|
||||
JobTypeMaxRuntimeSeconds: 1800,
|
||||
ExecutionTimeoutSeconds: 1800,
|
||||
},
|
||||
WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{
|
||||
"quiet_for_seconds": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 3600},
|
||||
},
|
||||
"fullness_ratio": {
|
||||
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.95},
|
||||
},
|
||||
"min_size_mb": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 30},
|
||||
},
|
||||
"preferred_tags": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (h *ErasureCodingHandler) Detect(
|
||||
ctx context.Context,
|
||||
request *plugin_pb.RunDetectionRequest,
|
||||
sender pluginworker.DetectionSender,
|
||||
) error {
|
||||
if request == nil {
|
||||
return fmt.Errorf("run detection request is nil")
|
||||
}
|
||||
if sender == nil {
|
||||
return fmt.Errorf("detection sender is nil")
|
||||
}
|
||||
if request.JobType != "" && request.JobType != "erasure_coding" {
|
||||
return fmt.Errorf("job type %q is not handled by erasure_coding worker", request.JobType)
|
||||
}
|
||||
|
||||
workerConfig := deriveErasureCodingWorkerConfig(request.GetWorkerConfigValues())
|
||||
|
||||
collectionFilter := strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "collection_filter", ""))
|
||||
if collectionFilter != "" {
|
||||
workerConfig.TaskConfig.CollectionFilter = collectionFilter
|
||||
}
|
||||
|
||||
masters := make([]string, 0)
|
||||
if request.ClusterContext != nil {
|
||||
masters = append(masters, request.ClusterContext.MasterGrpcAddresses...)
|
||||
}
|
||||
|
||||
metrics, activeTopology, err := h.collectVolumeMetrics(ctx, masters, collectionFilter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
clusterInfo := &workertypes.ClusterInfo{
|
||||
ActiveTopology: activeTopology,
|
||||
GrpcDialOption: h.grpcDialOption,
|
||||
DefaultReplicaPlacement: pluginworker.FetchDefaultReplicaPlacement(ctx, masters, h.grpcDialOption),
|
||||
}
|
||||
maxResults := int(request.MaxResults)
|
||||
if maxResults < 0 {
|
||||
maxResults = 0
|
||||
}
|
||||
results, hasMore, err := Detection(ctx, metrics, clusterInfo, workerConfig.TaskConfig, maxResults)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Stamp the admin-issued encode generation onto every EC proposal. DetectionSequence
|
||||
// is minted once per cycle on the single admin clock, so generations are globally
|
||||
// ordered even though detection runs on a rotating worker; this lets a stale worker's
|
||||
// shard cleanup fence against a newer run instead of wiping it.
|
||||
for _, result := range results {
|
||||
if result == nil || result.TypedParams == nil {
|
||||
continue
|
||||
}
|
||||
if ecp := result.TypedParams.GetErasureCodingParams(); ecp != nil {
|
||||
ecp.EncodeTsNs = request.DetectionSequence
|
||||
}
|
||||
}
|
||||
if traceErr := emitErasureCodingDetectionDecisionTrace(sender, metrics, workerConfig.TaskConfig, results, maxResults, hasMore); traceErr != nil {
|
||||
glog.Warningf("Plugin worker failed to emit erasure_coding detection trace: %v", traceErr)
|
||||
}
|
||||
|
||||
proposals := make([]*plugin_pb.JobProposal, 0, len(results))
|
||||
for _, result := range results {
|
||||
proposal, proposalErr := buildErasureCodingProposal(result, h.workingDir)
|
||||
if proposalErr != nil {
|
||||
glog.Warningf("Plugin worker skip invalid erasure_coding proposal: %v", proposalErr)
|
||||
continue
|
||||
}
|
||||
proposals = append(proposals, proposal)
|
||||
}
|
||||
|
||||
if err := sender.SendProposals(&plugin_pb.DetectionProposals{
|
||||
JobType: "erasure_coding",
|
||||
Proposals: proposals,
|
||||
HasMore: hasMore,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return sender.SendComplete(&plugin_pb.DetectionComplete{
|
||||
JobType: "erasure_coding",
|
||||
Success: true,
|
||||
TotalProposals: int32(len(proposals)),
|
||||
})
|
||||
}
|
||||
|
||||
func emitErasureCodingDetectionDecisionTrace(
|
||||
sender pluginworker.DetectionSender,
|
||||
metrics []*workertypes.VolumeHealthMetrics,
|
||||
taskConfig *Config,
|
||||
results []*workertypes.TaskDetectionResult,
|
||||
maxResults int,
|
||||
hasMore bool,
|
||||
) error {
|
||||
if sender == nil || taskConfig == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
quietThreshold := time.Duration(taskConfig.QuietForSeconds) * time.Second
|
||||
minSizeBytes := uint64(taskConfig.MinSizeMB) * 1024 * 1024
|
||||
allowedCollections, err := wildcard.CompileCollectionMatcher(taskConfig.CollectionFilter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
volumeGroups := make(map[uint32][]*workertypes.VolumeHealthMetrics)
|
||||
for _, metric := range metrics {
|
||||
if metric == nil {
|
||||
continue
|
||||
}
|
||||
volumeGroups[metric.VolumeID] = append(volumeGroups[metric.VolumeID], metric)
|
||||
}
|
||||
|
||||
skippedAlreadyEC := 0
|
||||
skippedTooSmall := 0
|
||||
skippedCollectionFilter := 0
|
||||
skippedQuietTime := 0
|
||||
skippedFullness := 0
|
||||
|
||||
for _, groupMetrics := range volumeGroups {
|
||||
if len(groupMetrics) == 0 {
|
||||
continue
|
||||
}
|
||||
metric := groupMetrics[0]
|
||||
for _, candidate := range groupMetrics {
|
||||
if candidate != nil && candidate.Server < metric.Server {
|
||||
metric = candidate
|
||||
}
|
||||
}
|
||||
if metric == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if metric.IsECVolume {
|
||||
skippedAlreadyEC++
|
||||
continue
|
||||
}
|
||||
if metric.Size < minSizeBytes {
|
||||
skippedTooSmall++
|
||||
continue
|
||||
}
|
||||
if !allowedCollections.Matches(metric.Collection) {
|
||||
skippedCollectionFilter++
|
||||
continue
|
||||
}
|
||||
if metric.Age < quietThreshold {
|
||||
skippedQuietTime++
|
||||
continue
|
||||
}
|
||||
if metric.FullnessRatio < taskConfig.FullnessRatio {
|
||||
skippedFullness++
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
totalVolumes := len(metrics)
|
||||
summarySuffix := ""
|
||||
if hasMore {
|
||||
summarySuffix = fmt.Sprintf(" (max_results=%d reached; remaining volumes not evaluated)", maxResults)
|
||||
}
|
||||
summaryMessage := ""
|
||||
if len(results) == 0 {
|
||||
summaryMessage = fmt.Sprintf(
|
||||
"EC detection: No tasks created for %d volumes%s (skipped: %d already EC, %d too small, %d filtered, %d not quiet, %d not full)",
|
||||
totalVolumes,
|
||||
summarySuffix,
|
||||
skippedAlreadyEC,
|
||||
skippedTooSmall,
|
||||
skippedCollectionFilter,
|
||||
skippedQuietTime,
|
||||
skippedFullness,
|
||||
)
|
||||
} else {
|
||||
summaryMessage = fmt.Sprintf(
|
||||
"EC detection: Created %d task(s)%s from %d volumes (skipped: %d already EC, %d too small, %d filtered, %d not quiet, %d not full)",
|
||||
len(results),
|
||||
summarySuffix,
|
||||
totalVolumes,
|
||||
skippedAlreadyEC,
|
||||
skippedTooSmall,
|
||||
skippedCollectionFilter,
|
||||
skippedQuietTime,
|
||||
skippedFullness,
|
||||
)
|
||||
}
|
||||
|
||||
if err := sender.SendActivity(pluginworker.BuildDetectorActivity("decision_summary", summaryMessage, map[string]*plugin_pb.ConfigValue{
|
||||
"total_volumes": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(totalVolumes)},
|
||||
},
|
||||
"selected_tasks": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(results))},
|
||||
},
|
||||
"max_results": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(maxResults)},
|
||||
},
|
||||
"has_more": {
|
||||
Kind: &plugin_pb.ConfigValue_BoolValue{BoolValue: hasMore},
|
||||
},
|
||||
"skipped_already_ec": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedAlreadyEC)},
|
||||
},
|
||||
"skipped_too_small": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedTooSmall)},
|
||||
},
|
||||
"skipped_filtered": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedCollectionFilter)},
|
||||
},
|
||||
"skipped_not_quiet": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedQuietTime)},
|
||||
},
|
||||
"skipped_not_full": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedFullness)},
|
||||
},
|
||||
"quiet_for_seconds": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.QuietForSeconds)},
|
||||
},
|
||||
"min_size_mb": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.MinSizeMB)},
|
||||
},
|
||||
"fullness_threshold_percent": {
|
||||
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: taskConfig.FullnessRatio * 100},
|
||||
},
|
||||
})); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
detailsEmitted := 0
|
||||
for _, metric := range metrics {
|
||||
if metric == nil || metric.IsECVolume {
|
||||
continue
|
||||
}
|
||||
sizeMB := float64(metric.Size) / (1024 * 1024)
|
||||
message := fmt.Sprintf(
|
||||
"ERASURE CODING: Volume %d: size=%.1fMB (need ≥%dMB), age=%s (need ≥%s), fullness=%.1f%% (need ≥%.1f%%)",
|
||||
metric.VolumeID,
|
||||
sizeMB,
|
||||
taskConfig.MinSizeMB,
|
||||
metric.Age.Truncate(time.Minute),
|
||||
quietThreshold.Truncate(time.Minute),
|
||||
metric.FullnessRatio*100,
|
||||
taskConfig.FullnessRatio*100,
|
||||
)
|
||||
if err := sender.SendActivity(pluginworker.BuildDetectorActivity("decision_volume", message, map[string]*plugin_pb.ConfigValue{
|
||||
"volume_id": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(metric.VolumeID)},
|
||||
},
|
||||
"size_mb": {
|
||||
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: sizeMB},
|
||||
},
|
||||
"required_min_size_mb": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.MinSizeMB)},
|
||||
},
|
||||
"age_seconds": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(metric.Age.Seconds())},
|
||||
},
|
||||
"required_quiet_for_seconds": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(taskConfig.QuietForSeconds)},
|
||||
},
|
||||
"fullness_percent": {
|
||||
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: metric.FullnessRatio * 100},
|
||||
},
|
||||
"required_fullness_percent": {
|
||||
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: taskConfig.FullnessRatio * 100},
|
||||
},
|
||||
})); err != nil {
|
||||
return err
|
||||
}
|
||||
detailsEmitted++
|
||||
if detailsEmitted >= 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *ErasureCodingHandler) Execute(
|
||||
ctx context.Context,
|
||||
request *plugin_pb.ExecuteJobRequest,
|
||||
sender pluginworker.ExecutionSender,
|
||||
) error {
|
||||
if request == nil || request.Job == nil {
|
||||
return fmt.Errorf("execute request/job is nil")
|
||||
}
|
||||
if sender == nil {
|
||||
return fmt.Errorf("execution sender is nil")
|
||||
}
|
||||
if request.Job.JobType != "" && request.Job.JobType != "erasure_coding" {
|
||||
return fmt.Errorf("job type %q is not handled by erasure_coding worker", request.Job.JobType)
|
||||
}
|
||||
|
||||
params, err := decodeErasureCodingTaskParams(request.Job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
applyErasureCodingExecutionDefaults(params, request.GetClusterContext(), h.workingDir)
|
||||
|
||||
if len(params.Sources) == 0 || strings.TrimSpace(params.Sources[0].Node) == "" {
|
||||
return fmt.Errorf("erasure coding source node is required")
|
||||
}
|
||||
if len(params.Targets) == 0 {
|
||||
return fmt.Errorf("erasure coding targets are required")
|
||||
}
|
||||
|
||||
task := NewErasureCodingTask(
|
||||
request.Job.JobId,
|
||||
params.Sources[0].Node,
|
||||
params.VolumeId,
|
||||
params.Collection,
|
||||
h.grpcDialOption,
|
||||
)
|
||||
execCtx, execCancel := context.WithCancel(ctx)
|
||||
defer execCancel()
|
||||
task.SetProgressCallback(func(progress float64, stage string) {
|
||||
message := fmt.Sprintf("erasure coding progress %.0f%%", progress)
|
||||
if strings.TrimSpace(stage) != "" {
|
||||
message = stage
|
||||
}
|
||||
if err := sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: request.Job.JobId,
|
||||
JobType: request.Job.JobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_RUNNING,
|
||||
ProgressPercent: progress,
|
||||
Stage: stage,
|
||||
Message: message,
|
||||
Activities: []*plugin_pb.ActivityEvent{
|
||||
pluginworker.BuildExecutorActivity(stage, message),
|
||||
},
|
||||
}); err != nil {
|
||||
execCancel()
|
||||
}
|
||||
})
|
||||
|
||||
if err := sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: request.Job.JobId,
|
||||
JobType: request.Job.JobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_ASSIGNED,
|
||||
ProgressPercent: 0,
|
||||
Stage: "assigned",
|
||||
Message: "erasure coding job accepted",
|
||||
Activities: []*plugin_pb.ActivityEvent{
|
||||
pluginworker.BuildExecutorActivity("assigned", "erasure coding job accepted"),
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := task.Execute(execCtx, params); err != nil {
|
||||
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: request.Job.JobId,
|
||||
JobType: request.Job.JobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_FAILED,
|
||||
ProgressPercent: 100,
|
||||
Stage: "failed",
|
||||
Message: err.Error(),
|
||||
Activities: []*plugin_pb.ActivityEvent{
|
||||
pluginworker.BuildExecutorActivity("failed", err.Error()),
|
||||
},
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
sourceNode := params.Sources[0].Node
|
||||
resultSummary := fmt.Sprintf("erasure coding completed for volume %d across %d targets", params.VolumeId, len(params.Targets))
|
||||
|
||||
return sender.SendCompleted(&plugin_pb.JobCompleted{
|
||||
JobId: request.Job.JobId,
|
||||
JobType: request.Job.JobType,
|
||||
Success: true,
|
||||
Result: &plugin_pb.JobResult{
|
||||
Summary: resultSummary,
|
||||
OutputValues: map[string]*plugin_pb.ConfigValue{
|
||||
"volume_id": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(params.VolumeId)},
|
||||
},
|
||||
"source_server": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceNode},
|
||||
},
|
||||
"target_count": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(params.Targets))},
|
||||
},
|
||||
},
|
||||
},
|
||||
Activities: []*plugin_pb.ActivityEvent{
|
||||
pluginworker.BuildExecutorActivity("completed", resultSummary),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (h *ErasureCodingHandler) collectVolumeMetrics(
|
||||
ctx context.Context,
|
||||
masterAddresses []string,
|
||||
collectionFilter string,
|
||||
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, error) {
|
||||
metrics, activeTopology, _, err := pluginworker.CollectVolumeMetricsFromMasters(ctx, masterAddresses, collectionFilter, h.grpcDialOption)
|
||||
return metrics, activeTopology, err
|
||||
}
|
||||
|
||||
func deriveErasureCodingWorkerConfig(values map[string]*plugin_pb.ConfigValue) *erasureCodingWorkerConfig {
|
||||
taskConfig := NewDefaultConfig()
|
||||
|
||||
quietForSeconds := pluginworker.ReadIntConfig(values, "quiet_for_seconds", taskConfig.QuietForSeconds)
|
||||
if quietForSeconds < 0 {
|
||||
quietForSeconds = 0
|
||||
}
|
||||
taskConfig.QuietForSeconds = quietForSeconds
|
||||
|
||||
fullnessRatio := pluginworker.ReadDoubleConfig(values, "fullness_ratio", taskConfig.FullnessRatio)
|
||||
if fullnessRatio < 0 {
|
||||
fullnessRatio = 0
|
||||
}
|
||||
if fullnessRatio > 1 {
|
||||
fullnessRatio = 1
|
||||
}
|
||||
taskConfig.FullnessRatio = fullnessRatio
|
||||
|
||||
minSizeMB := pluginworker.ReadIntConfig(values, "min_size_mb", taskConfig.MinSizeMB)
|
||||
if minSizeMB < 1 {
|
||||
minSizeMB = 1
|
||||
}
|
||||
taskConfig.MinSizeMB = minSizeMB
|
||||
|
||||
taskConfig.PreferredTags = util.NormalizeTagList(pluginworker.ReadStringListConfig(values, "preferred_tags"))
|
||||
|
||||
taskConfig.ReplicaPlacement = strings.TrimSpace(pluginworker.ReadStringConfig(values, "replica_placement", taskConfig.ReplicaPlacement))
|
||||
|
||||
return &erasureCodingWorkerConfig{
|
||||
TaskConfig: taskConfig,
|
||||
}
|
||||
}
|
||||
|
||||
func buildErasureCodingProposal(
|
||||
result *workertypes.TaskDetectionResult,
|
||||
baseWorkingDir string,
|
||||
) (*plugin_pb.JobProposal, error) {
|
||||
if result == nil {
|
||||
return nil, fmt.Errorf("task detection result is nil")
|
||||
}
|
||||
if result.TypedParams == nil {
|
||||
return nil, fmt.Errorf("missing typed params for volume %d", result.VolumeID)
|
||||
}
|
||||
params := proto.Clone(result.TypedParams).(*worker_pb.TaskParams)
|
||||
applyErasureCodingExecutionDefaults(params, nil, baseWorkingDir)
|
||||
|
||||
paramsPayload, err := proto.Marshal(params)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal task params: %w", err)
|
||||
}
|
||||
|
||||
proposalID := strings.TrimSpace(result.TaskID)
|
||||
if proposalID == "" {
|
||||
proposalID = fmt.Sprintf("erasure-coding-%d-%d", result.VolumeID, time.Now().UnixNano())
|
||||
}
|
||||
|
||||
dedupeKey := fmt.Sprintf("erasure_coding:%d", result.VolumeID)
|
||||
if result.Collection != "" {
|
||||
dedupeKey += ":" + result.Collection
|
||||
}
|
||||
|
||||
sourceNode := ""
|
||||
if len(params.Sources) > 0 {
|
||||
sourceNode = strings.TrimSpace(params.Sources[0].Node)
|
||||
}
|
||||
|
||||
summary := fmt.Sprintf("Erasure code volume %d", result.VolumeID)
|
||||
if sourceNode != "" {
|
||||
summary = fmt.Sprintf("Erasure code volume %d from %s", result.VolumeID, sourceNode)
|
||||
}
|
||||
|
||||
// EC encoding reads the full volume, computes shards, and writes 14
|
||||
// shards out to target nodes. Budget 10 min/GB (roughly 2x a plain copy)
|
||||
// so the scheduler grants a deadline scaled to volume size.
|
||||
volumeSizeGB := int64(result.TypedParams.VolumeSize/1024/1024/1024) + 1
|
||||
estimatedRuntimeSeconds := volumeSizeGB * 10 * 60
|
||||
|
||||
return &plugin_pb.JobProposal{
|
||||
ProposalId: proposalID,
|
||||
DedupeKey: dedupeKey,
|
||||
JobType: "erasure_coding",
|
||||
Priority: pluginworker.MapTaskPriority(result.Priority),
|
||||
Summary: summary,
|
||||
Detail: strings.TrimSpace(result.Reason),
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"task_params_pb": {
|
||||
Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: paramsPayload},
|
||||
},
|
||||
"volume_id": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(result.VolumeID)},
|
||||
},
|
||||
"source_server": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: sourceNode},
|
||||
},
|
||||
"collection": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Collection},
|
||||
},
|
||||
"target_count": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(params.Targets))},
|
||||
},
|
||||
"estimated_runtime_seconds": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: estimatedRuntimeSeconds},
|
||||
},
|
||||
},
|
||||
Labels: map[string]string{
|
||||
"task_type": "erasure_coding",
|
||||
"volume_id": fmt.Sprintf("%d", result.VolumeID),
|
||||
"collection": result.Collection,
|
||||
"source_node": sourceNode,
|
||||
"target_count": fmt.Sprintf("%d", len(params.Targets)),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func decodeErasureCodingTaskParams(job *plugin_pb.JobSpec) (*worker_pb.TaskParams, error) {
|
||||
if job == nil {
|
||||
return nil, fmt.Errorf("job spec is nil")
|
||||
}
|
||||
|
||||
if payload := pluginworker.ReadBytesConfig(job.Parameters, "task_params_pb"); len(payload) > 0 {
|
||||
params := &worker_pb.TaskParams{}
|
||||
if err := proto.Unmarshal(payload, params); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal task_params_pb: %w", err)
|
||||
}
|
||||
if params.TaskId == "" {
|
||||
params.TaskId = job.JobId
|
||||
}
|
||||
return params, nil
|
||||
}
|
||||
|
||||
volumeID := pluginworker.ReadUint32Config(job.Parameters, "volume_id", 0)
|
||||
sourceNode := strings.TrimSpace(pluginworker.ReadStringConfig(job.Parameters, "source_server", ""))
|
||||
if sourceNode == "" {
|
||||
sourceNode = strings.TrimSpace(pluginworker.ReadStringConfig(job.Parameters, "server", ""))
|
||||
}
|
||||
targetServers := pluginworker.ReadStringListConfig(job.Parameters, "target_servers")
|
||||
if len(targetServers) == 0 {
|
||||
targetServers = pluginworker.ReadStringListConfig(job.Parameters, "targets")
|
||||
}
|
||||
collection := pluginworker.ReadStringConfig(job.Parameters, "collection", "")
|
||||
|
||||
dataShards := pluginworker.ReadInt32Config(job.Parameters, "data_shards", int32(ecstorage.DataShardsCount))
|
||||
if dataShards <= 0 {
|
||||
dataShards = int32(ecstorage.DataShardsCount)
|
||||
}
|
||||
parityShards := pluginworker.ReadInt32Config(job.Parameters, "parity_shards", int32(ecstorage.ParityShardsCount))
|
||||
if parityShards <= 0 {
|
||||
parityShards = int32(ecstorage.ParityShardsCount)
|
||||
}
|
||||
sourceDiskType := strings.TrimSpace(pluginworker.ReadStringConfig(job.Parameters, "source_disk_type", ""))
|
||||
totalShards := int(dataShards + parityShards)
|
||||
|
||||
if volumeID == 0 {
|
||||
return nil, fmt.Errorf("missing volume_id in job parameters")
|
||||
}
|
||||
if sourceNode == "" {
|
||||
return nil, fmt.Errorf("missing source_server in job parameters")
|
||||
}
|
||||
if len(targetServers) == 0 {
|
||||
return nil, fmt.Errorf("missing target_servers in job parameters")
|
||||
}
|
||||
if len(targetServers) < totalShards {
|
||||
return nil, fmt.Errorf("insufficient target_servers: got %d, need at least %d", len(targetServers), totalShards)
|
||||
}
|
||||
|
||||
shardAssignments := assignECShardIDs(totalShards, len(targetServers))
|
||||
targets := make([]*worker_pb.TaskTarget, 0, len(targetServers))
|
||||
for i := 0; i < len(targetServers); i++ {
|
||||
targetNode := strings.TrimSpace(targetServers[i])
|
||||
if targetNode == "" {
|
||||
continue
|
||||
}
|
||||
targets = append(targets, &worker_pb.TaskTarget{
|
||||
Node: targetNode,
|
||||
VolumeId: volumeID,
|
||||
ShardIds: shardAssignments[i],
|
||||
})
|
||||
}
|
||||
if len(targets) < totalShards {
|
||||
return nil, fmt.Errorf("insufficient non-empty target_servers after normalization: got %d, need at least %d", len(targets), totalShards)
|
||||
}
|
||||
|
||||
return &worker_pb.TaskParams{
|
||||
TaskId: job.JobId,
|
||||
VolumeId: volumeID,
|
||||
Collection: collection,
|
||||
Sources: []*worker_pb.TaskSource{
|
||||
{
|
||||
Node: sourceNode,
|
||||
VolumeId: volumeID,
|
||||
},
|
||||
},
|
||||
Targets: targets,
|
||||
TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
|
||||
ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
|
||||
DataShards: dataShards,
|
||||
ParityShards: parityShards,
|
||||
SourceDiskType: sourceDiskType,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func applyErasureCodingExecutionDefaults(
|
||||
params *worker_pb.TaskParams,
|
||||
clusterContext *plugin_pb.ClusterContext,
|
||||
baseWorkingDir string,
|
||||
) {
|
||||
if params == nil {
|
||||
return
|
||||
}
|
||||
|
||||
ecParams := params.GetErasureCodingParams()
|
||||
if ecParams == nil {
|
||||
ecParams = &worker_pb.ErasureCodingTaskParams{
|
||||
DataShards: ecstorage.DataShardsCount,
|
||||
ParityShards: ecstorage.ParityShardsCount,
|
||||
}
|
||||
params.TaskParams = &worker_pb.TaskParams_ErasureCodingParams{ErasureCodingParams: ecParams}
|
||||
}
|
||||
|
||||
if ecParams.DataShards <= 0 {
|
||||
ecParams.DataShards = ecstorage.DataShardsCount
|
||||
}
|
||||
if ecParams.ParityShards <= 0 {
|
||||
ecParams.ParityShards = ecstorage.ParityShardsCount
|
||||
}
|
||||
ecParams.WorkingDir = defaultErasureCodingWorkingDir(baseWorkingDir)
|
||||
ecParams.CleanupSource = true
|
||||
if strings.TrimSpace(ecParams.MasterClient) == "" && clusterContext != nil && len(clusterContext.MasterGrpcAddresses) > 0 {
|
||||
ecParams.MasterClient = clusterContext.MasterGrpcAddresses[0]
|
||||
}
|
||||
|
||||
totalShards := int(ecParams.DataShards + ecParams.ParityShards)
|
||||
if totalShards <= 0 {
|
||||
totalShards = ecstorage.TotalShardsCount
|
||||
}
|
||||
needsShardAssignment := false
|
||||
for _, target := range params.Targets {
|
||||
if target == nil || len(target.ShardIds) == 0 {
|
||||
needsShardAssignment = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if needsShardAssignment && len(params.Targets) > 0 {
|
||||
assignments := assignECShardIDs(totalShards, len(params.Targets))
|
||||
for i := 0; i < len(params.Targets); i++ {
|
||||
if params.Targets[i] == nil {
|
||||
continue
|
||||
}
|
||||
if len(params.Targets[i].ShardIds) == 0 {
|
||||
params.Targets[i].ShardIds = assignments[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assignECShardIDs(totalShards int, targetCount int) [][]uint32 {
|
||||
if targetCount <= 0 {
|
||||
return nil
|
||||
}
|
||||
if totalShards <= 0 {
|
||||
totalShards = ecstorage.TotalShardsCount
|
||||
}
|
||||
|
||||
assignments := make([][]uint32, targetCount)
|
||||
for i := 0; i < totalShards; i++ {
|
||||
targetIndex := i % targetCount
|
||||
assignments[targetIndex] = append(assignments[targetIndex], uint32(i))
|
||||
}
|
||||
return assignments
|
||||
}
|
||||
|
||||
func defaultErasureCodingWorkingDir(baseWorkingDir string) string {
|
||||
dir := strings.TrimSpace(baseWorkingDir)
|
||||
if dir == "" {
|
||||
return filepath.Join(".", "erasure_coding")
|
||||
}
|
||||
return filepath.Join(dir, "erasure_coding")
|
||||
}
|
||||
343
weed/worker/tasks/erasure_coding/plugin_handler_test.go
Normal file
343
weed/worker/tasks/erasure_coding/plugin_handler_test.go
Normal file
@@ -0,0 +1,343 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
ecstorage "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func TestDecodeErasureCodingTaskParamsFromPayload(t *testing.T) {
|
||||
expected := &worker_pb.TaskParams{
|
||||
TaskId: "task-ec-1",
|
||||
VolumeId: 88,
|
||||
Collection: "images",
|
||||
Sources: []*worker_pb.TaskSource{
|
||||
{
|
||||
Node: "10.0.0.1:8080",
|
||||
VolumeId: 88,
|
||||
},
|
||||
},
|
||||
Targets: []*worker_pb.TaskTarget{
|
||||
{
|
||||
Node: "10.0.0.2:8080",
|
||||
VolumeId: 88,
|
||||
ShardIds: []uint32{0, 10},
|
||||
},
|
||||
},
|
||||
TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
|
||||
ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
|
||||
DataShards: ecstorage.DataShardsCount,
|
||||
ParityShards: ecstorage.ParityShardsCount,
|
||||
WorkingDir: "/tmp/ec-work",
|
||||
CleanupSource: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
payload, err := proto.Marshal(expected)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal payload: %v", err)
|
||||
}
|
||||
|
||||
job := &plugin_pb.JobSpec{
|
||||
JobId: "job-from-admin",
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
|
||||
},
|
||||
}
|
||||
|
||||
actual, err := decodeErasureCodingTaskParams(job)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeErasureCodingTaskParams() err = %v", err)
|
||||
}
|
||||
if !proto.Equal(expected, actual) {
|
||||
t.Fatalf("decoded params mismatch\nexpected: %+v\nactual: %+v", expected, actual)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeErasureCodingTaskParamsFallback(t *testing.T) {
|
||||
targetServers := make([]string, 0, ecstorage.TotalShardsCount)
|
||||
for i := 0; i < ecstorage.TotalShardsCount; i++ {
|
||||
targetServers = append(targetServers, "10.0.0."+string(rune('a'+i))+":8080")
|
||||
}
|
||||
|
||||
job := &plugin_pb.JobSpec{
|
||||
JobId: "job-ec-2",
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"volume_id": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7},
|
||||
},
|
||||
"source_server": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "127.0.0.1:8080"},
|
||||
},
|
||||
"target_servers": {
|
||||
Kind: &plugin_pb.ConfigValue_StringList{
|
||||
StringList: &plugin_pb.StringList{Values: targetServers},
|
||||
},
|
||||
},
|
||||
"collection": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "videos"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
params, err := decodeErasureCodingTaskParams(job)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeErasureCodingTaskParams() err = %v", err)
|
||||
}
|
||||
if params.TaskId != "job-ec-2" || params.VolumeId != 7 || params.Collection != "videos" {
|
||||
t.Fatalf("unexpected basic params: %+v", params)
|
||||
}
|
||||
if len(params.Sources) != 1 || params.Sources[0].Node != "127.0.0.1:8080" {
|
||||
t.Fatalf("unexpected sources: %+v", params.Sources)
|
||||
}
|
||||
if len(params.Targets) != ecstorage.TotalShardsCount {
|
||||
t.Fatalf("unexpected target count: %d", len(params.Targets))
|
||||
}
|
||||
if params.GetErasureCodingParams() == nil {
|
||||
t.Fatalf("expected fallback erasure coding params")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveErasureCodingWorkerConfig(t *testing.T) {
|
||||
values := map[string]*plugin_pb.ConfigValue{
|
||||
"quiet_for_seconds": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 720},
|
||||
},
|
||||
"fullness_ratio": {
|
||||
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.92},
|
||||
},
|
||||
"min_size_mb": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 128},
|
||||
},
|
||||
}
|
||||
|
||||
cfg := deriveErasureCodingWorkerConfig(values)
|
||||
if cfg.TaskConfig.QuietForSeconds != 720 {
|
||||
t.Fatalf("expected quiet_for_seconds 720, got %d", cfg.TaskConfig.QuietForSeconds)
|
||||
}
|
||||
if cfg.TaskConfig.FullnessRatio != 0.92 {
|
||||
t.Fatalf("expected fullness_ratio 0.92, got %v", cfg.TaskConfig.FullnessRatio)
|
||||
}
|
||||
if cfg.TaskConfig.MinSizeMB != 128 {
|
||||
t.Fatalf("expected min_size_mb 128, got %d", cfg.TaskConfig.MinSizeMB)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildErasureCodingProposal(t *testing.T) {
|
||||
params := &worker_pb.TaskParams{
|
||||
TaskId: "ec-task-1",
|
||||
VolumeId: 99,
|
||||
Collection: "c1",
|
||||
Sources: []*worker_pb.TaskSource{
|
||||
{
|
||||
Node: "source-a:8080",
|
||||
VolumeId: 99,
|
||||
},
|
||||
},
|
||||
Targets: []*worker_pb.TaskTarget{
|
||||
{
|
||||
Node: "target-a:8080",
|
||||
VolumeId: 99,
|
||||
ShardIds: []uint32{0, 10},
|
||||
},
|
||||
{
|
||||
Node: "target-b:8080",
|
||||
VolumeId: 99,
|
||||
ShardIds: []uint32{1, 11},
|
||||
},
|
||||
},
|
||||
TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
|
||||
ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
|
||||
DataShards: ecstorage.DataShardsCount,
|
||||
ParityShards: ecstorage.ParityShardsCount,
|
||||
},
|
||||
},
|
||||
}
|
||||
result := &workertypes.TaskDetectionResult{
|
||||
TaskID: "ec-task-1",
|
||||
TaskType: workertypes.TaskTypeErasureCoding,
|
||||
VolumeID: 99,
|
||||
Server: "source-a",
|
||||
Collection: "c1",
|
||||
Priority: workertypes.TaskPriorityLow,
|
||||
Reason: "quiet and full",
|
||||
TypedParams: params,
|
||||
}
|
||||
|
||||
proposal, err := buildErasureCodingProposal(result, "")
|
||||
if err != nil {
|
||||
t.Fatalf("buildErasureCodingProposal() err = %v", err)
|
||||
}
|
||||
if proposal.JobType != "erasure_coding" {
|
||||
t.Fatalf("unexpected job type %q", proposal.JobType)
|
||||
}
|
||||
if proposal.Parameters["task_params_pb"] == nil {
|
||||
t.Fatalf("expected serialized task params")
|
||||
}
|
||||
if proposal.Labels["source_node"] != "source-a:8080" {
|
||||
t.Fatalf("unexpected source label %q", proposal.Labels["source_node"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestErasureCodingHandlerRejectsUnsupportedJobType(t *testing.T) {
|
||||
handler := NewErasureCodingHandler(nil, "")
|
||||
err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
|
||||
JobType: "vacuum",
|
||||
}, noopDetectionSender{})
|
||||
if err == nil {
|
||||
t.Fatalf("expected detect job type mismatch error")
|
||||
}
|
||||
|
||||
err = handler.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
|
||||
Job: &plugin_pb.JobSpec{JobId: "job-1", JobType: "vacuum"},
|
||||
}, noopExecutionSender{})
|
||||
if err == nil {
|
||||
t.Fatalf("expected execute job type mismatch error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitErasureCodingDetectionDecisionTraceNoTasks(t *testing.T) {
|
||||
sender := &recordingDetectionSender{}
|
||||
config := NewDefaultConfig()
|
||||
config.QuietForSeconds = 5 * 60
|
||||
config.MinSizeMB = 30
|
||||
config.FullnessRatio = 0.91
|
||||
|
||||
metrics := []*workertypes.VolumeHealthMetrics{
|
||||
{
|
||||
VolumeID: 20,
|
||||
Size: 0,
|
||||
Age: 218*time.Hour + 41*time.Minute,
|
||||
FullnessRatio: 0,
|
||||
},
|
||||
{
|
||||
VolumeID: 27,
|
||||
Size: uint64(16 * 1024 * 1024 / 10),
|
||||
Age: 91*time.Hour + time.Minute,
|
||||
FullnessRatio: 0.002,
|
||||
},
|
||||
{
|
||||
VolumeID: 12,
|
||||
Size: 0,
|
||||
Age: 219*time.Hour + 49*time.Minute,
|
||||
FullnessRatio: 0,
|
||||
},
|
||||
}
|
||||
|
||||
if err := emitErasureCodingDetectionDecisionTrace(sender, metrics, config, nil, 0, false); err != nil {
|
||||
t.Fatalf("emitErasureCodingDetectionDecisionTrace error: %v", err)
|
||||
}
|
||||
if len(sender.events) < 4 {
|
||||
t.Fatalf("expected at least 4 detection events, got %d", len(sender.events))
|
||||
}
|
||||
|
||||
if sender.events[0].Source != plugin_pb.ActivitySource_ACTIVITY_SOURCE_DETECTOR {
|
||||
t.Fatalf("expected detector source, got %v", sender.events[0].Source)
|
||||
}
|
||||
if !strings.Contains(sender.events[0].Message, "EC detection: No tasks created for 3 volumes") {
|
||||
t.Fatalf("unexpected summary message: %q", sender.events[0].Message)
|
||||
}
|
||||
if !strings.Contains(sender.events[1].Message, "ERASURE CODING: Volume 20: size=0.0MB") {
|
||||
t.Fatalf("unexpected first detail message: %q", sender.events[1].Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestErasureCodingDescriptorOmitsLocalExecutionFields(t *testing.T) {
|
||||
descriptor := NewErasureCodingHandler(nil, "").Descriptor()
|
||||
if descriptor == nil || descriptor.WorkerConfigForm == nil {
|
||||
t.Fatalf("expected worker config form in descriptor")
|
||||
}
|
||||
if workerConfigFormHasField(descriptor.WorkerConfigForm, "working_dir") {
|
||||
t.Fatalf("unexpected working_dir in erasure coding worker config form")
|
||||
}
|
||||
if workerConfigFormHasField(descriptor.WorkerConfigForm, "cleanup_source") {
|
||||
t.Fatalf("unexpected cleanup_source in erasure coding worker config form")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyErasureCodingExecutionDefaultsForcesLocalFields(t *testing.T) {
|
||||
baseWorkingDir := "/var/lib/seaweedfs-worker"
|
||||
params := &worker_pb.TaskParams{
|
||||
TaskId: "ec-test",
|
||||
VolumeId: 100,
|
||||
TaskParams: &worker_pb.TaskParams_ErasureCodingParams{
|
||||
ErasureCodingParams: &worker_pb.ErasureCodingTaskParams{
|
||||
DataShards: ecstorage.DataShardsCount,
|
||||
ParityShards: ecstorage.ParityShardsCount,
|
||||
WorkingDir: "/tmp/custom-from-job",
|
||||
CleanupSource: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
applyErasureCodingExecutionDefaults(params, nil, baseWorkingDir)
|
||||
|
||||
ecParams := params.GetErasureCodingParams()
|
||||
if ecParams == nil {
|
||||
t.Fatalf("expected erasure coding params")
|
||||
}
|
||||
if ecParams.WorkingDir != defaultErasureCodingWorkingDir(baseWorkingDir) {
|
||||
t.Fatalf("expected local working_dir %q, got %q", defaultErasureCodingWorkingDir(baseWorkingDir), ecParams.WorkingDir)
|
||||
}
|
||||
if !ecParams.CleanupSource {
|
||||
t.Fatalf("expected cleanup_source true")
|
||||
}
|
||||
}
|
||||
|
||||
type noopDetectionSender struct{}
|
||||
|
||||
func (noopDetectionSender) SendProposals(*plugin_pb.DetectionProposals) error { return nil }
|
||||
func (noopDetectionSender) SendComplete(*plugin_pb.DetectionComplete) error { return nil }
|
||||
func (noopDetectionSender) SendActivity(*plugin_pb.ActivityEvent) error { return nil }
|
||||
|
||||
type noopExecutionSender struct{}
|
||||
|
||||
func (noopExecutionSender) SendProgress(*plugin_pb.JobProgressUpdate) error { return nil }
|
||||
func (noopExecutionSender) SendCompleted(*plugin_pb.JobCompleted) error { return nil }
|
||||
|
||||
type recordingDetectionSender struct {
|
||||
proposals *plugin_pb.DetectionProposals
|
||||
complete *plugin_pb.DetectionComplete
|
||||
events []*plugin_pb.ActivityEvent
|
||||
}
|
||||
|
||||
func (r *recordingDetectionSender) SendProposals(proposals *plugin_pb.DetectionProposals) error {
|
||||
r.proposals = proposals
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recordingDetectionSender) SendComplete(complete *plugin_pb.DetectionComplete) error {
|
||||
r.complete = complete
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recordingDetectionSender) SendActivity(event *plugin_pb.ActivityEvent) error {
|
||||
if event != nil {
|
||||
r.events = append(r.events, event)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func workerConfigFormHasField(form *plugin_pb.ConfigForm, fieldName string) bool {
|
||||
if form == nil {
|
||||
return false
|
||||
}
|
||||
for _, section := range form.Sections {
|
||||
if section == nil {
|
||||
continue
|
||||
}
|
||||
for _, field := range section.Fields {
|
||||
if field != nil && field.Name == fieldName {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
96
weed/worker/tasks/erasure_coding/register.go
Normal file
96
weed/worker/tasks/erasure_coding/register.go
Normal file
@@ -0,0 +1,96 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// Global variable to hold the task definition for configuration updates
|
||||
var globalTaskDef *base.TaskDefinition
|
||||
|
||||
// Auto-register this task when the package is imported
|
||||
func init() {
|
||||
RegisterErasureCodingTask()
|
||||
|
||||
// Register config updater
|
||||
tasks.AutoRegisterConfigUpdater(types.TaskTypeErasureCoding, UpdateConfigFromPersistence)
|
||||
}
|
||||
|
||||
// RegisterErasureCodingTask registers the erasure coding task with the new architecture
|
||||
func RegisterErasureCodingTask() {
|
||||
// Create configuration instance
|
||||
config := NewDefaultConfig()
|
||||
|
||||
// Create shared gRPC dial option using TLS configuration
|
||||
dialOpt := security.LoadClientTLS(util.GetViper(), "grpc.worker")
|
||||
|
||||
// Create complete task definition
|
||||
taskDef := &base.TaskDefinition{
|
||||
Type: types.TaskTypeErasureCoding,
|
||||
Name: "erasure_coding",
|
||||
DisplayName: "Erasure Coding",
|
||||
Description: "Applies erasure coding to volumes for data protection",
|
||||
Icon: "fas fa-shield-alt text-success",
|
||||
Capabilities: []string{"erasure_coding", "data_protection"},
|
||||
|
||||
Config: config,
|
||||
ConfigSpec: GetConfigSpec(),
|
||||
CreateTask: func(params *worker_pb.TaskParams) (types.Task, error) {
|
||||
if params == nil {
|
||||
return nil, fmt.Errorf("task parameters are required")
|
||||
}
|
||||
if len(params.Sources) == 0 {
|
||||
return nil, fmt.Errorf("at least one source is required for erasure coding task")
|
||||
}
|
||||
return NewErasureCodingTask(
|
||||
fmt.Sprintf("erasure_coding-%d", params.VolumeId),
|
||||
params.Sources[0].Node, // Use first source node
|
||||
params.VolumeId,
|
||||
params.Collection,
|
||||
dialOpt,
|
||||
), nil
|
||||
},
|
||||
DetectionFunc: func(metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo, config base.TaskConfig) ([]*types.TaskDetectionResult, error) {
|
||||
results, _, err := Detection(context.Background(), metrics, clusterInfo, config, 0)
|
||||
return results, err
|
||||
},
|
||||
ScanInterval: 1 * time.Hour,
|
||||
SchedulingFunc: Scheduling,
|
||||
MaxConcurrent: 1,
|
||||
RepeatInterval: 24 * time.Hour,
|
||||
}
|
||||
|
||||
// Store task definition globally for configuration updates
|
||||
globalTaskDef = taskDef
|
||||
|
||||
// Register everything with a single function call!
|
||||
base.RegisterTask(taskDef)
|
||||
}
|
||||
|
||||
// UpdateConfigFromPersistence updates the erasure coding configuration from persistence
|
||||
func UpdateConfigFromPersistence(configPersistence interface{}) error {
|
||||
if globalTaskDef == nil {
|
||||
return fmt.Errorf("erasure coding task not registered")
|
||||
}
|
||||
|
||||
// Load configuration from persistence
|
||||
newConfig := LoadConfigFromPersistence(configPersistence)
|
||||
if newConfig == nil {
|
||||
return fmt.Errorf("failed to load configuration from persistence")
|
||||
}
|
||||
|
||||
// Update the task definition's config
|
||||
globalTaskDef.Config = newConfig
|
||||
|
||||
glog.V(1).Infof("Updated erasure coding task configuration from persistence")
|
||||
return nil
|
||||
}
|
||||
40
weed/worker/tasks/erasure_coding/scheduling.go
Normal file
40
weed/worker/tasks/erasure_coding/scheduling.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package erasure_coding
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// Scheduling implements the scheduling logic for erasure coding tasks
|
||||
func Scheduling(task *types.TaskInput, runningTasks []*types.TaskInput, availableWorkers []*types.WorkerData, config base.TaskConfig) bool {
|
||||
ecConfig := config.(*Config)
|
||||
|
||||
// Check if we have available workers
|
||||
if len(availableWorkers) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// Count running EC tasks
|
||||
runningCount := 0
|
||||
for _, runningTask := range runningTasks {
|
||||
if runningTask.Type == types.TaskTypeErasureCoding {
|
||||
runningCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Check concurrency limit
|
||||
if runningCount >= ecConfig.MaxConcurrent {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check if any worker can handle EC tasks
|
||||
for _, worker := range availableWorkers {
|
||||
for _, capability := range worker.Capabilities {
|
||||
if capability == types.TaskTypeErasureCoding {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
Reference in New Issue
Block a user