snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
183
weed/worker/tasks/vacuum/config.go
Normal file
183
weed/worker/tasks/vacuum/config.go
Normal file
@@ -0,0 +1,183 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/config"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
)
|
||||
|
||||
// Config extends BaseConfig with vacuum-specific settings
|
||||
type Config struct {
|
||||
base.BaseConfig
|
||||
GarbageThreshold float64 `json:"garbage_threshold"`
|
||||
MinVolumeAgeSeconds int `json:"min_volume_age_seconds"`
|
||||
}
|
||||
|
||||
// NewDefaultConfig creates a new default vacuum configuration
|
||||
func NewDefaultConfig() *Config {
|
||||
return &Config{
|
||||
BaseConfig: base.BaseConfig{
|
||||
Enabled: true,
|
||||
ScanIntervalSeconds: 2 * 60 * 60, // 2 hours
|
||||
MaxConcurrent: 2,
|
||||
},
|
||||
GarbageThreshold: 0.3, // 30%
|
||||
MinVolumeAgeSeconds: 24 * 60 * 60, // 24 hours
|
||||
}
|
||||
}
|
||||
|
||||
// ToTaskPolicy converts configuration to a TaskPolicy protobuf message
|
||||
func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
|
||||
return &worker_pb.TaskPolicy{
|
||||
Enabled: c.Enabled,
|
||||
MaxConcurrent: int32(c.MaxConcurrent),
|
||||
RepeatIntervalSeconds: int32(c.ScanIntervalSeconds),
|
||||
CheckIntervalSeconds: int32(c.ScanIntervalSeconds),
|
||||
TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
|
||||
VacuumConfig: &worker_pb.VacuumTaskConfig{
|
||||
GarbageThreshold: float64(c.GarbageThreshold),
|
||||
MinVolumeAgeHours: int32((c.MinVolumeAgeSeconds + 3599) / 3600), // round up so sub-hour values don't become 0
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// FromTaskPolicy loads configuration from a TaskPolicy protobuf message
|
||||
func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
|
||||
if policy == nil {
|
||||
return fmt.Errorf("policy is nil")
|
||||
}
|
||||
|
||||
// Set general TaskPolicy fields
|
||||
c.Enabled = policy.Enabled
|
||||
c.MaxConcurrent = int(policy.MaxConcurrent)
|
||||
c.ScanIntervalSeconds = int(policy.RepeatIntervalSeconds) // Direct seconds-to-seconds mapping
|
||||
|
||||
// Set vacuum-specific fields from the task config
|
||||
if vacuumConfig := policy.GetVacuumConfig(); vacuumConfig != nil {
|
||||
c.GarbageThreshold = float64(vacuumConfig.GarbageThreshold)
|
||||
c.MinVolumeAgeSeconds = int(vacuumConfig.MinVolumeAgeHours * 3600) // Convert hours to seconds
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadConfigFromPersistence loads configuration from the persistence layer if available
|
||||
func LoadConfigFromPersistence(configPersistence interface{}) *Config {
|
||||
config := NewDefaultConfig()
|
||||
|
||||
// Try to load from persistence if available
|
||||
if persistence, ok := configPersistence.(interface {
|
||||
LoadVacuumTaskPolicy() (*worker_pb.TaskPolicy, error)
|
||||
}); ok {
|
||||
policy, err := persistence.LoadVacuumTaskPolicy()
|
||||
switch {
|
||||
case err != nil:
|
||||
glog.Warningf("Could not read the persisted vacuum configuration, falling back to defaults: %v", err)
|
||||
case policy == nil:
|
||||
glog.V(1).Infof("No vacuum configuration persisted yet, using defaults")
|
||||
default:
|
||||
if err := config.FromTaskPolicy(policy); err != nil {
|
||||
glog.Warningf("Could not apply the persisted vacuum configuration, falling back to defaults: %v", err)
|
||||
} else {
|
||||
glog.V(1).Infof("Loaded vacuum configuration from persistence")
|
||||
return config
|
||||
}
|
||||
}
|
||||
} else if configPersistence != nil {
|
||||
// A store was handed in but does not expose the accessor, so the persisted
|
||||
// settings are silently ignored - always a wiring bug, never a normal state.
|
||||
glog.Warningf("%T cannot provide the persisted vacuum configuration: it has no LoadVacuumTaskPolicy() method, "+
|
||||
"so the compiled-in defaults are used and any saved vacuum settings are ignored", configPersistence)
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Using default vacuum configuration")
|
||||
return config
|
||||
}
|
||||
|
||||
// GetConfigSpec returns the configuration schema for vacuum tasks
|
||||
func GetConfigSpec() base.ConfigSpec {
|
||||
return base.ConfigSpec{
|
||||
Fields: []*config.Field{
|
||||
{
|
||||
Name: "enabled",
|
||||
JSONName: "enabled",
|
||||
Type: config.FieldTypeBool,
|
||||
DefaultValue: true,
|
||||
Required: false,
|
||||
DisplayName: "Enable Vacuum Tasks",
|
||||
Description: "Whether vacuum tasks should be automatically created",
|
||||
HelpText: "Toggle this to enable or disable automatic vacuum task generation",
|
||||
InputType: "checkbox",
|
||||
CSSClasses: "form-check-input",
|
||||
},
|
||||
{
|
||||
Name: "scan_interval_seconds",
|
||||
JSONName: "scan_interval_seconds",
|
||||
Type: config.FieldTypeInterval,
|
||||
DefaultValue: 2 * 60 * 60,
|
||||
MinValue: 10 * 60,
|
||||
MaxValue: 24 * 60 * 60,
|
||||
Required: true,
|
||||
DisplayName: "Scan Interval",
|
||||
Description: "How often to scan for volumes needing vacuum",
|
||||
HelpText: "The system will check for volumes that need vacuuming at this interval",
|
||||
Placeholder: "2",
|
||||
Unit: config.UnitHours,
|
||||
InputType: "interval",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "max_concurrent",
|
||||
JSONName: "max_concurrent",
|
||||
Type: config.FieldTypeInt,
|
||||
DefaultValue: 2,
|
||||
MinValue: 1,
|
||||
MaxValue: 10,
|
||||
Required: true,
|
||||
DisplayName: "Max Concurrent Tasks",
|
||||
Description: "Maximum number of vacuum tasks that can run simultaneously",
|
||||
HelpText: "Limits the number of vacuum operations running at the same time to control system load",
|
||||
Placeholder: "2 (default)",
|
||||
Unit: config.UnitCount,
|
||||
InputType: "number",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "garbage_threshold",
|
||||
JSONName: "garbage_threshold",
|
||||
Type: config.FieldTypeFloat,
|
||||
DefaultValue: 0.3,
|
||||
MinValue: 0.0,
|
||||
MaxValue: 1.0,
|
||||
Required: true,
|
||||
DisplayName: "Garbage Percentage Threshold",
|
||||
Description: "Trigger vacuum when garbage ratio exceeds this percentage",
|
||||
HelpText: "Volumes with more deleted content than this threshold will be vacuumed",
|
||||
Placeholder: "0.30 (30%)",
|
||||
Unit: config.UnitNone,
|
||||
InputType: "number",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "min_volume_age_seconds",
|
||||
JSONName: "min_volume_age_seconds",
|
||||
Type: config.FieldTypeInterval,
|
||||
DefaultValue: 24 * 60 * 60,
|
||||
MinValue: 1 * 60 * 60,
|
||||
MaxValue: 7 * 24 * 60 * 60,
|
||||
Required: true,
|
||||
DisplayName: "Minimum Volume Age",
|
||||
Description: "Only vacuum volumes older than this duration",
|
||||
HelpText: "Prevents vacuuming of recently created volumes that may still be actively written to",
|
||||
Placeholder: "24",
|
||||
Unit: config.UnitHours,
|
||||
InputType: "interval",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
75
weed/worker/tasks/vacuum/config_persistence_test.go
Normal file
75
weed/worker/tasks/vacuum/config_persistence_test.go
Normal file
@@ -0,0 +1,75 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
)
|
||||
|
||||
type stubVacuumStore struct {
|
||||
policy *worker_pb.TaskPolicy
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *stubVacuumStore) LoadVacuumTaskPolicy() (*worker_pb.TaskPolicy, error) {
|
||||
return s.policy, s.err
|
||||
}
|
||||
|
||||
// wrongShapedStore is what the maintenance manager used to be handed: a non-nil value that
|
||||
// does not satisfy the accessor the loader asserts on. It has to keep falling back to the
|
||||
// defaults, but no longer silently: a fallback the operator cannot see reads as a disabled task
|
||||
// that keeps running.
|
||||
type wrongShapedStore struct{}
|
||||
|
||||
func (wrongShapedStore) SomethingElse() {}
|
||||
|
||||
func TestLoadConfigFromPersistenceUsesPersistedPolicy(t *testing.T) {
|
||||
persisted := NewDefaultConfig()
|
||||
persisted.Enabled = false
|
||||
persisted.GarbageThreshold = 0.75
|
||||
persisted.MaxConcurrent = 5
|
||||
|
||||
loaded := LoadConfigFromPersistence(&stubVacuumStore{policy: persisted.ToTaskPolicy()})
|
||||
if loaded == nil {
|
||||
t.Fatal("LoadConfigFromPersistence returned nil")
|
||||
}
|
||||
if loaded.Enabled {
|
||||
t.Error("enabled = true, want the persisted false")
|
||||
}
|
||||
if loaded.GarbageThreshold != 0.75 {
|
||||
t.Errorf("garbage threshold = %v, want the persisted 0.75", loaded.GarbageThreshold)
|
||||
}
|
||||
if loaded.MaxConcurrent != 5 {
|
||||
t.Errorf("max concurrent = %d, want the persisted 5", loaded.MaxConcurrent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigFromPersistenceFallsBackToDefaults(t *testing.T) {
|
||||
defaults := NewDefaultConfig()
|
||||
|
||||
cases := map[string]interface{}{
|
||||
"no store configured": nil,
|
||||
"store without the value": &stubVacuumStore{},
|
||||
"store that errors": &stubVacuumStore{err: errors.New("disk on fire")},
|
||||
"store of the wrong type": wrongShapedStore{},
|
||||
}
|
||||
|
||||
for name, store := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
loaded := LoadConfigFromPersistence(store)
|
||||
if loaded == nil {
|
||||
t.Fatal("LoadConfigFromPersistence returned nil")
|
||||
}
|
||||
if !loaded.Enabled {
|
||||
t.Error("enabled = false, want the compiled-in default true")
|
||||
}
|
||||
if loaded.GarbageThreshold != defaults.GarbageThreshold {
|
||||
t.Errorf("garbage threshold = %v, want the default %v", loaded.GarbageThreshold, defaults.GarbageThreshold)
|
||||
}
|
||||
if loaded.ScanIntervalSeconds != defaults.ScanIntervalSeconds {
|
||||
t.Errorf("scan interval = %d, want the default %d", loaded.ScanIntervalSeconds, defaults.ScanIntervalSeconds)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
227
weed/worker/tasks/vacuum/detection.go
Normal file
227
weed/worker/tasks/vacuum/detection.go
Normal file
@@ -0,0 +1,227 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// Detection implements the detection logic for vacuum tasks
|
||||
func Detection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo, config base.TaskConfig) ([]*types.TaskDetectionResult, error) {
|
||||
if !config.IsEnabled() {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
vacuumConfig := config.(*Config)
|
||||
minVolumeAge := time.Duration(vacuumConfig.MinVolumeAgeSeconds) * time.Second
|
||||
|
||||
type volumeBucket struct {
|
||||
replicas []*types.VolumeHealthMetrics
|
||||
}
|
||||
buckets := make(map[uint32]*volumeBucket)
|
||||
order := make([]uint32, 0)
|
||||
for _, m := range metrics {
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
b, ok := buckets[m.VolumeID]
|
||||
if !ok {
|
||||
b = &volumeBucket{}
|
||||
buckets[m.VolumeID] = b
|
||||
order = append(order, m.VolumeID)
|
||||
}
|
||||
b.replicas = append(b.replicas, m)
|
||||
}
|
||||
|
||||
var results []*types.TaskDetectionResult
|
||||
skippedDueToGarbage := 0
|
||||
skippedDueToAge := 0
|
||||
debugCount := 0
|
||||
|
||||
for _, vid := range order {
|
||||
replicas := buckets[vid].replicas
|
||||
|
||||
// Pick the replica with the highest garbage ratio that also satisfies
|
||||
// the minimum-age gate as the "primary" used for priority + reason.
|
||||
// Master built-in vacuum filters per-replica again inside
|
||||
// batchVacuumVolumeCheck, so it's fine if only some replicas are
|
||||
// above threshold here — the worker's checkVacuumEligibility will
|
||||
// filter again at execute time.
|
||||
primary := pickPrimaryReplica(replicas, vacuumConfig.GarbageThreshold, minVolumeAge)
|
||||
if primary == nil {
|
||||
recordSkipReasons(replicas, vacuumConfig.GarbageThreshold, minVolumeAge, &skippedDueToGarbage, &skippedDueToAge, &debugCount)
|
||||
continue
|
||||
}
|
||||
|
||||
priority := types.TaskPriorityNormal
|
||||
if primary.GarbageRatio > 0.6 {
|
||||
priority = types.TaskPriorityHigh
|
||||
}
|
||||
|
||||
taskID := fmt.Sprintf("vacuum_vol_%d_%d", vid, time.Now().Unix())
|
||||
|
||||
result := &types.TaskDetectionResult{
|
||||
TaskID: taskID,
|
||||
TaskType: types.TaskTypeVacuum,
|
||||
VolumeID: vid,
|
||||
Server: primary.Server,
|
||||
Collection: primary.Collection,
|
||||
Priority: priority,
|
||||
Reason: "Volume has excessive garbage requiring vacuum",
|
||||
ScheduleAt: time.Now(),
|
||||
}
|
||||
|
||||
// Check if ANY task already exists in ActiveTopology for this volume
|
||||
if clusterInfo != nil && clusterInfo.ActiveTopology != nil {
|
||||
if clusterInfo.ActiveTopology.HasAnyTask(vid) {
|
||||
glog.V(2).Infof("VACUUM: Skipping volume %d, task already exists in ActiveTopology", vid)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
result.TypedParams = createVacuumTaskParams(result, replicas, vacuumConfig, clusterInfo)
|
||||
if result.TypedParams != nil {
|
||||
results = append(results, result)
|
||||
}
|
||||
}
|
||||
|
||||
if len(results) == 0 && len(metrics) > 0 {
|
||||
totalVolumes := len(metrics)
|
||||
glog.V(1).Infof("VACUUM: No tasks created for %d volume replicas across %d volumes. Threshold=%.2f%%, MinAge=%s. Skipped: %d (garbage<threshold), %d (age<minimum)",
|
||||
totalVolumes, len(buckets), vacuumConfig.GarbageThreshold*100, minVolumeAge, skippedDueToGarbage, skippedDueToAge)
|
||||
|
||||
for i, metric := range metrics {
|
||||
if i >= 3 {
|
||||
break
|
||||
}
|
||||
glog.V(1).Infof("VACUUM: Volume %d on %s: garbage=%.2f%% (need ≥%.2f%%), age=%s (need ≥%s)",
|
||||
metric.VolumeID, metric.Server, metric.GarbageRatio*100, vacuumConfig.GarbageThreshold*100,
|
||||
metric.Age.Truncate(time.Minute), minVolumeAge.Truncate(time.Minute))
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// pickPrimaryReplica returns the eligible replica with the highest garbage
|
||||
// ratio. Returns nil if no replica satisfies both gates.
|
||||
func pickPrimaryReplica(replicas []*types.VolumeHealthMetrics, garbageThreshold float64, minAge time.Duration) *types.VolumeHealthMetrics {
|
||||
var best *types.VolumeHealthMetrics
|
||||
for _, m := range replicas {
|
||||
if m.GarbageRatio < garbageThreshold {
|
||||
continue
|
||||
}
|
||||
if m.Age < minAge {
|
||||
continue
|
||||
}
|
||||
if best == nil || m.GarbageRatio > best.GarbageRatio {
|
||||
best = m
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// recordSkipReasons tallies skip counters for debug logging when no
|
||||
// replica of a volume qualified for vacuum.
|
||||
func recordSkipReasons(replicas []*types.VolumeHealthMetrics, garbageThreshold float64, minAge time.Duration, garbageSkip, ageSkip, debugCount *int) {
|
||||
for _, m := range replicas {
|
||||
if *debugCount >= 5 {
|
||||
return
|
||||
}
|
||||
if m.GarbageRatio < garbageThreshold {
|
||||
*garbageSkip++
|
||||
}
|
||||
if m.Age < minAge {
|
||||
*ageSkip++
|
||||
}
|
||||
*debugCount++
|
||||
}
|
||||
}
|
||||
|
||||
// createVacuumTaskParams creates typed parameters for a vacuum task whose
|
||||
// Sources list contains every replica of the volume. Worker-side
|
||||
// performVacuum iterates over Sources and vacuums each replica.
|
||||
func createVacuumTaskParams(task *types.TaskDetectionResult, replicas []*types.VolumeHealthMetrics, vacuumConfig *Config, clusterInfo *types.ClusterInfo) *worker_pb.TaskParams {
|
||||
garbageThreshold := 0.3
|
||||
verifyChecksum := true
|
||||
batchSize := int32(1000)
|
||||
workingDir := ""
|
||||
|
||||
if vacuumConfig != nil {
|
||||
garbageThreshold = vacuumConfig.GarbageThreshold
|
||||
}
|
||||
|
||||
if clusterInfo == nil || clusterInfo.ActiveTopology == nil {
|
||||
glog.Errorf("Topology not available for vacuum task on volume %d, skipping", task.VolumeID)
|
||||
return nil
|
||||
}
|
||||
|
||||
sources := make([]*worker_pb.TaskSource, 0, len(replicas))
|
||||
seen := make(map[string]struct{}, len(replicas))
|
||||
for _, m := range replicas {
|
||||
address, err := util.ResolveServerAddress(m.Server, clusterInfo.ActiveTopology)
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to resolve address for server %s for vacuum task on volume %d, dropping replica: %v", m.Server, task.VolumeID, err)
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[address]; ok {
|
||||
continue
|
||||
}
|
||||
seen[address] = struct{}{}
|
||||
sources = append(sources, &worker_pb.TaskSource{
|
||||
Node: address,
|
||||
VolumeId: task.VolumeID,
|
||||
EstimatedSize: m.Size,
|
||||
DataCenter: m.DataCenter,
|
||||
Rack: m.Rack,
|
||||
})
|
||||
}
|
||||
|
||||
if len(sources) == 0 {
|
||||
glog.Errorf("No resolvable replicas for vacuum task on volume %d, skipping", task.VolumeID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Use the primary replica (matches task.Server) for the canonical size
|
||||
// recorded in TaskParams. Master built-in keeps a single volumeSizeLimit
|
||||
// here so timeouts are stable across replicas.
|
||||
canonical := primaryReplicaByServer(replicas, task.Server)
|
||||
var canonicalSize uint64
|
||||
if canonical != nil {
|
||||
canonicalSize = canonical.Size
|
||||
}
|
||||
|
||||
return &worker_pb.TaskParams{
|
||||
TaskId: task.TaskID,
|
||||
VolumeId: task.VolumeID,
|
||||
Collection: task.Collection,
|
||||
VolumeSize: canonicalSize,
|
||||
Sources: sources,
|
||||
TaskParams: &worker_pb.TaskParams_VacuumParams{
|
||||
VacuumParams: &worker_pb.VacuumTaskParams{
|
||||
GarbageThreshold: garbageThreshold,
|
||||
ForceVacuum: false,
|
||||
BatchSize: batchSize,
|
||||
WorkingDir: workingDir,
|
||||
VerifyChecksum: verifyChecksum,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func primaryReplicaByServer(replicas []*types.VolumeHealthMetrics, server string) *types.VolumeHealthMetrics {
|
||||
for _, m := range replicas {
|
||||
if m.Server == server {
|
||||
return m
|
||||
}
|
||||
}
|
||||
if len(replicas) > 0 {
|
||||
return replicas[0]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
611
weed/worker/tasks/vacuum/plugin_handler.go
Normal file
611
weed/worker/tasks/vacuum/plugin_handler.go
Normal file
@@ -0,0 +1,611 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultVacuumTaskBatchSize = int32(1000)
|
||||
DefaultMaxExecutionConcurrency = int32(2)
|
||||
)
|
||||
|
||||
func init() {
|
||||
pluginworker.RegisterHandler(pluginworker.HandlerFactory{
|
||||
JobType: "vacuum",
|
||||
Category: pluginworker.CategoryDefault,
|
||||
Aliases: []string{"vol.vacuum", "volume.vacuum"},
|
||||
Build: func(opts pluginworker.HandlerBuildOptions) (pluginworker.JobHandler, error) {
|
||||
return NewVacuumHandler(opts.GrpcDialOption, int32(opts.MaxExecute)), nil
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// VacuumHandler is the plugin job handler for vacuum job type.
|
||||
type VacuumHandler struct {
|
||||
grpcDialOption grpc.DialOption
|
||||
maxExecutionConcurrency int32
|
||||
}
|
||||
|
||||
// NewVacuumHandler creates a VacuumHandler with the given gRPC dial option and
|
||||
// maximum execution concurrency. When maxExecutionConcurrency is zero or
|
||||
// negative, DefaultMaxExecutionConcurrency is used as the fallback.
|
||||
func NewVacuumHandler(grpcDialOption grpc.DialOption, maxExecutionConcurrency int32) *VacuumHandler {
|
||||
return &VacuumHandler{grpcDialOption: grpcDialOption, maxExecutionConcurrency: maxExecutionConcurrency}
|
||||
}
|
||||
|
||||
// Capability returns the job type capability for the vacuum handler.
|
||||
// MaxExecutionConcurrency reflects the value passed at construction time,
|
||||
// falling back to DefaultMaxExecutionConcurrency when unset.
|
||||
func (h *VacuumHandler) Capability() *plugin_pb.JobTypeCapability {
|
||||
maxExec := h.maxExecutionConcurrency
|
||||
if maxExec <= 0 {
|
||||
maxExec = DefaultMaxExecutionConcurrency
|
||||
}
|
||||
return &plugin_pb.JobTypeCapability{
|
||||
JobType: "vacuum",
|
||||
CanDetect: true,
|
||||
CanExecute: true,
|
||||
MaxDetectionConcurrency: 1,
|
||||
MaxExecutionConcurrency: maxExec,
|
||||
DisplayName: "Volume Vacuum",
|
||||
Description: "Reclaims disk space by removing deleted files from volumes",
|
||||
Weight: 60,
|
||||
}
|
||||
}
|
||||
|
||||
func (h *VacuumHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
|
||||
return &plugin_pb.JobTypeDescriptor{
|
||||
JobType: "vacuum",
|
||||
DisplayName: "Volume Vacuum",
|
||||
Description: "Detect and vacuum volumes with high garbage ratio",
|
||||
Icon: "fas fa-broom",
|
||||
DescriptorVersion: 1,
|
||||
AdminConfigForm: &plugin_pb.ConfigForm{
|
||||
FormId: "vacuum-admin",
|
||||
Title: "Vacuum Admin Config",
|
||||
Description: "Admin-side controls for vacuum detection scope.",
|
||||
Sections: []*plugin_pb.ConfigSection{
|
||||
{
|
||||
SectionId: "scope",
|
||||
Title: "Scope",
|
||||
Description: "Optional filter to restrict detection.",
|
||||
Fields: []*plugin_pb.ConfigField{
|
||||
{
|
||||
Name: "collection_filter",
|
||||
Label: "Collection Filter",
|
||||
Description: "Filter collections for vacuum detection. Use ALL_COLLECTIONS (default) to treat all volumes as one pool, EACH_COLLECTION to run detection separately per collection, or a comma-separated list of names, wildcards, or regex patterns.",
|
||||
Placeholder: "ALL_COLLECTIONS",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
|
||||
},
|
||||
{
|
||||
Name: "volume_state",
|
||||
Label: "Volume State Filter",
|
||||
Description: "Filter volumes by state: ALL (default), ACTIVE (writable volumes below size limit), or FULL (read-only volumes above size limit).",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_ENUM,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_SELECT,
|
||||
Options: []*plugin_pb.ConfigOption{
|
||||
{Value: string(pluginworker.VolumeStateAll), Label: "All Volumes"},
|
||||
{Value: string(pluginworker.VolumeStateActive), Label: "Active (writable)"},
|
||||
{Value: string(pluginworker.VolumeStateFull), Label: "Full (read-only)"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "data_center_filter",
|
||||
Label: "Data Center Filter",
|
||||
Description: "Only vacuum volumes in matching data centers (comma-separated, wildcards supported). Leave empty for all.",
|
||||
Placeholder: "all data centers",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
|
||||
},
|
||||
{
|
||||
Name: "rack_filter",
|
||||
Label: "Rack Filter",
|
||||
Description: "Only vacuum volumes on matching racks (comma-separated, wildcards supported). Leave empty for all.",
|
||||
Placeholder: "all racks",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
|
||||
},
|
||||
{
|
||||
Name: "node_filter",
|
||||
Label: "Node Filter",
|
||||
Description: "Only vacuum volumes on matching nodes (comma-separated, wildcards supported). Leave empty for all.",
|
||||
Placeholder: "all nodes",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
DefaultValues: map[string]*plugin_pb.ConfigValue{
|
||||
"collection_filter": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
|
||||
},
|
||||
"volume_state": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: string(pluginworker.VolumeStateAll)},
|
||||
},
|
||||
"data_center_filter": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
|
||||
},
|
||||
"rack_filter": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
|
||||
},
|
||||
"node_filter": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
|
||||
},
|
||||
},
|
||||
},
|
||||
WorkerConfigForm: &plugin_pb.ConfigForm{
|
||||
FormId: "vacuum-worker",
|
||||
Title: "Vacuum Worker Config",
|
||||
Description: "Worker-side vacuum thresholds.",
|
||||
Sections: []*plugin_pb.ConfigSection{
|
||||
{
|
||||
SectionId: "thresholds",
|
||||
Title: "Thresholds",
|
||||
Description: "Detection thresholds and timing constraints.",
|
||||
Fields: []*plugin_pb.ConfigField{
|
||||
{
|
||||
Name: "garbage_threshold",
|
||||
Label: "Garbage Threshold",
|
||||
Description: "Detect volumes with garbage ratio >= threshold.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_DOUBLE,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
Required: true,
|
||||
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0}},
|
||||
MaxValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 1}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
DefaultValues: map[string]*plugin_pb.ConfigValue{
|
||||
"garbage_threshold": {
|
||||
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.3},
|
||||
},
|
||||
},
|
||||
},
|
||||
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
|
||||
Enabled: true,
|
||||
DetectionIntervalMinutes: 17,
|
||||
DetectionTimeoutSeconds: 120,
|
||||
MaxJobsPerDetection: 200,
|
||||
GlobalExecutionConcurrency: 16,
|
||||
PerWorkerExecutionConcurrency: 4,
|
||||
RetryLimit: 1,
|
||||
RetryBackoffSeconds: 10,
|
||||
JobTypeMaxRuntimeSeconds: 1800,
|
||||
ExecutionTimeoutSeconds: 1800,
|
||||
},
|
||||
WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{
|
||||
"garbage_threshold": {
|
||||
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.3},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (h *VacuumHandler) Detect(ctx context.Context, request *plugin_pb.RunDetectionRequest, sender pluginworker.DetectionSender) error {
|
||||
if request == nil {
|
||||
return fmt.Errorf("run detection request is nil")
|
||||
}
|
||||
if sender == nil {
|
||||
return fmt.Errorf("detection sender is nil")
|
||||
}
|
||||
if request.JobType != "" && request.JobType != "vacuum" {
|
||||
return fmt.Errorf("job type %q is not handled by vacuum worker", request.JobType)
|
||||
}
|
||||
|
||||
workerConfig := deriveVacuumConfig(request.GetWorkerConfigValues())
|
||||
collectionFilter := strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "collection_filter", ""))
|
||||
masters := make([]string, 0)
|
||||
if request.ClusterContext != nil {
|
||||
masters = append(masters, request.ClusterContext.MasterGrpcAddresses...)
|
||||
}
|
||||
metrics, activeTopology, err := h.collectVolumeMetrics(ctx, masters, collectionFilter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
volState := pluginworker.VolumeState(strings.ToUpper(strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "volume_state", string(pluginworker.VolumeStateAll)))))
|
||||
metrics = pluginworker.FilterMetricsByVolumeState(metrics, volState)
|
||||
|
||||
dataCenterFilter := strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "data_center_filter", ""))
|
||||
rackFilter := strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "rack_filter", ""))
|
||||
nodeFilter := strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "node_filter", ""))
|
||||
|
||||
if dataCenterFilter != "" || rackFilter != "" || nodeFilter != "" {
|
||||
metrics = pluginworker.FilterMetricsByLocation(metrics, dataCenterFilter, rackFilter, nodeFilter)
|
||||
}
|
||||
|
||||
clusterInfo := &workertypes.ClusterInfo{ActiveTopology: activeTopology}
|
||||
results, err := Detection(metrics, clusterInfo, workerConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if traceErr := emitVacuumDetectionDecisionTrace(sender, metrics, workerConfig, results); traceErr != nil {
|
||||
glog.Warningf("Plugin worker failed to emit vacuum detection trace: %v", traceErr)
|
||||
}
|
||||
|
||||
maxResults := int(request.MaxResults)
|
||||
hasMore := false
|
||||
if maxResults > 0 && len(results) > maxResults {
|
||||
hasMore = true
|
||||
results = results[:maxResults]
|
||||
}
|
||||
|
||||
proposals := make([]*plugin_pb.JobProposal, 0, len(results))
|
||||
for _, result := range results {
|
||||
proposal, proposalErr := buildVacuumProposal(result)
|
||||
if proposalErr != nil {
|
||||
glog.Warningf("Plugin worker skip invalid vacuum proposal: %v", proposalErr)
|
||||
continue
|
||||
}
|
||||
proposals = append(proposals, proposal)
|
||||
}
|
||||
|
||||
if err := sender.SendProposals(&plugin_pb.DetectionProposals{
|
||||
JobType: "vacuum",
|
||||
Proposals: proposals,
|
||||
HasMore: hasMore,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return sender.SendComplete(&plugin_pb.DetectionComplete{
|
||||
JobType: "vacuum",
|
||||
Success: true,
|
||||
TotalProposals: int32(len(proposals)),
|
||||
})
|
||||
}
|
||||
|
||||
func emitVacuumDetectionDecisionTrace(
|
||||
sender pluginworker.DetectionSender,
|
||||
metrics []*workertypes.VolumeHealthMetrics,
|
||||
workerConfig *Config,
|
||||
results []*workertypes.TaskDetectionResult,
|
||||
) error {
|
||||
if sender == nil || workerConfig == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
totalVolumes := len(metrics)
|
||||
|
||||
skippedDueToGarbage := 0
|
||||
for _, metric := range metrics {
|
||||
if metric == nil {
|
||||
continue
|
||||
}
|
||||
if metric.GarbageRatio >= workerConfig.GarbageThreshold {
|
||||
continue
|
||||
}
|
||||
skippedDueToGarbage++
|
||||
}
|
||||
|
||||
summaryMessage := ""
|
||||
summaryStage := "decision_summary"
|
||||
if len(results) == 0 {
|
||||
summaryMessage = fmt.Sprintf(
|
||||
"VACUUM: No tasks created for %d volumes. Threshold=%.2f%%. Skipped: %d (garbage<threshold)",
|
||||
totalVolumes,
|
||||
workerConfig.GarbageThreshold*100,
|
||||
skippedDueToGarbage,
|
||||
)
|
||||
} else {
|
||||
summaryMessage = fmt.Sprintf(
|
||||
"VACUUM: Created %d task(s) from %d volumes. Threshold=%.2f%%",
|
||||
len(results),
|
||||
totalVolumes,
|
||||
workerConfig.GarbageThreshold*100,
|
||||
)
|
||||
}
|
||||
|
||||
if err := sender.SendActivity(pluginworker.BuildDetectorActivity(summaryStage, summaryMessage, map[string]*plugin_pb.ConfigValue{
|
||||
"total_volumes": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(totalVolumes)},
|
||||
},
|
||||
"selected_tasks": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(results))},
|
||||
},
|
||||
"garbage_threshold_percent": {
|
||||
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: workerConfig.GarbageThreshold * 100},
|
||||
},
|
||||
"skipped_garbage": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedDueToGarbage)},
|
||||
},
|
||||
})); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
limit := 3
|
||||
if len(metrics) < limit {
|
||||
limit = len(metrics)
|
||||
}
|
||||
for i := 0; i < limit; i++ {
|
||||
metric := metrics[i]
|
||||
if metric == nil {
|
||||
continue
|
||||
}
|
||||
message := fmt.Sprintf(
|
||||
"VACUUM: Volume %d: garbage=%.2f%% (need ≥%.2f%%)",
|
||||
metric.VolumeID,
|
||||
metric.GarbageRatio*100,
|
||||
workerConfig.GarbageThreshold*100,
|
||||
)
|
||||
if err := sender.SendActivity(pluginworker.BuildDetectorActivity("decision_volume", message, map[string]*plugin_pb.ConfigValue{
|
||||
"volume_id": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(metric.VolumeID)},
|
||||
},
|
||||
"garbage_percent": {
|
||||
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: metric.GarbageRatio * 100},
|
||||
},
|
||||
"required_garbage_percent": {
|
||||
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: workerConfig.GarbageThreshold * 100},
|
||||
},
|
||||
})); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *VacuumHandler) Execute(ctx context.Context, request *plugin_pb.ExecuteJobRequest, sender pluginworker.ExecutionSender) error {
|
||||
if request == nil || request.Job == nil {
|
||||
return fmt.Errorf("execute request/job is nil")
|
||||
}
|
||||
if sender == nil {
|
||||
return fmt.Errorf("execution sender is nil")
|
||||
}
|
||||
if request.Job.JobType != "" && request.Job.JobType != "vacuum" {
|
||||
return fmt.Errorf("job type %q is not handled by vacuum worker", request.Job.JobType)
|
||||
}
|
||||
|
||||
params, err := decodeVacuumTaskParams(request.Job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(params.Sources) == 0 || strings.TrimSpace(params.Sources[0].Node) == "" {
|
||||
return fmt.Errorf("vacuum task source node is required")
|
||||
}
|
||||
|
||||
workerConfig := deriveVacuumConfig(request.GetWorkerConfigValues())
|
||||
if vacuumParams := params.GetVacuumParams(); vacuumParams != nil {
|
||||
if vacuumParams.GarbageThreshold <= 0 {
|
||||
vacuumParams.GarbageThreshold = workerConfig.GarbageThreshold
|
||||
}
|
||||
} else {
|
||||
params.TaskParams = &worker_pb.TaskParams_VacuumParams{
|
||||
VacuumParams: &worker_pb.VacuumTaskParams{
|
||||
GarbageThreshold: workerConfig.GarbageThreshold,
|
||||
BatchSize: defaultVacuumTaskBatchSize,
|
||||
VerifyChecksum: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
servers := make([]string, 0, len(params.Sources))
|
||||
for _, src := range params.Sources {
|
||||
if src != nil && src.Node != "" {
|
||||
servers = append(servers, src.Node)
|
||||
}
|
||||
}
|
||||
task := NewVacuumTask(
|
||||
request.Job.JobId,
|
||||
servers,
|
||||
params.VolumeId,
|
||||
params.Collection,
|
||||
h.grpcDialOption,
|
||||
)
|
||||
execCtx, execCancel := context.WithCancel(ctx)
|
||||
defer execCancel()
|
||||
task.SetProgressCallback(func(progress float64, stage string) {
|
||||
message := fmt.Sprintf("vacuum progress %.0f%%", progress)
|
||||
if strings.TrimSpace(stage) != "" {
|
||||
message = stage
|
||||
}
|
||||
if err := sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: request.Job.JobId,
|
||||
JobType: request.Job.JobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_RUNNING,
|
||||
ProgressPercent: progress,
|
||||
Stage: stage,
|
||||
Message: message,
|
||||
Activities: []*plugin_pb.ActivityEvent{
|
||||
pluginworker.BuildExecutorActivity(stage, message),
|
||||
},
|
||||
}); err != nil {
|
||||
execCancel()
|
||||
}
|
||||
})
|
||||
|
||||
if err := sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: request.Job.JobId,
|
||||
JobType: request.Job.JobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_ASSIGNED,
|
||||
ProgressPercent: 0,
|
||||
Stage: "assigned",
|
||||
Message: "vacuum job accepted",
|
||||
Activities: []*plugin_pb.ActivityEvent{
|
||||
pluginworker.BuildExecutorActivity("assigned", "vacuum job accepted"),
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := task.Execute(execCtx, params); err != nil {
|
||||
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: request.Job.JobId,
|
||||
JobType: request.Job.JobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_FAILED,
|
||||
ProgressPercent: 100,
|
||||
Stage: "failed",
|
||||
Message: err.Error(),
|
||||
Activities: []*plugin_pb.ActivityEvent{
|
||||
pluginworker.BuildExecutorActivity("failed", err.Error()),
|
||||
},
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
resultSummary := fmt.Sprintf("vacuum completed for volume %d", params.VolumeId)
|
||||
return sender.SendCompleted(&plugin_pb.JobCompleted{
|
||||
JobId: request.Job.JobId,
|
||||
JobType: request.Job.JobType,
|
||||
Success: true,
|
||||
Result: &plugin_pb.JobResult{
|
||||
Summary: resultSummary,
|
||||
OutputValues: map[string]*plugin_pb.ConfigValue{
|
||||
"volume_id": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(params.VolumeId)},
|
||||
},
|
||||
"server": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: params.Sources[0].Node},
|
||||
},
|
||||
},
|
||||
},
|
||||
Activities: []*plugin_pb.ActivityEvent{
|
||||
pluginworker.BuildExecutorActivity("completed", resultSummary),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (h *VacuumHandler) collectVolumeMetrics(
|
||||
ctx context.Context,
|
||||
masterAddresses []string,
|
||||
collectionFilter string,
|
||||
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, error) {
|
||||
metrics, activeTopology, _, err := pluginworker.CollectVolumeMetricsFromMasters(ctx, masterAddresses, collectionFilter, h.grpcDialOption)
|
||||
return metrics, activeTopology, err
|
||||
}
|
||||
|
||||
func deriveVacuumConfig(values map[string]*plugin_pb.ConfigValue) *Config {
|
||||
config := NewDefaultConfig()
|
||||
config.GarbageThreshold = pluginworker.ReadDoubleConfig(values, "garbage_threshold", config.GarbageThreshold)
|
||||
// Match the shell/master vacuum, which applies no volume-age gate (default 0),
|
||||
// but honor an explicit override instead of silently dropping it.
|
||||
config.MinVolumeAgeSeconds = pluginworker.ReadIntConfig(values, "min_volume_age_seconds", 0)
|
||||
return config
|
||||
}
|
||||
|
||||
func buildVacuumProposal(result *workertypes.TaskDetectionResult) (*plugin_pb.JobProposal, error) {
|
||||
if result == nil {
|
||||
return nil, fmt.Errorf("task detection result is nil")
|
||||
}
|
||||
if result.TypedParams == nil {
|
||||
return nil, fmt.Errorf("missing typed params for volume %d", result.VolumeID)
|
||||
}
|
||||
|
||||
paramsPayload, err := proto.Marshal(result.TypedParams)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal task params: %w", err)
|
||||
}
|
||||
|
||||
proposalID := strings.TrimSpace(result.TaskID)
|
||||
if proposalID == "" {
|
||||
proposalID = fmt.Sprintf("vacuum-%d-%d", result.VolumeID, time.Now().UnixNano())
|
||||
}
|
||||
|
||||
dedupeKey := fmt.Sprintf("vacuum:%d", result.VolumeID)
|
||||
if result.Collection != "" {
|
||||
dedupeKey = dedupeKey + ":" + result.Collection
|
||||
}
|
||||
|
||||
summary := fmt.Sprintf("Vacuum volume %d", result.VolumeID)
|
||||
if strings.TrimSpace(result.Server) != "" {
|
||||
summary = summary + " on " + result.Server
|
||||
}
|
||||
|
||||
// Estimate runtime: 5 min/GB (compact + commit + overhead)
|
||||
volumeSizeGB := int64(result.TypedParams.VolumeSize/1024/1024/1024) + 1
|
||||
estimatedRuntimeSeconds := volumeSizeGB * 5 * 60
|
||||
|
||||
return &plugin_pb.JobProposal{
|
||||
ProposalId: proposalID,
|
||||
DedupeKey: dedupeKey,
|
||||
JobType: "vacuum",
|
||||
Priority: pluginworker.MapTaskPriority(result.Priority),
|
||||
Summary: summary,
|
||||
Detail: strings.TrimSpace(result.Reason),
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"task_params_pb": {
|
||||
Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: paramsPayload},
|
||||
},
|
||||
"volume_id": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(result.VolumeID)},
|
||||
},
|
||||
"server": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Server},
|
||||
},
|
||||
"collection": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Collection},
|
||||
},
|
||||
"estimated_runtime_seconds": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: estimatedRuntimeSeconds},
|
||||
},
|
||||
},
|
||||
Labels: map[string]string{
|
||||
"task_type": "vacuum",
|
||||
"volume_id": fmt.Sprintf("%d", result.VolumeID),
|
||||
"collection": result.Collection,
|
||||
"source_node": result.Server,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func decodeVacuumTaskParams(job *plugin_pb.JobSpec) (*worker_pb.TaskParams, error) {
|
||||
if job == nil {
|
||||
return nil, fmt.Errorf("job spec is nil")
|
||||
}
|
||||
|
||||
if payload := pluginworker.ReadBytesConfig(job.Parameters, "task_params_pb"); len(payload) > 0 {
|
||||
params := &worker_pb.TaskParams{}
|
||||
if err := proto.Unmarshal(payload, params); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal task_params_pb: %w", err)
|
||||
}
|
||||
if params.TaskId == "" {
|
||||
params.TaskId = job.JobId
|
||||
}
|
||||
return params, nil
|
||||
}
|
||||
|
||||
volumeID := pluginworker.ReadUint32Config(job.Parameters, "volume_id", 0)
|
||||
server := pluginworker.ReadStringConfig(job.Parameters, "server", "")
|
||||
collection := pluginworker.ReadStringConfig(job.Parameters, "collection", "")
|
||||
if volumeID == 0 {
|
||||
return nil, fmt.Errorf("missing volume_id in job parameters")
|
||||
}
|
||||
if strings.TrimSpace(server) == "" {
|
||||
return nil, fmt.Errorf("missing server in job parameters")
|
||||
}
|
||||
|
||||
return &worker_pb.TaskParams{
|
||||
TaskId: job.JobId,
|
||||
VolumeId: volumeID,
|
||||
Collection: collection,
|
||||
Sources: []*worker_pb.TaskSource{
|
||||
{
|
||||
Node: server,
|
||||
VolumeId: volumeID,
|
||||
},
|
||||
},
|
||||
TaskParams: &worker_pb.TaskParams_VacuumParams{
|
||||
VacuumParams: &worker_pb.VacuumTaskParams{
|
||||
GarbageThreshold: 0.3,
|
||||
BatchSize: defaultVacuumTaskBatchSize,
|
||||
VerifyChecksum: true,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
373
weed/worker/tasks/vacuum/plugin_handler_test.go
Normal file
373
weed/worker/tasks/vacuum/plugin_handler_test.go
Normal file
@@ -0,0 +1,373 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
func TestDecodeVacuumTaskParamsFromPayload(t *testing.T) {
|
||||
expected := &worker_pb.TaskParams{
|
||||
TaskId: "task-1",
|
||||
VolumeId: 42,
|
||||
Collection: "photos",
|
||||
Sources: []*worker_pb.TaskSource{
|
||||
{
|
||||
Node: "10.0.0.1:8080",
|
||||
VolumeId: 42,
|
||||
},
|
||||
},
|
||||
TaskParams: &worker_pb.TaskParams_VacuumParams{
|
||||
VacuumParams: &worker_pb.VacuumTaskParams{
|
||||
GarbageThreshold: 0.33,
|
||||
BatchSize: 500,
|
||||
VerifyChecksum: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
payload, err := proto.Marshal(expected)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal payload: %v", err)
|
||||
}
|
||||
|
||||
job := &plugin_pb.JobSpec{
|
||||
JobId: "job-from-admin",
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
|
||||
},
|
||||
}
|
||||
|
||||
actual, err := decodeVacuumTaskParams(job)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeVacuumTaskParams() err = %v", err)
|
||||
}
|
||||
if !proto.Equal(expected, actual) {
|
||||
t.Fatalf("decoded params mismatch\nexpected: %+v\nactual: %+v", expected, actual)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeVacuumTaskParamsFallback(t *testing.T) {
|
||||
job := &plugin_pb.JobSpec{
|
||||
JobId: "job-2",
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"volume_id": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7}},
|
||||
"server": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "127.0.0.1:8080"}},
|
||||
"collection": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "videos"}},
|
||||
},
|
||||
}
|
||||
|
||||
params, err := decodeVacuumTaskParams(job)
|
||||
if err != nil {
|
||||
t.Fatalf("decodeVacuumTaskParams() err = %v", err)
|
||||
}
|
||||
if params.TaskId != "job-2" || params.VolumeId != 7 || params.Collection != "videos" {
|
||||
t.Fatalf("unexpected basic params: %+v", params)
|
||||
}
|
||||
if len(params.Sources) != 1 || params.Sources[0].Node != "127.0.0.1:8080" {
|
||||
t.Fatalf("unexpected sources: %+v", params.Sources)
|
||||
}
|
||||
if params.GetVacuumParams() == nil {
|
||||
t.Fatalf("expected fallback vacuum params")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveVacuumConfigAllowsZeroValues(t *testing.T) {
|
||||
values := map[string]*plugin_pb.ConfigValue{
|
||||
"garbage_threshold": {
|
||||
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0},
|
||||
},
|
||||
}
|
||||
|
||||
cfg := deriveVacuumConfig(values)
|
||||
if cfg.GarbageThreshold != 0 {
|
||||
t.Fatalf("expected garbage_threshold 0, got %v", cfg.GarbageThreshold)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMasterAddressCandidates(t *testing.T) {
|
||||
candidates := pluginworker.MasterAddressCandidates("localhost:9333")
|
||||
if len(candidates) != 2 {
|
||||
t.Fatalf("expected 2 candidates, got %d: %v", len(candidates), candidates)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, candidate := range candidates {
|
||||
seen[candidate] = true
|
||||
}
|
||||
if !seen["localhost:9333"] {
|
||||
t.Fatalf("expected original address in candidates: %v", candidates)
|
||||
}
|
||||
if !seen["localhost:19333"] {
|
||||
t.Fatalf("expected grpc address in candidates: %v", candidates)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldSkipDetectionByInterval(t *testing.T) {
|
||||
if pluginworker.ShouldSkipDetectionByInterval(nil, 10) {
|
||||
t.Fatalf("expected false when timestamp is nil")
|
||||
}
|
||||
if pluginworker.ShouldSkipDetectionByInterval(timestamppb.Now(), 0) {
|
||||
t.Fatalf("expected false when min interval is zero")
|
||||
}
|
||||
|
||||
recent := timestamppb.New(time.Now().Add(-5 * time.Second))
|
||||
if !pluginworker.ShouldSkipDetectionByInterval(recent, 10) {
|
||||
t.Fatalf("expected true for recent successful run")
|
||||
}
|
||||
|
||||
old := timestamppb.New(time.Now().Add(-30 * time.Second))
|
||||
if pluginworker.ShouldSkipDetectionByInterval(old, 10) {
|
||||
t.Fatalf("expected false for old successful run")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVacuumHandlerRejectsUnsupportedJobType(t *testing.T) {
|
||||
handler := NewVacuumHandler(nil, 0)
|
||||
err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
|
||||
JobType: "balance",
|
||||
}, noopDetectionSender{})
|
||||
if err == nil {
|
||||
t.Fatalf("expected detect job type mismatch error")
|
||||
}
|
||||
|
||||
err = handler.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
|
||||
Job: &plugin_pb.JobSpec{JobId: "job-1", JobType: "balance"},
|
||||
}, noopExecutionSender{})
|
||||
if err == nil {
|
||||
t.Fatalf("expected execute job type mismatch error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildExecutorActivity(t *testing.T) {
|
||||
activity := pluginworker.BuildExecutorActivity("running", "vacuum in progress")
|
||||
if activity == nil {
|
||||
t.Fatalf("expected non-nil activity")
|
||||
}
|
||||
if activity.Source != plugin_pb.ActivitySource_ACTIVITY_SOURCE_EXECUTOR {
|
||||
t.Fatalf("unexpected source: %v", activity.Source)
|
||||
}
|
||||
if activity.Stage != "running" {
|
||||
t.Fatalf("unexpected stage: %q", activity.Stage)
|
||||
}
|
||||
if activity.Message != "vacuum in progress" {
|
||||
t.Fatalf("unexpected message: %q", activity.Message)
|
||||
}
|
||||
if activity.CreatedAt == nil {
|
||||
t.Fatalf("expected created_at timestamp")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmitVacuumDetectionDecisionTraceNoTasks(t *testing.T) {
|
||||
sender := &recordingDetectionSender{}
|
||||
config := NewDefaultConfig()
|
||||
config.GarbageThreshold = 0.3
|
||||
config.MinVolumeAgeSeconds = 0
|
||||
|
||||
metrics := []*workertypes.VolumeHealthMetrics{
|
||||
{
|
||||
VolumeID: 17,
|
||||
GarbageRatio: 0,
|
||||
Age: 218*time.Hour + 23*time.Minute,
|
||||
},
|
||||
{
|
||||
VolumeID: 16,
|
||||
GarbageRatio: 0,
|
||||
Age: 218*time.Hour + 22*time.Minute,
|
||||
},
|
||||
{
|
||||
VolumeID: 6,
|
||||
GarbageRatio: 0,
|
||||
Age: 90*time.Hour + 42*time.Minute,
|
||||
},
|
||||
}
|
||||
|
||||
if err := emitVacuumDetectionDecisionTrace(sender, metrics, config, nil); err != nil {
|
||||
t.Fatalf("emitVacuumDetectionDecisionTrace error: %v", err)
|
||||
}
|
||||
if len(sender.events) < 4 {
|
||||
t.Fatalf("expected at least 4 detection events, got %d", len(sender.events))
|
||||
}
|
||||
|
||||
if sender.events[0].Source != plugin_pb.ActivitySource_ACTIVITY_SOURCE_DETECTOR {
|
||||
t.Fatalf("expected detector source, got %v", sender.events[0].Source)
|
||||
}
|
||||
if !strings.Contains(sender.events[0].Message, "VACUUM: No tasks created for 3 volumes") {
|
||||
t.Fatalf("unexpected summary message: %q", sender.events[0].Message)
|
||||
}
|
||||
if !strings.Contains(sender.events[1].Message, "VACUUM: Volume 17: garbage=0.00%") {
|
||||
t.Fatalf("unexpected first detail message: %q", sender.events[1].Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVacuumDescriptorHasNewFields(t *testing.T) {
|
||||
handler := NewVacuumHandler(nil, 2)
|
||||
desc := handler.Descriptor()
|
||||
if desc == nil {
|
||||
t.Fatal("descriptor is nil")
|
||||
}
|
||||
adminForm := desc.AdminConfigForm
|
||||
if adminForm == nil {
|
||||
t.Fatal("admin config form is nil")
|
||||
}
|
||||
if len(adminForm.Sections) == 0 {
|
||||
t.Fatal("admin config form has no sections")
|
||||
}
|
||||
|
||||
scopeSection := adminForm.Sections[0]
|
||||
fieldNames := make(map[string]*plugin_pb.ConfigField)
|
||||
for _, f := range scopeSection.Fields {
|
||||
fieldNames[f.Name] = f
|
||||
}
|
||||
|
||||
requiredFields := []string{
|
||||
"collection_filter",
|
||||
"volume_state",
|
||||
"data_center_filter",
|
||||
"rack_filter",
|
||||
"node_filter",
|
||||
}
|
||||
for _, name := range requiredFields {
|
||||
if _, ok := fieldNames[name]; !ok {
|
||||
t.Errorf("missing field %q in admin config scope section", name)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify volume_state is an enum with correct options.
|
||||
vsField := fieldNames["volume_state"]
|
||||
if vsField.FieldType != plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_ENUM {
|
||||
t.Errorf("volume_state field type = %v, want ENUM", vsField.FieldType)
|
||||
}
|
||||
if len(vsField.Options) != 3 {
|
||||
t.Errorf("volume_state options count = %d, want 3", len(vsField.Options))
|
||||
}
|
||||
|
||||
// Verify default values exist for new fields.
|
||||
defaultKeys := []string{"volume_state", "data_center_filter", "rack_filter", "node_filter"}
|
||||
for _, key := range defaultKeys {
|
||||
if _, ok := adminForm.DefaultValues[key]; !ok {
|
||||
t.Errorf("missing default value for %q", key)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify volume_state default is ALL.
|
||||
vsDefault := adminForm.DefaultValues["volume_state"]
|
||||
if sv, ok := vsDefault.Kind.(*plugin_pb.ConfigValue_StringValue); !ok || sv.StringValue != "ALL" {
|
||||
t.Errorf("volume_state default = %v, want ALL", vsDefault)
|
||||
}
|
||||
|
||||
// Verify collection_filter description mentions ALL_COLLECTIONS.
|
||||
cfField := fieldNames["collection_filter"]
|
||||
if cfField.Placeholder != "ALL_COLLECTIONS" {
|
||||
t.Errorf("collection_filter placeholder = %q, want ALL_COLLECTIONS", cfField.Placeholder)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVacuumFiltersVolumeState(t *testing.T) {
|
||||
metrics := []*workertypes.VolumeHealthMetrics{
|
||||
{VolumeID: 1, FullnessRatio: 0.5}, // active
|
||||
{VolumeID: 2, FullnessRatio: 1.5}, // full
|
||||
{VolumeID: 3, FullnessRatio: 0.9}, // active
|
||||
{VolumeID: 4, FullnessRatio: 1.01}, // full (at threshold)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
state pluginworker.VolumeState
|
||||
wantIDs []uint32
|
||||
}{
|
||||
{"ALL returns all", pluginworker.VolumeStateAll, []uint32{1, 2, 3, 4}},
|
||||
{"ACTIVE returns writable", pluginworker.VolumeStateActive, []uint32{1, 3}},
|
||||
{"FULL returns read-only", pluginworker.VolumeStateFull, []uint32{2, 4}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
filtered := pluginworker.FilterMetricsByVolumeState(metrics, tt.state)
|
||||
if len(filtered) != len(tt.wantIDs) {
|
||||
t.Fatalf("got %d metrics, want %d", len(filtered), len(tt.wantIDs))
|
||||
}
|
||||
for i, m := range filtered {
|
||||
if m.VolumeID != tt.wantIDs[i] {
|
||||
t.Errorf("filtered[%d].VolumeID = %d, want %d", i, m.VolumeID, tt.wantIDs[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVacuumFiltersLocation(t *testing.T) {
|
||||
metrics := []*workertypes.VolumeHealthMetrics{
|
||||
{VolumeID: 1, DataCenter: "dc1", Rack: "r1", Server: "s1"},
|
||||
{VolumeID: 2, DataCenter: "dc1", Rack: "r2", Server: "s2"},
|
||||
{VolumeID: 3, DataCenter: "dc2", Rack: "r1", Server: "s3"},
|
||||
{VolumeID: 4, DataCenter: "dc2", Rack: "r3", Server: "s4"},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
dc string
|
||||
rack string
|
||||
node string
|
||||
wantIDs []uint32
|
||||
}{
|
||||
{"dc filter", "dc1", "", "", []uint32{1, 2}},
|
||||
{"rack filter", "", "r1", "", []uint32{1, 3}},
|
||||
{"node filter", "", "", "s2,s4", []uint32{2, 4}},
|
||||
{"dc + rack", "dc1", "r1", "", []uint32{1}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
filtered := pluginworker.FilterMetricsByLocation(metrics, tt.dc, tt.rack, tt.node)
|
||||
if len(filtered) != len(tt.wantIDs) {
|
||||
t.Fatalf("got %d metrics, want %d", len(filtered), len(tt.wantIDs))
|
||||
}
|
||||
for i, m := range filtered {
|
||||
if m.VolumeID != tt.wantIDs[i] {
|
||||
t.Errorf("filtered[%d].VolumeID = %d, want %d", i, m.VolumeID, tt.wantIDs[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type noopDetectionSender struct{}
|
||||
|
||||
func (noopDetectionSender) SendProposals(*plugin_pb.DetectionProposals) error { return nil }
|
||||
func (noopDetectionSender) SendComplete(*plugin_pb.DetectionComplete) error { return nil }
|
||||
func (noopDetectionSender) SendActivity(*plugin_pb.ActivityEvent) error { return nil }
|
||||
|
||||
type noopExecutionSender struct{}
|
||||
|
||||
func (noopExecutionSender) SendProgress(*plugin_pb.JobProgressUpdate) error { return nil }
|
||||
func (noopExecutionSender) SendCompleted(*plugin_pb.JobCompleted) error { return nil }
|
||||
|
||||
type recordingDetectionSender struct {
|
||||
proposals *plugin_pb.DetectionProposals
|
||||
complete *plugin_pb.DetectionComplete
|
||||
events []*plugin_pb.ActivityEvent
|
||||
}
|
||||
|
||||
func (r *recordingDetectionSender) SendProposals(proposals *plugin_pb.DetectionProposals) error {
|
||||
r.proposals = proposals
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recordingDetectionSender) SendComplete(complete *plugin_pb.DetectionComplete) error {
|
||||
r.complete = complete
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recordingDetectionSender) SendActivity(event *plugin_pb.ActivityEvent) error {
|
||||
if event != nil {
|
||||
r.events = append(r.events, event)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
98
weed/worker/tasks/vacuum/register.go
Normal file
98
weed/worker/tasks/vacuum/register.go
Normal file
@@ -0,0 +1,98 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// Global variable to hold the task definition for configuration updates
|
||||
var globalTaskDef *base.TaskDefinition
|
||||
|
||||
// Auto-register this task when the package is imported
|
||||
func init() {
|
||||
RegisterVacuumTask()
|
||||
|
||||
// Register config updater
|
||||
tasks.AutoRegisterConfigUpdater(types.TaskTypeVacuum, UpdateConfigFromPersistence)
|
||||
}
|
||||
|
||||
// RegisterVacuumTask registers the vacuum task with the new architecture
|
||||
func RegisterVacuumTask() {
|
||||
// Create configuration instance
|
||||
config := NewDefaultConfig()
|
||||
|
||||
// Create shared gRPC dial option using TLS configuration
|
||||
dialOpt := security.LoadClientTLS(util.GetViper(), "grpc.worker")
|
||||
|
||||
// Create complete task definition
|
||||
taskDef := &base.TaskDefinition{
|
||||
Type: types.TaskTypeVacuum,
|
||||
Name: "vacuum",
|
||||
DisplayName: "Volume Vacuum",
|
||||
Description: "Reclaims disk space by removing deleted files from volumes",
|
||||
Icon: "fas fa-broom text-primary",
|
||||
Capabilities: []string{"vacuum", "storage"},
|
||||
|
||||
Config: config,
|
||||
ConfigSpec: GetConfigSpec(),
|
||||
CreateTask: func(params *worker_pb.TaskParams) (types.Task, error) {
|
||||
if params == nil {
|
||||
return nil, fmt.Errorf("task parameters are required")
|
||||
}
|
||||
if len(params.Sources) == 0 {
|
||||
return nil, fmt.Errorf("at least one source is required for vacuum task")
|
||||
}
|
||||
servers := make([]string, 0, len(params.Sources))
|
||||
for _, src := range params.Sources {
|
||||
if src != nil && src.Node != "" {
|
||||
servers = append(servers, src.Node)
|
||||
}
|
||||
}
|
||||
return NewVacuumTask(
|
||||
fmt.Sprintf("vacuum-%d", params.VolumeId),
|
||||
servers,
|
||||
params.VolumeId,
|
||||
params.Collection,
|
||||
dialOpt,
|
||||
), nil
|
||||
},
|
||||
DetectionFunc: Detection,
|
||||
ScanInterval: 2 * time.Hour,
|
||||
SchedulingFunc: Scheduling,
|
||||
MaxConcurrent: 2,
|
||||
RepeatInterval: 7 * 24 * time.Hour,
|
||||
}
|
||||
|
||||
// Store task definition globally for configuration updates
|
||||
globalTaskDef = taskDef
|
||||
|
||||
// Register everything with a single function call!
|
||||
base.RegisterTask(taskDef)
|
||||
}
|
||||
|
||||
// UpdateConfigFromPersistence updates the vacuum configuration from persistence
|
||||
func UpdateConfigFromPersistence(configPersistence interface{}) error {
|
||||
if globalTaskDef == nil {
|
||||
return fmt.Errorf("vacuum task not registered")
|
||||
}
|
||||
|
||||
// Load configuration from persistence
|
||||
newConfig := LoadConfigFromPersistence(configPersistence)
|
||||
if newConfig == nil {
|
||||
return fmt.Errorf("failed to load configuration from persistence")
|
||||
}
|
||||
|
||||
// Update the task definition's config
|
||||
globalTaskDef.Config = newConfig
|
||||
|
||||
glog.V(1).Infof("Updated vacuum task configuration from persistence")
|
||||
return nil
|
||||
}
|
||||
37
weed/worker/tasks/vacuum/scheduling.go
Normal file
37
weed/worker/tasks/vacuum/scheduling.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// Scheduling implements the scheduling logic for vacuum tasks
|
||||
func Scheduling(task *types.TaskInput, runningTasks []*types.TaskInput, availableWorkers []*types.WorkerData, config base.TaskConfig) bool {
|
||||
vacuumConfig := config.(*Config)
|
||||
|
||||
// Count running vacuum tasks
|
||||
runningVacuumCount := 0
|
||||
for _, runningTask := range runningTasks {
|
||||
if runningTask.Type == types.TaskTypeVacuum {
|
||||
runningVacuumCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Check concurrency limit
|
||||
if runningVacuumCount >= vacuumConfig.MaxConcurrent {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for available workers with vacuum capability
|
||||
for _, worker := range availableWorkers {
|
||||
if worker.CurrentLoad < worker.MaxConcurrent {
|
||||
for _, capability := range worker.Capabilities {
|
||||
if capability == types.TaskTypeVacuum {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
444
weed/worker/tasks/vacuum/vacuum_task.go
Normal file
444
weed/worker/tasks/vacuum/vacuum_task.go
Normal file
@@ -0,0 +1,444 @@
|
||||
package vacuum
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types/base"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
// VacuumTask implements the Task interface.
|
||||
//
|
||||
// One task covers all replicas of a volume so behavior matches the master
|
||||
// built-in vacuum (see topology.Topology.vacuumOneVolumeId): Check across
|
||||
// every replica → filter to those whose garbage ratio meets the threshold
|
||||
// → Compact/Commit/Cleanup that subset. Treating one replica per task (the
|
||||
// prior behavior) drops the other N-1 replicas because the dispatcher
|
||||
// gates duplicate tasks per volume via ActiveTopology.HasAnyTask.
|
||||
type VacuumTask struct {
|
||||
*base.BaseTask
|
||||
servers []string
|
||||
volumeID uint32
|
||||
collection string
|
||||
garbageThreshold float64
|
||||
progress float64
|
||||
grpcDialOption grpc.DialOption
|
||||
volumeSize uint64
|
||||
vacuumTargets []string // populated by checkVacuumEligibility — subset of servers that pass the per-replica garbage re-check and proceed to Compact/Commit/Cleanup
|
||||
}
|
||||
|
||||
// NewVacuumTask creates a new unified vacuum task instance covering every
|
||||
// replica server reported by the dispatcher.
|
||||
func NewVacuumTask(id string, servers []string, volumeID uint32, collection string, grpcDialOption grpc.DialOption) *VacuumTask {
|
||||
deduped := dedupePreserveOrder(servers)
|
||||
return &VacuumTask{
|
||||
BaseTask: base.NewBaseTask(id, types.TaskTypeVacuum),
|
||||
servers: deduped,
|
||||
volumeID: volumeID,
|
||||
collection: collection,
|
||||
garbageThreshold: 0.3, // Default 30% threshold
|
||||
grpcDialOption: grpcDialOption,
|
||||
}
|
||||
}
|
||||
|
||||
// Execute implements the UnifiedTask interface
|
||||
func (t *VacuumTask) Execute(ctx context.Context, params *worker_pb.TaskParams) error {
|
||||
if params == nil {
|
||||
return fmt.Errorf("task parameters are required")
|
||||
}
|
||||
|
||||
vacuumParams := params.GetVacuumParams()
|
||||
if vacuumParams == nil {
|
||||
return fmt.Errorf("vacuum parameters are required")
|
||||
}
|
||||
|
||||
t.garbageThreshold = vacuumParams.GarbageThreshold
|
||||
t.volumeSize = params.VolumeSize
|
||||
|
||||
t.GetLogger().WithFields(map[string]interface{}{
|
||||
"volume_id": t.volumeID,
|
||||
"servers": t.servers,
|
||||
"collection": t.collection,
|
||||
"garbage_threshold": t.garbageThreshold,
|
||||
}).Info("Starting vacuum task")
|
||||
|
||||
if len(t.servers) == 0 {
|
||||
return fmt.Errorf("no source servers configured for vacuum task")
|
||||
}
|
||||
|
||||
// Step 1: Check vacuum eligibility for each replica. Mirrors
|
||||
// topology.batchVacuumVolumeCheck — only replicas whose garbage is at
|
||||
// or above the threshold proceed to Compact/Commit/Cleanup.
|
||||
t.ReportProgress(10.0)
|
||||
t.GetLogger().Info("Checking volume status")
|
||||
targets, currentGarbageRatios, err := t.checkVacuumEligibility(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check vacuum eligibility: %w", err)
|
||||
}
|
||||
|
||||
if len(targets) == 0 {
|
||||
t.GetLogger().WithFields(map[string]interface{}{
|
||||
"garbage_ratios": currentGarbageRatios,
|
||||
"required_threshold": t.garbageThreshold,
|
||||
}).Info("No replica meets vacuum criteria, skipping")
|
||||
t.ReportProgress(100.0)
|
||||
return nil
|
||||
}
|
||||
t.vacuumTargets = targets
|
||||
|
||||
// Step 2: Perform vacuum (compact + commit + cleanup) across every
|
||||
// target replica.
|
||||
t.ReportProgress(50.0)
|
||||
t.GetLogger().WithFields(map[string]interface{}{
|
||||
"vacuum_targets": targets,
|
||||
"garbage_ratios": currentGarbageRatios,
|
||||
"threshold": t.garbageThreshold,
|
||||
}).Info("Performing vacuum operation")
|
||||
|
||||
if err := t.performVacuum(ctx); err != nil {
|
||||
return fmt.Errorf("failed to perform vacuum: %w", err)
|
||||
}
|
||||
|
||||
// Step 3: Verify vacuum results on each target replica.
|
||||
t.ReportProgress(90.0)
|
||||
t.GetLogger().Info("Verifying vacuum results")
|
||||
if err := t.verifyVacuumResults(ctx); err != nil {
|
||||
glog.Warningf("Vacuum verification failed: %v", err)
|
||||
// Don't fail the task - vacuum operation itself succeeded
|
||||
}
|
||||
|
||||
t.ReportProgress(100.0)
|
||||
glog.Infof("Vacuum task completed successfully: volume %d on %v (garbage ratios %v)",
|
||||
t.volumeID, targets, currentGarbageRatios)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate implements the UnifiedTask interface
|
||||
func (t *VacuumTask) Validate(params *worker_pb.TaskParams) error {
|
||||
if params == nil {
|
||||
return fmt.Errorf("task parameters are required")
|
||||
}
|
||||
|
||||
vacuumParams := params.GetVacuumParams()
|
||||
if vacuumParams == nil {
|
||||
return fmt.Errorf("vacuum parameters are required")
|
||||
}
|
||||
|
||||
if params.VolumeId != t.volumeID {
|
||||
return fmt.Errorf("volume ID mismatch: expected %d, got %d", t.volumeID, params.VolumeId)
|
||||
}
|
||||
|
||||
// Every server the task was created with must appear in the params'
|
||||
// Sources list. The dispatcher fills Sources from the detection-time
|
||||
// replica set, so a mismatch means the worker received stale routing.
|
||||
sourceSet := make(map[string]struct{}, len(params.Sources))
|
||||
for _, source := range params.Sources {
|
||||
if source == nil {
|
||||
continue
|
||||
}
|
||||
sourceSet[source.Node] = struct{}{}
|
||||
}
|
||||
for _, server := range t.servers {
|
||||
if _, ok := sourceSet[server]; !ok {
|
||||
return fmt.Errorf("task server %s not present in params.Sources", server)
|
||||
}
|
||||
}
|
||||
|
||||
if vacuumParams.GarbageThreshold < 0 || vacuumParams.GarbageThreshold > 1.0 {
|
||||
return fmt.Errorf("invalid garbage threshold: %f (must be between 0.0 and 1.0)", vacuumParams.GarbageThreshold)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// EstimateTime implements the UnifiedTask interface
|
||||
func (t *VacuumTask) EstimateTime(params *worker_pb.TaskParams) time.Duration {
|
||||
// Basic estimate based on simulated steps
|
||||
return 14 * time.Second // Sum of all step durations
|
||||
}
|
||||
|
||||
// GetProgress returns current progress
|
||||
func (t *VacuumTask) GetProgress() float64 {
|
||||
return t.progress
|
||||
}
|
||||
|
||||
// vacuumTimeout returns a dynamic timeout scaled by volume size, matching the
|
||||
// topology vacuum approach. base is the per-GB multiplier (e.g. 1 minute for
|
||||
// check, 3 minutes for compact).
|
||||
func (t *VacuumTask) vacuumTimeout(base time.Duration) time.Duration {
|
||||
if t.volumeSize == 0 {
|
||||
glog.V(1).Infof("volume %d has no size metric, using minimum timeout", t.volumeID)
|
||||
}
|
||||
sizeGB := int64(t.volumeSize/1024/1024/1024) + 1
|
||||
return base * time.Duration(sizeGB)
|
||||
}
|
||||
|
||||
// Helper methods for real vacuum operations
|
||||
|
||||
// checkVacuumEligibility queries every replica's current garbage ratio.
|
||||
// Mirrors topology.batchVacuumVolumeCheck's all-or-nothing contract: if
|
||||
// any replica's check fails (unreachable, RPC error, etc.) the entire
|
||||
// task aborts. Vacuuming only the replicas that responded while
|
||||
// silently skipping unreachable ones would compact the responders'
|
||||
// garbage but leave the unreachable replica still carrying it,
|
||||
// producing divergence the moment that replica comes back.
|
||||
//
|
||||
// Returns the subset of servers whose garbage is at or above the
|
||||
// configured threshold, alongside a per-server ratio map for logging.
|
||||
func (t *VacuumTask) checkVacuumEligibility(ctx context.Context) ([]string, map[string]float64, error) {
|
||||
ratios := make(map[string]float64, len(t.servers))
|
||||
for _, server := range t.servers {
|
||||
ratio, err := t.checkOneVacuumEligibility(ctx, server)
|
||||
if err != nil {
|
||||
return nil, ratios, fmt.Errorf("vacuum check on %s for volume %d: %w (aborting; refusing to vacuum subset of replicas)", server, t.volumeID, err)
|
||||
}
|
||||
ratios[server] = ratio
|
||||
glog.V(1).Infof("Volume %d on %s garbage ratio: %.2f%%, threshold: %.2f%%",
|
||||
t.volumeID, server, ratio*100, t.garbageThreshold*100)
|
||||
}
|
||||
|
||||
eligible := make([]string, 0, len(ratios))
|
||||
for _, server := range t.servers {
|
||||
if ratios[server] >= t.garbageThreshold {
|
||||
eligible = append(eligible, server)
|
||||
}
|
||||
}
|
||||
return eligible, ratios, nil
|
||||
}
|
||||
|
||||
func (t *VacuumTask) checkOneVacuumEligibility(ctx context.Context, server string) (float64, error) {
|
||||
var garbageRatio float64
|
||||
err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), t.grpcDialOption,
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
checkCtx, cancel := context.WithTimeout(ctx, t.vacuumTimeout(time.Minute))
|
||||
defer cancel()
|
||||
resp, err := client.VacuumVolumeCheck(checkCtx, &volume_server_pb.VacuumVolumeCheckRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check volume vacuum status: %v", err)
|
||||
}
|
||||
garbageRatio = resp.GarbageRatio
|
||||
return nil
|
||||
})
|
||||
return garbageRatio, err
|
||||
}
|
||||
|
||||
// performVacuum runs the three-phase vacuum protocol that mirrors
|
||||
// master built-in vacuum (topology.vacuumOneVolumeId):
|
||||
//
|
||||
// Phase 1 (Compact): build the new .cpd/.cpx files on every target.
|
||||
// If any replica fails, roll back by Cleanup'ing the .cp* temp files
|
||||
// on every target and abort — no replica has yet swapped its active
|
||||
// files, so no replica is committed.
|
||||
//
|
||||
// Phase 2 (Commit): swap each target's active files with its .cp*
|
||||
// files. Best-effort, matching batchVacuumVolumeCommit: per-replica
|
||||
// errors are logged and surfaced together, but once any replica has
|
||||
// swapped there is no clean rollback for the others, so we do not
|
||||
// retry or undo. An operator must reconcile a partial commit
|
||||
// failure.
|
||||
//
|
||||
// Phase 3 (Mark Writable): re-notify the master per replica so the
|
||||
// volume re-enters the writable set, the worker analog of
|
||||
// batchVacuumVolumeCommit's per-replica SetVolumeAvailable. Skipped
|
||||
// when a replica came back read-only. Best-effort; never fails the
|
||||
// task.
|
||||
//
|
||||
// Interleaving Compact→Commit→Cleanup per replica (the prior behavior)
|
||||
// could leave a committed first replica beside an uncompacted second
|
||||
// replica when Compact on the second failed — replica divergence with
|
||||
// no automatic recovery.
|
||||
func (t *VacuumTask) performVacuum(ctx context.Context) error {
|
||||
// Phase 1: Compact all targets.
|
||||
for _, server := range t.vacuumTargets {
|
||||
if err := t.compactOne(ctx, server); err != nil {
|
||||
t.cleanupAll(ctx)
|
||||
return fmt.Errorf("vacuum compact on %s: %w", server, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 2: Commit all targets, tracking whether any replica is still
|
||||
// read-only after the swap.
|
||||
var commitErrors []error
|
||||
anyReadOnly := false
|
||||
for _, server := range t.vacuumTargets {
|
||||
resp, err := t.commitOne(ctx, server)
|
||||
if err != nil {
|
||||
glog.Errorf("vacuum commit on %s for volume %d: %v", server, t.volumeID, err)
|
||||
commitErrors = append(commitErrors, fmt.Errorf("%s: %w", server, err))
|
||||
continue
|
||||
}
|
||||
if resp.GetIsReadOnly() {
|
||||
anyReadOnly = true
|
||||
}
|
||||
}
|
||||
if len(commitErrors) > 0 {
|
||||
return fmt.Errorf("vacuum commit failed on %d/%d replicas: %v",
|
||||
len(commitErrors), len(t.vacuumTargets), commitErrors)
|
||||
}
|
||||
|
||||
// Phase 3: re-notify the master so the volume re-enters the writable
|
||||
// set. The worker's only lever is VolumeMarkWritable, which clears the
|
||||
// read-only flag and runs notifyMasterVolumeReadonly(false). Gate on
|
||||
// the commit's IsReadOnly exactly as SetVolumeAvailable does: a replica
|
||||
// still read-only (operator-set, EIO-quarantined, or disk-space-low)
|
||||
// must stay out, and recovers on its own via the next ReadOnly=false
|
||||
// heartbeat — force-clearing the flag here would override that. The
|
||||
// worker is not told the master's size limit, so the isFullCapacity
|
||||
// guard is left to the next capacity heartbeat. Best-effort: the vacuum
|
||||
// itself already succeeded.
|
||||
if anyReadOnly {
|
||||
glog.V(0).Infof("post-vacuum: volume %d still read-only on a replica, leaving it out of writables", t.volumeID)
|
||||
return nil
|
||||
}
|
||||
for _, server := range t.vacuumTargets {
|
||||
if err := t.markWritableOne(ctx, server); err != nil {
|
||||
glog.Warningf("post-vacuum mark writable on %s for volume %d: %v", server, t.volumeID, err)
|
||||
continue
|
||||
}
|
||||
glog.V(0).Infof("post-vacuum marked volume %d writable on %s", t.volumeID, server)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *VacuumTask) compactOne(ctx context.Context, server string) error {
|
||||
return operation.WithVolumeServerClient(false, pb.ServerAddress(server), t.grpcDialOption,
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
t.GetLogger().Info("Compacting volume on %s", server)
|
||||
compactCtx, cancel := context.WithTimeout(ctx, t.vacuumTimeout(3*time.Minute))
|
||||
defer cancel()
|
||||
stream, err := client.VacuumVolumeCompact(compactCtx, &volume_server_pb.VacuumVolumeCompactRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("vacuum compact start: %v", err)
|
||||
}
|
||||
for {
|
||||
resp, recvErr := stream.Recv()
|
||||
if recvErr != nil {
|
||||
if recvErr == io.EOF {
|
||||
break
|
||||
}
|
||||
return fmt.Errorf("vacuum compact stream: %v", recvErr)
|
||||
}
|
||||
glog.V(2).Infof("Volume %d on %s compact progress: %d bytes", t.volumeID, server, resp.ProcessedBytes)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (t *VacuumTask) commitOne(ctx context.Context, server string) (*volume_server_pb.VacuumVolumeCommitResponse, error) {
|
||||
var resp *volume_server_pb.VacuumVolumeCommitResponse
|
||||
err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), t.grpcDialOption,
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
t.GetLogger().Info("Committing vacuum on %s", server)
|
||||
commitCtx, cancel := context.WithTimeout(ctx, t.vacuumTimeout(time.Minute))
|
||||
defer cancel()
|
||||
var err error
|
||||
resp, err = client.VacuumVolumeCommit(commitCtx, &volume_server_pb.VacuumVolumeCommitRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("vacuum commit: %v", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func (t *VacuumTask) cleanupOne(ctx context.Context, server string) error {
|
||||
return operation.WithVolumeServerClient(false, pb.ServerAddress(server), t.grpcDialOption,
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
cleanupCtx, cancel := context.WithTimeout(ctx, t.vacuumTimeout(time.Minute))
|
||||
defer cancel()
|
||||
_, err := client.VacuumVolumeCleanup(cleanupCtx, &volume_server_pb.VacuumVolumeCleanupRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func (t *VacuumTask) markWritableOne(ctx context.Context, server string) error {
|
||||
return operation.WithVolumeServerClient(false, pb.ServerAddress(server), t.grpcDialOption,
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
// VolumeMarkWritable is a metadata RPC (reopen idx + flags +
|
||||
// notifyMasterVolumeReadonly heartbeat) — millisecond-scale and
|
||||
// independent of volume size. A flat 1m cap prevents an
|
||||
// unresponsive replica from blocking Phase 3 for hours on a
|
||||
// TB-scale volume where vacuumTimeout() would balloon.
|
||||
markCtx, cancel := context.WithTimeout(ctx, time.Minute)
|
||||
defer cancel()
|
||||
_, err := client.VolumeMarkWritable(markCtx, &volume_server_pb.VolumeMarkWritableRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// cleanupAll removes the .cpd/.cpx/.cpldb temp files on every target.
|
||||
// Used to roll back when Compact fails on one replica after others
|
||||
// have already created their temp files. Per-target failures are
|
||||
// logged but never bubble up — the rollback is best-effort.
|
||||
func (t *VacuumTask) cleanupAll(ctx context.Context) {
|
||||
for _, server := range t.vacuumTargets {
|
||||
if err := t.cleanupOne(ctx, server); err != nil {
|
||||
glog.Warningf("rollback cleanup on %s for volume %d: %v", server, t.volumeID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// verifyVacuumResults checks each target replica's post-vacuum garbage
|
||||
// ratio. Failures are logged at WARN — the task does not fail because the
|
||||
// vacuum itself already succeeded.
|
||||
func (t *VacuumTask) verifyVacuumResults(ctx context.Context) error {
|
||||
for _, server := range t.vacuumTargets {
|
||||
err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), t.grpcDialOption,
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
verifyCtx, cancel := context.WithTimeout(ctx, t.vacuumTimeout(time.Minute))
|
||||
defer cancel()
|
||||
resp, err := client.VacuumVolumeCheck(verifyCtx, &volume_server_pb.VacuumVolumeCheckRequest{
|
||||
VolumeId: t.volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to verify vacuum results: %v", err)
|
||||
}
|
||||
glog.V(1).Infof("Volume %d on %s post-vacuum garbage ratio: %.2f%%",
|
||||
t.volumeID, server, resp.GarbageRatio*100)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
glog.Warningf("post-vacuum verify on %s: %v", server, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// dedupePreserveOrder returns servers with duplicates removed, keeping the
|
||||
// first occurrence's position. Detection sometimes hands the same node
|
||||
// address in multiple Sources (e.g. EC variants); we coalesce them so each
|
||||
// physical replica is vacuumed exactly once.
|
||||
func dedupePreserveOrder(servers []string) []string {
|
||||
seen := make(map[string]struct{}, len(servers))
|
||||
out := make([]string, 0, len(servers))
|
||||
for _, s := range servers {
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[s]; ok {
|
||||
continue
|
||||
}
|
||||
seen[s] = struct{}{}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user