snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,183 @@
package vacuum
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/admin/config"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
)
// Config extends BaseConfig with vacuum-specific settings
type Config struct {
base.BaseConfig
GarbageThreshold float64 `json:"garbage_threshold"`
MinVolumeAgeSeconds int `json:"min_volume_age_seconds"`
}
// NewDefaultConfig creates a new default vacuum configuration
func NewDefaultConfig() *Config {
return &Config{
BaseConfig: base.BaseConfig{
Enabled: true,
ScanIntervalSeconds: 2 * 60 * 60, // 2 hours
MaxConcurrent: 2,
},
GarbageThreshold: 0.3, // 30%
MinVolumeAgeSeconds: 24 * 60 * 60, // 24 hours
}
}
// ToTaskPolicy converts configuration to a TaskPolicy protobuf message
func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
return &worker_pb.TaskPolicy{
Enabled: c.Enabled,
MaxConcurrent: int32(c.MaxConcurrent),
RepeatIntervalSeconds: int32(c.ScanIntervalSeconds),
CheckIntervalSeconds: int32(c.ScanIntervalSeconds),
TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
VacuumConfig: &worker_pb.VacuumTaskConfig{
GarbageThreshold: float64(c.GarbageThreshold),
MinVolumeAgeHours: int32((c.MinVolumeAgeSeconds + 3599) / 3600), // round up so sub-hour values don't become 0
},
},
}
}
// FromTaskPolicy loads configuration from a TaskPolicy protobuf message
func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
if policy == nil {
return fmt.Errorf("policy is nil")
}
// Set general TaskPolicy fields
c.Enabled = policy.Enabled
c.MaxConcurrent = int(policy.MaxConcurrent)
c.ScanIntervalSeconds = int(policy.RepeatIntervalSeconds) // Direct seconds-to-seconds mapping
// Set vacuum-specific fields from the task config
if vacuumConfig := policy.GetVacuumConfig(); vacuumConfig != nil {
c.GarbageThreshold = float64(vacuumConfig.GarbageThreshold)
c.MinVolumeAgeSeconds = int(vacuumConfig.MinVolumeAgeHours * 3600) // Convert hours to seconds
}
return nil
}
// LoadConfigFromPersistence loads configuration from the persistence layer if available
func LoadConfigFromPersistence(configPersistence interface{}) *Config {
config := NewDefaultConfig()
// Try to load from persistence if available
if persistence, ok := configPersistence.(interface {
LoadVacuumTaskPolicy() (*worker_pb.TaskPolicy, error)
}); ok {
policy, err := persistence.LoadVacuumTaskPolicy()
switch {
case err != nil:
glog.Warningf("Could not read the persisted vacuum configuration, falling back to defaults: %v", err)
case policy == nil:
glog.V(1).Infof("No vacuum configuration persisted yet, using defaults")
default:
if err := config.FromTaskPolicy(policy); err != nil {
glog.Warningf("Could not apply the persisted vacuum configuration, falling back to defaults: %v", err)
} else {
glog.V(1).Infof("Loaded vacuum configuration from persistence")
return config
}
}
} else if configPersistence != nil {
// A store was handed in but does not expose the accessor, so the persisted
// settings are silently ignored - always a wiring bug, never a normal state.
glog.Warningf("%T cannot provide the persisted vacuum configuration: it has no LoadVacuumTaskPolicy() method, "+
"so the compiled-in defaults are used and any saved vacuum settings are ignored", configPersistence)
}
glog.V(1).Infof("Using default vacuum configuration")
return config
}
// GetConfigSpec returns the configuration schema for vacuum tasks
func GetConfigSpec() base.ConfigSpec {
return base.ConfigSpec{
Fields: []*config.Field{
{
Name: "enabled",
JSONName: "enabled",
Type: config.FieldTypeBool,
DefaultValue: true,
Required: false,
DisplayName: "Enable Vacuum Tasks",
Description: "Whether vacuum tasks should be automatically created",
HelpText: "Toggle this to enable or disable automatic vacuum task generation",
InputType: "checkbox",
CSSClasses: "form-check-input",
},
{
Name: "scan_interval_seconds",
JSONName: "scan_interval_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 2 * 60 * 60,
MinValue: 10 * 60,
MaxValue: 24 * 60 * 60,
Required: true,
DisplayName: "Scan Interval",
Description: "How often to scan for volumes needing vacuum",
HelpText: "The system will check for volumes that need vacuuming at this interval",
Placeholder: "2",
Unit: config.UnitHours,
InputType: "interval",
CSSClasses: "form-control",
},
{
Name: "max_concurrent",
JSONName: "max_concurrent",
Type: config.FieldTypeInt,
DefaultValue: 2,
MinValue: 1,
MaxValue: 10,
Required: true,
DisplayName: "Max Concurrent Tasks",
Description: "Maximum number of vacuum tasks that can run simultaneously",
HelpText: "Limits the number of vacuum operations running at the same time to control system load",
Placeholder: "2 (default)",
Unit: config.UnitCount,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "garbage_threshold",
JSONName: "garbage_threshold",
Type: config.FieldTypeFloat,
DefaultValue: 0.3,
MinValue: 0.0,
MaxValue: 1.0,
Required: true,
DisplayName: "Garbage Percentage Threshold",
Description: "Trigger vacuum when garbage ratio exceeds this percentage",
HelpText: "Volumes with more deleted content than this threshold will be vacuumed",
Placeholder: "0.30 (30%)",
Unit: config.UnitNone,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "min_volume_age_seconds",
JSONName: "min_volume_age_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 24 * 60 * 60,
MinValue: 1 * 60 * 60,
MaxValue: 7 * 24 * 60 * 60,
Required: true,
DisplayName: "Minimum Volume Age",
Description: "Only vacuum volumes older than this duration",
HelpText: "Prevents vacuuming of recently created volumes that may still be actively written to",
Placeholder: "24",
Unit: config.UnitHours,
InputType: "interval",
CSSClasses: "form-control",
},
},
}
}

View File

@@ -0,0 +1,75 @@
package vacuum
import (
"errors"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
)
type stubVacuumStore struct {
policy *worker_pb.TaskPolicy
err error
}
func (s *stubVacuumStore) LoadVacuumTaskPolicy() (*worker_pb.TaskPolicy, error) {
return s.policy, s.err
}
// wrongShapedStore is what the maintenance manager used to be handed: a non-nil value that
// does not satisfy the accessor the loader asserts on. It has to keep falling back to the
// defaults, but no longer silently: a fallback the operator cannot see reads as a disabled task
// that keeps running.
type wrongShapedStore struct{}
func (wrongShapedStore) SomethingElse() {}
func TestLoadConfigFromPersistenceUsesPersistedPolicy(t *testing.T) {
persisted := NewDefaultConfig()
persisted.Enabled = false
persisted.GarbageThreshold = 0.75
persisted.MaxConcurrent = 5
loaded := LoadConfigFromPersistence(&stubVacuumStore{policy: persisted.ToTaskPolicy()})
if loaded == nil {
t.Fatal("LoadConfigFromPersistence returned nil")
}
if loaded.Enabled {
t.Error("enabled = true, want the persisted false")
}
if loaded.GarbageThreshold != 0.75 {
t.Errorf("garbage threshold = %v, want the persisted 0.75", loaded.GarbageThreshold)
}
if loaded.MaxConcurrent != 5 {
t.Errorf("max concurrent = %d, want the persisted 5", loaded.MaxConcurrent)
}
}
func TestLoadConfigFromPersistenceFallsBackToDefaults(t *testing.T) {
defaults := NewDefaultConfig()
cases := map[string]interface{}{
"no store configured": nil,
"store without the value": &stubVacuumStore{},
"store that errors": &stubVacuumStore{err: errors.New("disk on fire")},
"store of the wrong type": wrongShapedStore{},
}
for name, store := range cases {
t.Run(name, func(t *testing.T) {
loaded := LoadConfigFromPersistence(store)
if loaded == nil {
t.Fatal("LoadConfigFromPersistence returned nil")
}
if !loaded.Enabled {
t.Error("enabled = false, want the compiled-in default true")
}
if loaded.GarbageThreshold != defaults.GarbageThreshold {
t.Errorf("garbage threshold = %v, want the default %v", loaded.GarbageThreshold, defaults.GarbageThreshold)
}
if loaded.ScanIntervalSeconds != defaults.ScanIntervalSeconds {
t.Errorf("scan interval = %d, want the default %d", loaded.ScanIntervalSeconds, defaults.ScanIntervalSeconds)
}
})
}
}

View File

@@ -0,0 +1,227 @@
package vacuum
import (
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/util"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// Detection implements the detection logic for vacuum tasks
func Detection(metrics []*types.VolumeHealthMetrics, clusterInfo *types.ClusterInfo, config base.TaskConfig) ([]*types.TaskDetectionResult, error) {
if !config.IsEnabled() {
return nil, nil
}
vacuumConfig := config.(*Config)
minVolumeAge := time.Duration(vacuumConfig.MinVolumeAgeSeconds) * time.Second
type volumeBucket struct {
replicas []*types.VolumeHealthMetrics
}
buckets := make(map[uint32]*volumeBucket)
order := make([]uint32, 0)
for _, m := range metrics {
if m == nil {
continue
}
b, ok := buckets[m.VolumeID]
if !ok {
b = &volumeBucket{}
buckets[m.VolumeID] = b
order = append(order, m.VolumeID)
}
b.replicas = append(b.replicas, m)
}
var results []*types.TaskDetectionResult
skippedDueToGarbage := 0
skippedDueToAge := 0
debugCount := 0
for _, vid := range order {
replicas := buckets[vid].replicas
// Pick the replica with the highest garbage ratio that also satisfies
// the minimum-age gate as the "primary" used for priority + reason.
// Master built-in vacuum filters per-replica again inside
// batchVacuumVolumeCheck, so it's fine if only some replicas are
// above threshold here — the worker's checkVacuumEligibility will
// filter again at execute time.
primary := pickPrimaryReplica(replicas, vacuumConfig.GarbageThreshold, minVolumeAge)
if primary == nil {
recordSkipReasons(replicas, vacuumConfig.GarbageThreshold, minVolumeAge, &skippedDueToGarbage, &skippedDueToAge, &debugCount)
continue
}
priority := types.TaskPriorityNormal
if primary.GarbageRatio > 0.6 {
priority = types.TaskPriorityHigh
}
taskID := fmt.Sprintf("vacuum_vol_%d_%d", vid, time.Now().Unix())
result := &types.TaskDetectionResult{
TaskID: taskID,
TaskType: types.TaskTypeVacuum,
VolumeID: vid,
Server: primary.Server,
Collection: primary.Collection,
Priority: priority,
Reason: "Volume has excessive garbage requiring vacuum",
ScheduleAt: time.Now(),
}
// Check if ANY task already exists in ActiveTopology for this volume
if clusterInfo != nil && clusterInfo.ActiveTopology != nil {
if clusterInfo.ActiveTopology.HasAnyTask(vid) {
glog.V(2).Infof("VACUUM: Skipping volume %d, task already exists in ActiveTopology", vid)
continue
}
}
result.TypedParams = createVacuumTaskParams(result, replicas, vacuumConfig, clusterInfo)
if result.TypedParams != nil {
results = append(results, result)
}
}
if len(results) == 0 && len(metrics) > 0 {
totalVolumes := len(metrics)
glog.V(1).Infof("VACUUM: No tasks created for %d volume replicas across %d volumes. Threshold=%.2f%%, MinAge=%s. Skipped: %d (garbage<threshold), %d (age<minimum)",
totalVolumes, len(buckets), vacuumConfig.GarbageThreshold*100, minVolumeAge, skippedDueToGarbage, skippedDueToAge)
for i, metric := range metrics {
if i >= 3 {
break
}
glog.V(1).Infof("VACUUM: Volume %d on %s: garbage=%.2f%% (need ≥%.2f%%), age=%s (need ≥%s)",
metric.VolumeID, metric.Server, metric.GarbageRatio*100, vacuumConfig.GarbageThreshold*100,
metric.Age.Truncate(time.Minute), minVolumeAge.Truncate(time.Minute))
}
}
return results, nil
}
// pickPrimaryReplica returns the eligible replica with the highest garbage
// ratio. Returns nil if no replica satisfies both gates.
func pickPrimaryReplica(replicas []*types.VolumeHealthMetrics, garbageThreshold float64, minAge time.Duration) *types.VolumeHealthMetrics {
var best *types.VolumeHealthMetrics
for _, m := range replicas {
if m.GarbageRatio < garbageThreshold {
continue
}
if m.Age < minAge {
continue
}
if best == nil || m.GarbageRatio > best.GarbageRatio {
best = m
}
}
return best
}
// recordSkipReasons tallies skip counters for debug logging when no
// replica of a volume qualified for vacuum.
func recordSkipReasons(replicas []*types.VolumeHealthMetrics, garbageThreshold float64, minAge time.Duration, garbageSkip, ageSkip, debugCount *int) {
for _, m := range replicas {
if *debugCount >= 5 {
return
}
if m.GarbageRatio < garbageThreshold {
*garbageSkip++
}
if m.Age < minAge {
*ageSkip++
}
*debugCount++
}
}
// createVacuumTaskParams creates typed parameters for a vacuum task whose
// Sources list contains every replica of the volume. Worker-side
// performVacuum iterates over Sources and vacuums each replica.
func createVacuumTaskParams(task *types.TaskDetectionResult, replicas []*types.VolumeHealthMetrics, vacuumConfig *Config, clusterInfo *types.ClusterInfo) *worker_pb.TaskParams {
garbageThreshold := 0.3
verifyChecksum := true
batchSize := int32(1000)
workingDir := ""
if vacuumConfig != nil {
garbageThreshold = vacuumConfig.GarbageThreshold
}
if clusterInfo == nil || clusterInfo.ActiveTopology == nil {
glog.Errorf("Topology not available for vacuum task on volume %d, skipping", task.VolumeID)
return nil
}
sources := make([]*worker_pb.TaskSource, 0, len(replicas))
seen := make(map[string]struct{}, len(replicas))
for _, m := range replicas {
address, err := util.ResolveServerAddress(m.Server, clusterInfo.ActiveTopology)
if err != nil {
glog.Warningf("Failed to resolve address for server %s for vacuum task on volume %d, dropping replica: %v", m.Server, task.VolumeID, err)
continue
}
if _, ok := seen[address]; ok {
continue
}
seen[address] = struct{}{}
sources = append(sources, &worker_pb.TaskSource{
Node: address,
VolumeId: task.VolumeID,
EstimatedSize: m.Size,
DataCenter: m.DataCenter,
Rack: m.Rack,
})
}
if len(sources) == 0 {
glog.Errorf("No resolvable replicas for vacuum task on volume %d, skipping", task.VolumeID)
return nil
}
// Use the primary replica (matches task.Server) for the canonical size
// recorded in TaskParams. Master built-in keeps a single volumeSizeLimit
// here so timeouts are stable across replicas.
canonical := primaryReplicaByServer(replicas, task.Server)
var canonicalSize uint64
if canonical != nil {
canonicalSize = canonical.Size
}
return &worker_pb.TaskParams{
TaskId: task.TaskID,
VolumeId: task.VolumeID,
Collection: task.Collection,
VolumeSize: canonicalSize,
Sources: sources,
TaskParams: &worker_pb.TaskParams_VacuumParams{
VacuumParams: &worker_pb.VacuumTaskParams{
GarbageThreshold: garbageThreshold,
ForceVacuum: false,
BatchSize: batchSize,
WorkingDir: workingDir,
VerifyChecksum: verifyChecksum,
},
},
}
}
func primaryReplicaByServer(replicas []*types.VolumeHealthMetrics, server string) *types.VolumeHealthMetrics {
for _, m := range replicas {
if m.Server == server {
return m
}
}
if len(replicas) > 0 {
return replicas[0]
}
return nil
}

View File

@@ -0,0 +1,611 @@
package vacuum
import (
"context"
"fmt"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
"google.golang.org/grpc"
"google.golang.org/protobuf/proto"
)
const (
defaultVacuumTaskBatchSize = int32(1000)
DefaultMaxExecutionConcurrency = int32(2)
)
func init() {
pluginworker.RegisterHandler(pluginworker.HandlerFactory{
JobType: "vacuum",
Category: pluginworker.CategoryDefault,
Aliases: []string{"vol.vacuum", "volume.vacuum"},
Build: func(opts pluginworker.HandlerBuildOptions) (pluginworker.JobHandler, error) {
return NewVacuumHandler(opts.GrpcDialOption, int32(opts.MaxExecute)), nil
},
})
}
// VacuumHandler is the plugin job handler for vacuum job type.
type VacuumHandler struct {
grpcDialOption grpc.DialOption
maxExecutionConcurrency int32
}
// NewVacuumHandler creates a VacuumHandler with the given gRPC dial option and
// maximum execution concurrency. When maxExecutionConcurrency is zero or
// negative, DefaultMaxExecutionConcurrency is used as the fallback.
func NewVacuumHandler(grpcDialOption grpc.DialOption, maxExecutionConcurrency int32) *VacuumHandler {
return &VacuumHandler{grpcDialOption: grpcDialOption, maxExecutionConcurrency: maxExecutionConcurrency}
}
// Capability returns the job type capability for the vacuum handler.
// MaxExecutionConcurrency reflects the value passed at construction time,
// falling back to DefaultMaxExecutionConcurrency when unset.
func (h *VacuumHandler) Capability() *plugin_pb.JobTypeCapability {
maxExec := h.maxExecutionConcurrency
if maxExec <= 0 {
maxExec = DefaultMaxExecutionConcurrency
}
return &plugin_pb.JobTypeCapability{
JobType: "vacuum",
CanDetect: true,
CanExecute: true,
MaxDetectionConcurrency: 1,
MaxExecutionConcurrency: maxExec,
DisplayName: "Volume Vacuum",
Description: "Reclaims disk space by removing deleted files from volumes",
Weight: 60,
}
}
func (h *VacuumHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
return &plugin_pb.JobTypeDescriptor{
JobType: "vacuum",
DisplayName: "Volume Vacuum",
Description: "Detect and vacuum volumes with high garbage ratio",
Icon: "fas fa-broom",
DescriptorVersion: 1,
AdminConfigForm: &plugin_pb.ConfigForm{
FormId: "vacuum-admin",
Title: "Vacuum Admin Config",
Description: "Admin-side controls for vacuum detection scope.",
Sections: []*plugin_pb.ConfigSection{
{
SectionId: "scope",
Title: "Scope",
Description: "Optional filter to restrict detection.",
Fields: []*plugin_pb.ConfigField{
{
Name: "collection_filter",
Label: "Collection Filter",
Description: "Filter collections for vacuum detection. Use ALL_COLLECTIONS (default) to treat all volumes as one pool, EACH_COLLECTION to run detection separately per collection, or a comma-separated list of names, wildcards, or regex patterns.",
Placeholder: "ALL_COLLECTIONS",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
},
{
Name: "volume_state",
Label: "Volume State Filter",
Description: "Filter volumes by state: ALL (default), ACTIVE (writable volumes below size limit), or FULL (read-only volumes above size limit).",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_ENUM,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_SELECT,
Options: []*plugin_pb.ConfigOption{
{Value: string(pluginworker.VolumeStateAll), Label: "All Volumes"},
{Value: string(pluginworker.VolumeStateActive), Label: "Active (writable)"},
{Value: string(pluginworker.VolumeStateFull), Label: "Full (read-only)"},
},
},
{
Name: "data_center_filter",
Label: "Data Center Filter",
Description: "Only vacuum volumes in matching data centers (comma-separated, wildcards supported). Leave empty for all.",
Placeholder: "all data centers",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
},
{
Name: "rack_filter",
Label: "Rack Filter",
Description: "Only vacuum volumes on matching racks (comma-separated, wildcards supported). Leave empty for all.",
Placeholder: "all racks",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
},
{
Name: "node_filter",
Label: "Node Filter",
Description: "Only vacuum volumes on matching nodes (comma-separated, wildcards supported). Leave empty for all.",
Placeholder: "all nodes",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
},
},
},
},
DefaultValues: map[string]*plugin_pb.ConfigValue{
"collection_filter": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
},
"volume_state": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: string(pluginworker.VolumeStateAll)},
},
"data_center_filter": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
},
"rack_filter": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
},
"node_filter": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
},
},
},
WorkerConfigForm: &plugin_pb.ConfigForm{
FormId: "vacuum-worker",
Title: "Vacuum Worker Config",
Description: "Worker-side vacuum thresholds.",
Sections: []*plugin_pb.ConfigSection{
{
SectionId: "thresholds",
Title: "Thresholds",
Description: "Detection thresholds and timing constraints.",
Fields: []*plugin_pb.ConfigField{
{
Name: "garbage_threshold",
Label: "Garbage Threshold",
Description: "Detect volumes with garbage ratio >= threshold.",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_DOUBLE,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
Required: true,
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0}},
MaxValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 1}},
},
},
},
},
DefaultValues: map[string]*plugin_pb.ConfigValue{
"garbage_threshold": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.3},
},
},
},
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
Enabled: true,
DetectionIntervalMinutes: 17,
DetectionTimeoutSeconds: 120,
MaxJobsPerDetection: 200,
GlobalExecutionConcurrency: 16,
PerWorkerExecutionConcurrency: 4,
RetryLimit: 1,
RetryBackoffSeconds: 10,
JobTypeMaxRuntimeSeconds: 1800,
ExecutionTimeoutSeconds: 1800,
},
WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{
"garbage_threshold": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.3},
},
},
}
}
func (h *VacuumHandler) Detect(ctx context.Context, request *plugin_pb.RunDetectionRequest, sender pluginworker.DetectionSender) error {
if request == nil {
return fmt.Errorf("run detection request is nil")
}
if sender == nil {
return fmt.Errorf("detection sender is nil")
}
if request.JobType != "" && request.JobType != "vacuum" {
return fmt.Errorf("job type %q is not handled by vacuum worker", request.JobType)
}
workerConfig := deriveVacuumConfig(request.GetWorkerConfigValues())
collectionFilter := strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "collection_filter", ""))
masters := make([]string, 0)
if request.ClusterContext != nil {
masters = append(masters, request.ClusterContext.MasterGrpcAddresses...)
}
metrics, activeTopology, err := h.collectVolumeMetrics(ctx, masters, collectionFilter)
if err != nil {
return err
}
volState := pluginworker.VolumeState(strings.ToUpper(strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "volume_state", string(pluginworker.VolumeStateAll)))))
metrics = pluginworker.FilterMetricsByVolumeState(metrics, volState)
dataCenterFilter := strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "data_center_filter", ""))
rackFilter := strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "rack_filter", ""))
nodeFilter := strings.TrimSpace(pluginworker.ReadStringConfig(request.GetAdminConfigValues(), "node_filter", ""))
if dataCenterFilter != "" || rackFilter != "" || nodeFilter != "" {
metrics = pluginworker.FilterMetricsByLocation(metrics, dataCenterFilter, rackFilter, nodeFilter)
}
clusterInfo := &workertypes.ClusterInfo{ActiveTopology: activeTopology}
results, err := Detection(metrics, clusterInfo, workerConfig)
if err != nil {
return err
}
if traceErr := emitVacuumDetectionDecisionTrace(sender, metrics, workerConfig, results); traceErr != nil {
glog.Warningf("Plugin worker failed to emit vacuum detection trace: %v", traceErr)
}
maxResults := int(request.MaxResults)
hasMore := false
if maxResults > 0 && len(results) > maxResults {
hasMore = true
results = results[:maxResults]
}
proposals := make([]*plugin_pb.JobProposal, 0, len(results))
for _, result := range results {
proposal, proposalErr := buildVacuumProposal(result)
if proposalErr != nil {
glog.Warningf("Plugin worker skip invalid vacuum proposal: %v", proposalErr)
continue
}
proposals = append(proposals, proposal)
}
if err := sender.SendProposals(&plugin_pb.DetectionProposals{
JobType: "vacuum",
Proposals: proposals,
HasMore: hasMore,
}); err != nil {
return err
}
return sender.SendComplete(&plugin_pb.DetectionComplete{
JobType: "vacuum",
Success: true,
TotalProposals: int32(len(proposals)),
})
}
func emitVacuumDetectionDecisionTrace(
sender pluginworker.DetectionSender,
metrics []*workertypes.VolumeHealthMetrics,
workerConfig *Config,
results []*workertypes.TaskDetectionResult,
) error {
if sender == nil || workerConfig == nil {
return nil
}
totalVolumes := len(metrics)
skippedDueToGarbage := 0
for _, metric := range metrics {
if metric == nil {
continue
}
if metric.GarbageRatio >= workerConfig.GarbageThreshold {
continue
}
skippedDueToGarbage++
}
summaryMessage := ""
summaryStage := "decision_summary"
if len(results) == 0 {
summaryMessage = fmt.Sprintf(
"VACUUM: No tasks created for %d volumes. Threshold=%.2f%%. Skipped: %d (garbage<threshold)",
totalVolumes,
workerConfig.GarbageThreshold*100,
skippedDueToGarbage,
)
} else {
summaryMessage = fmt.Sprintf(
"VACUUM: Created %d task(s) from %d volumes. Threshold=%.2f%%",
len(results),
totalVolumes,
workerConfig.GarbageThreshold*100,
)
}
if err := sender.SendActivity(pluginworker.BuildDetectorActivity(summaryStage, summaryMessage, map[string]*plugin_pb.ConfigValue{
"total_volumes": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(totalVolumes)},
},
"selected_tasks": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(results))},
},
"garbage_threshold_percent": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: workerConfig.GarbageThreshold * 100},
},
"skipped_garbage": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(skippedDueToGarbage)},
},
})); err != nil {
return err
}
limit := 3
if len(metrics) < limit {
limit = len(metrics)
}
for i := 0; i < limit; i++ {
metric := metrics[i]
if metric == nil {
continue
}
message := fmt.Sprintf(
"VACUUM: Volume %d: garbage=%.2f%% (need ≥%.2f%%)",
metric.VolumeID,
metric.GarbageRatio*100,
workerConfig.GarbageThreshold*100,
)
if err := sender.SendActivity(pluginworker.BuildDetectorActivity("decision_volume", message, map[string]*plugin_pb.ConfigValue{
"volume_id": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(metric.VolumeID)},
},
"garbage_percent": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: metric.GarbageRatio * 100},
},
"required_garbage_percent": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: workerConfig.GarbageThreshold * 100},
},
})); err != nil {
return err
}
}
return nil
}
func (h *VacuumHandler) Execute(ctx context.Context, request *plugin_pb.ExecuteJobRequest, sender pluginworker.ExecutionSender) error {
if request == nil || request.Job == nil {
return fmt.Errorf("execute request/job is nil")
}
if sender == nil {
return fmt.Errorf("execution sender is nil")
}
if request.Job.JobType != "" && request.Job.JobType != "vacuum" {
return fmt.Errorf("job type %q is not handled by vacuum worker", request.Job.JobType)
}
params, err := decodeVacuumTaskParams(request.Job)
if err != nil {
return err
}
if len(params.Sources) == 0 || strings.TrimSpace(params.Sources[0].Node) == "" {
return fmt.Errorf("vacuum task source node is required")
}
workerConfig := deriveVacuumConfig(request.GetWorkerConfigValues())
if vacuumParams := params.GetVacuumParams(); vacuumParams != nil {
if vacuumParams.GarbageThreshold <= 0 {
vacuumParams.GarbageThreshold = workerConfig.GarbageThreshold
}
} else {
params.TaskParams = &worker_pb.TaskParams_VacuumParams{
VacuumParams: &worker_pb.VacuumTaskParams{
GarbageThreshold: workerConfig.GarbageThreshold,
BatchSize: defaultVacuumTaskBatchSize,
VerifyChecksum: true,
},
}
}
servers := make([]string, 0, len(params.Sources))
for _, src := range params.Sources {
if src != nil && src.Node != "" {
servers = append(servers, src.Node)
}
}
task := NewVacuumTask(
request.Job.JobId,
servers,
params.VolumeId,
params.Collection,
h.grpcDialOption,
)
execCtx, execCancel := context.WithCancel(ctx)
defer execCancel()
task.SetProgressCallback(func(progress float64, stage string) {
message := fmt.Sprintf("vacuum progress %.0f%%", progress)
if strings.TrimSpace(stage) != "" {
message = stage
}
if err := sender.SendProgress(&plugin_pb.JobProgressUpdate{
JobId: request.Job.JobId,
JobType: request.Job.JobType,
State: plugin_pb.JobState_JOB_STATE_RUNNING,
ProgressPercent: progress,
Stage: stage,
Message: message,
Activities: []*plugin_pb.ActivityEvent{
pluginworker.BuildExecutorActivity(stage, message),
},
}); err != nil {
execCancel()
}
})
if err := sender.SendProgress(&plugin_pb.JobProgressUpdate{
JobId: request.Job.JobId,
JobType: request.Job.JobType,
State: plugin_pb.JobState_JOB_STATE_ASSIGNED,
ProgressPercent: 0,
Stage: "assigned",
Message: "vacuum job accepted",
Activities: []*plugin_pb.ActivityEvent{
pluginworker.BuildExecutorActivity("assigned", "vacuum job accepted"),
},
}); err != nil {
return err
}
if err := task.Execute(execCtx, params); err != nil {
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
JobId: request.Job.JobId,
JobType: request.Job.JobType,
State: plugin_pb.JobState_JOB_STATE_FAILED,
ProgressPercent: 100,
Stage: "failed",
Message: err.Error(),
Activities: []*plugin_pb.ActivityEvent{
pluginworker.BuildExecutorActivity("failed", err.Error()),
},
})
return err
}
resultSummary := fmt.Sprintf("vacuum completed for volume %d", params.VolumeId)
return sender.SendCompleted(&plugin_pb.JobCompleted{
JobId: request.Job.JobId,
JobType: request.Job.JobType,
Success: true,
Result: &plugin_pb.JobResult{
Summary: resultSummary,
OutputValues: map[string]*plugin_pb.ConfigValue{
"volume_id": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(params.VolumeId)},
},
"server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: params.Sources[0].Node},
},
},
},
Activities: []*plugin_pb.ActivityEvent{
pluginworker.BuildExecutorActivity("completed", resultSummary),
},
})
}
func (h *VacuumHandler) collectVolumeMetrics(
ctx context.Context,
masterAddresses []string,
collectionFilter string,
) ([]*workertypes.VolumeHealthMetrics, *topology.ActiveTopology, error) {
metrics, activeTopology, _, err := pluginworker.CollectVolumeMetricsFromMasters(ctx, masterAddresses, collectionFilter, h.grpcDialOption)
return metrics, activeTopology, err
}
func deriveVacuumConfig(values map[string]*plugin_pb.ConfigValue) *Config {
config := NewDefaultConfig()
config.GarbageThreshold = pluginworker.ReadDoubleConfig(values, "garbage_threshold", config.GarbageThreshold)
// Match the shell/master vacuum, which applies no volume-age gate (default 0),
// but honor an explicit override instead of silently dropping it.
config.MinVolumeAgeSeconds = pluginworker.ReadIntConfig(values, "min_volume_age_seconds", 0)
return config
}
func buildVacuumProposal(result *workertypes.TaskDetectionResult) (*plugin_pb.JobProposal, error) {
if result == nil {
return nil, fmt.Errorf("task detection result is nil")
}
if result.TypedParams == nil {
return nil, fmt.Errorf("missing typed params for volume %d", result.VolumeID)
}
paramsPayload, err := proto.Marshal(result.TypedParams)
if err != nil {
return nil, fmt.Errorf("marshal task params: %w", err)
}
proposalID := strings.TrimSpace(result.TaskID)
if proposalID == "" {
proposalID = fmt.Sprintf("vacuum-%d-%d", result.VolumeID, time.Now().UnixNano())
}
dedupeKey := fmt.Sprintf("vacuum:%d", result.VolumeID)
if result.Collection != "" {
dedupeKey = dedupeKey + ":" + result.Collection
}
summary := fmt.Sprintf("Vacuum volume %d", result.VolumeID)
if strings.TrimSpace(result.Server) != "" {
summary = summary + " on " + result.Server
}
// Estimate runtime: 5 min/GB (compact + commit + overhead)
volumeSizeGB := int64(result.TypedParams.VolumeSize/1024/1024/1024) + 1
estimatedRuntimeSeconds := volumeSizeGB * 5 * 60
return &plugin_pb.JobProposal{
ProposalId: proposalID,
DedupeKey: dedupeKey,
JobType: "vacuum",
Priority: pluginworker.MapTaskPriority(result.Priority),
Summary: summary,
Detail: strings.TrimSpace(result.Reason),
Parameters: map[string]*plugin_pb.ConfigValue{
"task_params_pb": {
Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: paramsPayload},
},
"volume_id": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(result.VolumeID)},
},
"server": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Server},
},
"collection": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: result.Collection},
},
"estimated_runtime_seconds": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: estimatedRuntimeSeconds},
},
},
Labels: map[string]string{
"task_type": "vacuum",
"volume_id": fmt.Sprintf("%d", result.VolumeID),
"collection": result.Collection,
"source_node": result.Server,
},
}, nil
}
func decodeVacuumTaskParams(job *plugin_pb.JobSpec) (*worker_pb.TaskParams, error) {
if job == nil {
return nil, fmt.Errorf("job spec is nil")
}
if payload := pluginworker.ReadBytesConfig(job.Parameters, "task_params_pb"); len(payload) > 0 {
params := &worker_pb.TaskParams{}
if err := proto.Unmarshal(payload, params); err != nil {
return nil, fmt.Errorf("unmarshal task_params_pb: %w", err)
}
if params.TaskId == "" {
params.TaskId = job.JobId
}
return params, nil
}
volumeID := pluginworker.ReadUint32Config(job.Parameters, "volume_id", 0)
server := pluginworker.ReadStringConfig(job.Parameters, "server", "")
collection := pluginworker.ReadStringConfig(job.Parameters, "collection", "")
if volumeID == 0 {
return nil, fmt.Errorf("missing volume_id in job parameters")
}
if strings.TrimSpace(server) == "" {
return nil, fmt.Errorf("missing server in job parameters")
}
return &worker_pb.TaskParams{
TaskId: job.JobId,
VolumeId: volumeID,
Collection: collection,
Sources: []*worker_pb.TaskSource{
{
Node: server,
VolumeId: volumeID,
},
},
TaskParams: &worker_pb.TaskParams_VacuumParams{
VacuumParams: &worker_pb.VacuumTaskParams{
GarbageThreshold: 0.3,
BatchSize: defaultVacuumTaskBatchSize,
VerifyChecksum: true,
},
},
}, nil
}

View File

@@ -0,0 +1,373 @@
package vacuum
import (
"context"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
workertypes "github.com/seaweedfs/seaweedfs/weed/worker/types"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
func TestDecodeVacuumTaskParamsFromPayload(t *testing.T) {
expected := &worker_pb.TaskParams{
TaskId: "task-1",
VolumeId: 42,
Collection: "photos",
Sources: []*worker_pb.TaskSource{
{
Node: "10.0.0.1:8080",
VolumeId: 42,
},
},
TaskParams: &worker_pb.TaskParams_VacuumParams{
VacuumParams: &worker_pb.VacuumTaskParams{
GarbageThreshold: 0.33,
BatchSize: 500,
VerifyChecksum: true,
},
},
}
payload, err := proto.Marshal(expected)
if err != nil {
t.Fatalf("marshal payload: %v", err)
}
job := &plugin_pb.JobSpec{
JobId: "job-from-admin",
Parameters: map[string]*plugin_pb.ConfigValue{
"task_params_pb": {Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: payload}},
},
}
actual, err := decodeVacuumTaskParams(job)
if err != nil {
t.Fatalf("decodeVacuumTaskParams() err = %v", err)
}
if !proto.Equal(expected, actual) {
t.Fatalf("decoded params mismatch\nexpected: %+v\nactual: %+v", expected, actual)
}
}
func TestDecodeVacuumTaskParamsFallback(t *testing.T) {
job := &plugin_pb.JobSpec{
JobId: "job-2",
Parameters: map[string]*plugin_pb.ConfigValue{
"volume_id": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7}},
"server": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "127.0.0.1:8080"}},
"collection": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "videos"}},
},
}
params, err := decodeVacuumTaskParams(job)
if err != nil {
t.Fatalf("decodeVacuumTaskParams() err = %v", err)
}
if params.TaskId != "job-2" || params.VolumeId != 7 || params.Collection != "videos" {
t.Fatalf("unexpected basic params: %+v", params)
}
if len(params.Sources) != 1 || params.Sources[0].Node != "127.0.0.1:8080" {
t.Fatalf("unexpected sources: %+v", params.Sources)
}
if params.GetVacuumParams() == nil {
t.Fatalf("expected fallback vacuum params")
}
}
func TestDeriveVacuumConfigAllowsZeroValues(t *testing.T) {
values := map[string]*plugin_pb.ConfigValue{
"garbage_threshold": {
Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0},
},
}
cfg := deriveVacuumConfig(values)
if cfg.GarbageThreshold != 0 {
t.Fatalf("expected garbage_threshold 0, got %v", cfg.GarbageThreshold)
}
}
func TestMasterAddressCandidates(t *testing.T) {
candidates := pluginworker.MasterAddressCandidates("localhost:9333")
if len(candidates) != 2 {
t.Fatalf("expected 2 candidates, got %d: %v", len(candidates), candidates)
}
seen := map[string]bool{}
for _, candidate := range candidates {
seen[candidate] = true
}
if !seen["localhost:9333"] {
t.Fatalf("expected original address in candidates: %v", candidates)
}
if !seen["localhost:19333"] {
t.Fatalf("expected grpc address in candidates: %v", candidates)
}
}
func TestShouldSkipDetectionByInterval(t *testing.T) {
if pluginworker.ShouldSkipDetectionByInterval(nil, 10) {
t.Fatalf("expected false when timestamp is nil")
}
if pluginworker.ShouldSkipDetectionByInterval(timestamppb.Now(), 0) {
t.Fatalf("expected false when min interval is zero")
}
recent := timestamppb.New(time.Now().Add(-5 * time.Second))
if !pluginworker.ShouldSkipDetectionByInterval(recent, 10) {
t.Fatalf("expected true for recent successful run")
}
old := timestamppb.New(time.Now().Add(-30 * time.Second))
if pluginworker.ShouldSkipDetectionByInterval(old, 10) {
t.Fatalf("expected false for old successful run")
}
}
func TestVacuumHandlerRejectsUnsupportedJobType(t *testing.T) {
handler := NewVacuumHandler(nil, 0)
err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
JobType: "balance",
}, noopDetectionSender{})
if err == nil {
t.Fatalf("expected detect job type mismatch error")
}
err = handler.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
Job: &plugin_pb.JobSpec{JobId: "job-1", JobType: "balance"},
}, noopExecutionSender{})
if err == nil {
t.Fatalf("expected execute job type mismatch error")
}
}
func TestBuildExecutorActivity(t *testing.T) {
activity := pluginworker.BuildExecutorActivity("running", "vacuum in progress")
if activity == nil {
t.Fatalf("expected non-nil activity")
}
if activity.Source != plugin_pb.ActivitySource_ACTIVITY_SOURCE_EXECUTOR {
t.Fatalf("unexpected source: %v", activity.Source)
}
if activity.Stage != "running" {
t.Fatalf("unexpected stage: %q", activity.Stage)
}
if activity.Message != "vacuum in progress" {
t.Fatalf("unexpected message: %q", activity.Message)
}
if activity.CreatedAt == nil {
t.Fatalf("expected created_at timestamp")
}
}
func TestEmitVacuumDetectionDecisionTraceNoTasks(t *testing.T) {
sender := &recordingDetectionSender{}
config := NewDefaultConfig()
config.GarbageThreshold = 0.3
config.MinVolumeAgeSeconds = 0
metrics := []*workertypes.VolumeHealthMetrics{
{
VolumeID: 17,
GarbageRatio: 0,
Age: 218*time.Hour + 23*time.Minute,
},
{
VolumeID: 16,
GarbageRatio: 0,
Age: 218*time.Hour + 22*time.Minute,
},
{
VolumeID: 6,
GarbageRatio: 0,
Age: 90*time.Hour + 42*time.Minute,
},
}
if err := emitVacuumDetectionDecisionTrace(sender, metrics, config, nil); err != nil {
t.Fatalf("emitVacuumDetectionDecisionTrace error: %v", err)
}
if len(sender.events) < 4 {
t.Fatalf("expected at least 4 detection events, got %d", len(sender.events))
}
if sender.events[0].Source != plugin_pb.ActivitySource_ACTIVITY_SOURCE_DETECTOR {
t.Fatalf("expected detector source, got %v", sender.events[0].Source)
}
if !strings.Contains(sender.events[0].Message, "VACUUM: No tasks created for 3 volumes") {
t.Fatalf("unexpected summary message: %q", sender.events[0].Message)
}
if !strings.Contains(sender.events[1].Message, "VACUUM: Volume 17: garbage=0.00%") {
t.Fatalf("unexpected first detail message: %q", sender.events[1].Message)
}
}
func TestVacuumDescriptorHasNewFields(t *testing.T) {
handler := NewVacuumHandler(nil, 2)
desc := handler.Descriptor()
if desc == nil {
t.Fatal("descriptor is nil")
}
adminForm := desc.AdminConfigForm
if adminForm == nil {
t.Fatal("admin config form is nil")
}
if len(adminForm.Sections) == 0 {
t.Fatal("admin config form has no sections")
}
scopeSection := adminForm.Sections[0]
fieldNames := make(map[string]*plugin_pb.ConfigField)
for _, f := range scopeSection.Fields {
fieldNames[f.Name] = f
}
requiredFields := []string{
"collection_filter",
"volume_state",
"data_center_filter",
"rack_filter",
"node_filter",
}
for _, name := range requiredFields {
if _, ok := fieldNames[name]; !ok {
t.Errorf("missing field %q in admin config scope section", name)
}
}
// Verify volume_state is an enum with correct options.
vsField := fieldNames["volume_state"]
if vsField.FieldType != plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_ENUM {
t.Errorf("volume_state field type = %v, want ENUM", vsField.FieldType)
}
if len(vsField.Options) != 3 {
t.Errorf("volume_state options count = %d, want 3", len(vsField.Options))
}
// Verify default values exist for new fields.
defaultKeys := []string{"volume_state", "data_center_filter", "rack_filter", "node_filter"}
for _, key := range defaultKeys {
if _, ok := adminForm.DefaultValues[key]; !ok {
t.Errorf("missing default value for %q", key)
}
}
// Verify volume_state default is ALL.
vsDefault := adminForm.DefaultValues["volume_state"]
if sv, ok := vsDefault.Kind.(*plugin_pb.ConfigValue_StringValue); !ok || sv.StringValue != "ALL" {
t.Errorf("volume_state default = %v, want ALL", vsDefault)
}
// Verify collection_filter description mentions ALL_COLLECTIONS.
cfField := fieldNames["collection_filter"]
if cfField.Placeholder != "ALL_COLLECTIONS" {
t.Errorf("collection_filter placeholder = %q, want ALL_COLLECTIONS", cfField.Placeholder)
}
}
func TestVacuumFiltersVolumeState(t *testing.T) {
metrics := []*workertypes.VolumeHealthMetrics{
{VolumeID: 1, FullnessRatio: 0.5}, // active
{VolumeID: 2, FullnessRatio: 1.5}, // full
{VolumeID: 3, FullnessRatio: 0.9}, // active
{VolumeID: 4, FullnessRatio: 1.01}, // full (at threshold)
}
tests := []struct {
name string
state pluginworker.VolumeState
wantIDs []uint32
}{
{"ALL returns all", pluginworker.VolumeStateAll, []uint32{1, 2, 3, 4}},
{"ACTIVE returns writable", pluginworker.VolumeStateActive, []uint32{1, 3}},
{"FULL returns read-only", pluginworker.VolumeStateFull, []uint32{2, 4}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
filtered := pluginworker.FilterMetricsByVolumeState(metrics, tt.state)
if len(filtered) != len(tt.wantIDs) {
t.Fatalf("got %d metrics, want %d", len(filtered), len(tt.wantIDs))
}
for i, m := range filtered {
if m.VolumeID != tt.wantIDs[i] {
t.Errorf("filtered[%d].VolumeID = %d, want %d", i, m.VolumeID, tt.wantIDs[i])
}
}
})
}
}
func TestVacuumFiltersLocation(t *testing.T) {
metrics := []*workertypes.VolumeHealthMetrics{
{VolumeID: 1, DataCenter: "dc1", Rack: "r1", Server: "s1"},
{VolumeID: 2, DataCenter: "dc1", Rack: "r2", Server: "s2"},
{VolumeID: 3, DataCenter: "dc2", Rack: "r1", Server: "s3"},
{VolumeID: 4, DataCenter: "dc2", Rack: "r3", Server: "s4"},
}
tests := []struct {
name string
dc string
rack string
node string
wantIDs []uint32
}{
{"dc filter", "dc1", "", "", []uint32{1, 2}},
{"rack filter", "", "r1", "", []uint32{1, 3}},
{"node filter", "", "", "s2,s4", []uint32{2, 4}},
{"dc + rack", "dc1", "r1", "", []uint32{1}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
filtered := pluginworker.FilterMetricsByLocation(metrics, tt.dc, tt.rack, tt.node)
if len(filtered) != len(tt.wantIDs) {
t.Fatalf("got %d metrics, want %d", len(filtered), len(tt.wantIDs))
}
for i, m := range filtered {
if m.VolumeID != tt.wantIDs[i] {
t.Errorf("filtered[%d].VolumeID = %d, want %d", i, m.VolumeID, tt.wantIDs[i])
}
}
})
}
}
type noopDetectionSender struct{}
func (noopDetectionSender) SendProposals(*plugin_pb.DetectionProposals) error { return nil }
func (noopDetectionSender) SendComplete(*plugin_pb.DetectionComplete) error { return nil }
func (noopDetectionSender) SendActivity(*plugin_pb.ActivityEvent) error { return nil }
type noopExecutionSender struct{}
func (noopExecutionSender) SendProgress(*plugin_pb.JobProgressUpdate) error { return nil }
func (noopExecutionSender) SendCompleted(*plugin_pb.JobCompleted) error { return nil }
type recordingDetectionSender struct {
proposals *plugin_pb.DetectionProposals
complete *plugin_pb.DetectionComplete
events []*plugin_pb.ActivityEvent
}
func (r *recordingDetectionSender) SendProposals(proposals *plugin_pb.DetectionProposals) error {
r.proposals = proposals
return nil
}
func (r *recordingDetectionSender) SendComplete(complete *plugin_pb.DetectionComplete) error {
r.complete = complete
return nil
}
func (r *recordingDetectionSender) SendActivity(event *plugin_pb.ActivityEvent) error {
if event != nil {
r.events = append(r.events, event)
}
return nil
}

View File

@@ -0,0 +1,98 @@
package vacuum
import (
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// Global variable to hold the task definition for configuration updates
var globalTaskDef *base.TaskDefinition
// Auto-register this task when the package is imported
func init() {
RegisterVacuumTask()
// Register config updater
tasks.AutoRegisterConfigUpdater(types.TaskTypeVacuum, UpdateConfigFromPersistence)
}
// RegisterVacuumTask registers the vacuum task with the new architecture
func RegisterVacuumTask() {
// Create configuration instance
config := NewDefaultConfig()
// Create shared gRPC dial option using TLS configuration
dialOpt := security.LoadClientTLS(util.GetViper(), "grpc.worker")
// Create complete task definition
taskDef := &base.TaskDefinition{
Type: types.TaskTypeVacuum,
Name: "vacuum",
DisplayName: "Volume Vacuum",
Description: "Reclaims disk space by removing deleted files from volumes",
Icon: "fas fa-broom text-primary",
Capabilities: []string{"vacuum", "storage"},
Config: config,
ConfigSpec: GetConfigSpec(),
CreateTask: func(params *worker_pb.TaskParams) (types.Task, error) {
if params == nil {
return nil, fmt.Errorf("task parameters are required")
}
if len(params.Sources) == 0 {
return nil, fmt.Errorf("at least one source is required for vacuum task")
}
servers := make([]string, 0, len(params.Sources))
for _, src := range params.Sources {
if src != nil && src.Node != "" {
servers = append(servers, src.Node)
}
}
return NewVacuumTask(
fmt.Sprintf("vacuum-%d", params.VolumeId),
servers,
params.VolumeId,
params.Collection,
dialOpt,
), nil
},
DetectionFunc: Detection,
ScanInterval: 2 * time.Hour,
SchedulingFunc: Scheduling,
MaxConcurrent: 2,
RepeatInterval: 7 * 24 * time.Hour,
}
// Store task definition globally for configuration updates
globalTaskDef = taskDef
// Register everything with a single function call!
base.RegisterTask(taskDef)
}
// UpdateConfigFromPersistence updates the vacuum configuration from persistence
func UpdateConfigFromPersistence(configPersistence interface{}) error {
if globalTaskDef == nil {
return fmt.Errorf("vacuum task not registered")
}
// Load configuration from persistence
newConfig := LoadConfigFromPersistence(configPersistence)
if newConfig == nil {
return fmt.Errorf("failed to load configuration from persistence")
}
// Update the task definition's config
globalTaskDef.Config = newConfig
glog.V(1).Infof("Updated vacuum task configuration from persistence")
return nil
}

View File

@@ -0,0 +1,37 @@
package vacuum
import (
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// Scheduling implements the scheduling logic for vacuum tasks
func Scheduling(task *types.TaskInput, runningTasks []*types.TaskInput, availableWorkers []*types.WorkerData, config base.TaskConfig) bool {
vacuumConfig := config.(*Config)
// Count running vacuum tasks
runningVacuumCount := 0
for _, runningTask := range runningTasks {
if runningTask.Type == types.TaskTypeVacuum {
runningVacuumCount++
}
}
// Check concurrency limit
if runningVacuumCount >= vacuumConfig.MaxConcurrent {
return false
}
// Check for available workers with vacuum capability
for _, worker := range availableWorkers {
if worker.CurrentLoad < worker.MaxConcurrent {
for _, capability := range worker.Capabilities {
if capability == types.TaskTypeVacuum {
return true
}
}
}
}
return false
}

View File

@@ -0,0 +1,444 @@
package vacuum
import (
"context"
"fmt"
"io"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
"github.com/seaweedfs/seaweedfs/weed/worker/types/base"
"google.golang.org/grpc"
)
// VacuumTask implements the Task interface.
//
// One task covers all replicas of a volume so behavior matches the master
// built-in vacuum (see topology.Topology.vacuumOneVolumeId): Check across
// every replica → filter to those whose garbage ratio meets the threshold
// → Compact/Commit/Cleanup that subset. Treating one replica per task (the
// prior behavior) drops the other N-1 replicas because the dispatcher
// gates duplicate tasks per volume via ActiveTopology.HasAnyTask.
type VacuumTask struct {
*base.BaseTask
servers []string
volumeID uint32
collection string
garbageThreshold float64
progress float64
grpcDialOption grpc.DialOption
volumeSize uint64
vacuumTargets []string // populated by checkVacuumEligibility — subset of servers that pass the per-replica garbage re-check and proceed to Compact/Commit/Cleanup
}
// NewVacuumTask creates a new unified vacuum task instance covering every
// replica server reported by the dispatcher.
func NewVacuumTask(id string, servers []string, volumeID uint32, collection string, grpcDialOption grpc.DialOption) *VacuumTask {
deduped := dedupePreserveOrder(servers)
return &VacuumTask{
BaseTask: base.NewBaseTask(id, types.TaskTypeVacuum),
servers: deduped,
volumeID: volumeID,
collection: collection,
garbageThreshold: 0.3, // Default 30% threshold
grpcDialOption: grpcDialOption,
}
}
// Execute implements the UnifiedTask interface
func (t *VacuumTask) Execute(ctx context.Context, params *worker_pb.TaskParams) error {
if params == nil {
return fmt.Errorf("task parameters are required")
}
vacuumParams := params.GetVacuumParams()
if vacuumParams == nil {
return fmt.Errorf("vacuum parameters are required")
}
t.garbageThreshold = vacuumParams.GarbageThreshold
t.volumeSize = params.VolumeSize
t.GetLogger().WithFields(map[string]interface{}{
"volume_id": t.volumeID,
"servers": t.servers,
"collection": t.collection,
"garbage_threshold": t.garbageThreshold,
}).Info("Starting vacuum task")
if len(t.servers) == 0 {
return fmt.Errorf("no source servers configured for vacuum task")
}
// Step 1: Check vacuum eligibility for each replica. Mirrors
// topology.batchVacuumVolumeCheck — only replicas whose garbage is at
// or above the threshold proceed to Compact/Commit/Cleanup.
t.ReportProgress(10.0)
t.GetLogger().Info("Checking volume status")
targets, currentGarbageRatios, err := t.checkVacuumEligibility(ctx)
if err != nil {
return fmt.Errorf("failed to check vacuum eligibility: %w", err)
}
if len(targets) == 0 {
t.GetLogger().WithFields(map[string]interface{}{
"garbage_ratios": currentGarbageRatios,
"required_threshold": t.garbageThreshold,
}).Info("No replica meets vacuum criteria, skipping")
t.ReportProgress(100.0)
return nil
}
t.vacuumTargets = targets
// Step 2: Perform vacuum (compact + commit + cleanup) across every
// target replica.
t.ReportProgress(50.0)
t.GetLogger().WithFields(map[string]interface{}{
"vacuum_targets": targets,
"garbage_ratios": currentGarbageRatios,
"threshold": t.garbageThreshold,
}).Info("Performing vacuum operation")
if err := t.performVacuum(ctx); err != nil {
return fmt.Errorf("failed to perform vacuum: %w", err)
}
// Step 3: Verify vacuum results on each target replica.
t.ReportProgress(90.0)
t.GetLogger().Info("Verifying vacuum results")
if err := t.verifyVacuumResults(ctx); err != nil {
glog.Warningf("Vacuum verification failed: %v", err)
// Don't fail the task - vacuum operation itself succeeded
}
t.ReportProgress(100.0)
glog.Infof("Vacuum task completed successfully: volume %d on %v (garbage ratios %v)",
t.volumeID, targets, currentGarbageRatios)
return nil
}
// Validate implements the UnifiedTask interface
func (t *VacuumTask) Validate(params *worker_pb.TaskParams) error {
if params == nil {
return fmt.Errorf("task parameters are required")
}
vacuumParams := params.GetVacuumParams()
if vacuumParams == nil {
return fmt.Errorf("vacuum parameters are required")
}
if params.VolumeId != t.volumeID {
return fmt.Errorf("volume ID mismatch: expected %d, got %d", t.volumeID, params.VolumeId)
}
// Every server the task was created with must appear in the params'
// Sources list. The dispatcher fills Sources from the detection-time
// replica set, so a mismatch means the worker received stale routing.
sourceSet := make(map[string]struct{}, len(params.Sources))
for _, source := range params.Sources {
if source == nil {
continue
}
sourceSet[source.Node] = struct{}{}
}
for _, server := range t.servers {
if _, ok := sourceSet[server]; !ok {
return fmt.Errorf("task server %s not present in params.Sources", server)
}
}
if vacuumParams.GarbageThreshold < 0 || vacuumParams.GarbageThreshold > 1.0 {
return fmt.Errorf("invalid garbage threshold: %f (must be between 0.0 and 1.0)", vacuumParams.GarbageThreshold)
}
return nil
}
// EstimateTime implements the UnifiedTask interface
func (t *VacuumTask) EstimateTime(params *worker_pb.TaskParams) time.Duration {
// Basic estimate based on simulated steps
return 14 * time.Second // Sum of all step durations
}
// GetProgress returns current progress
func (t *VacuumTask) GetProgress() float64 {
return t.progress
}
// vacuumTimeout returns a dynamic timeout scaled by volume size, matching the
// topology vacuum approach. base is the per-GB multiplier (e.g. 1 minute for
// check, 3 minutes for compact).
func (t *VacuumTask) vacuumTimeout(base time.Duration) time.Duration {
if t.volumeSize == 0 {
glog.V(1).Infof("volume %d has no size metric, using minimum timeout", t.volumeID)
}
sizeGB := int64(t.volumeSize/1024/1024/1024) + 1
return base * time.Duration(sizeGB)
}
// Helper methods for real vacuum operations
// checkVacuumEligibility queries every replica's current garbage ratio.
// Mirrors topology.batchVacuumVolumeCheck's all-or-nothing contract: if
// any replica's check fails (unreachable, RPC error, etc.) the entire
// task aborts. Vacuuming only the replicas that responded while
// silently skipping unreachable ones would compact the responders'
// garbage but leave the unreachable replica still carrying it,
// producing divergence the moment that replica comes back.
//
// Returns the subset of servers whose garbage is at or above the
// configured threshold, alongside a per-server ratio map for logging.
func (t *VacuumTask) checkVacuumEligibility(ctx context.Context) ([]string, map[string]float64, error) {
ratios := make(map[string]float64, len(t.servers))
for _, server := range t.servers {
ratio, err := t.checkOneVacuumEligibility(ctx, server)
if err != nil {
return nil, ratios, fmt.Errorf("vacuum check on %s for volume %d: %w (aborting; refusing to vacuum subset of replicas)", server, t.volumeID, err)
}
ratios[server] = ratio
glog.V(1).Infof("Volume %d on %s garbage ratio: %.2f%%, threshold: %.2f%%",
t.volumeID, server, ratio*100, t.garbageThreshold*100)
}
eligible := make([]string, 0, len(ratios))
for _, server := range t.servers {
if ratios[server] >= t.garbageThreshold {
eligible = append(eligible, server)
}
}
return eligible, ratios, nil
}
func (t *VacuumTask) checkOneVacuumEligibility(ctx context.Context, server string) (float64, error) {
var garbageRatio float64
err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
checkCtx, cancel := context.WithTimeout(ctx, t.vacuumTimeout(time.Minute))
defer cancel()
resp, err := client.VacuumVolumeCheck(checkCtx, &volume_server_pb.VacuumVolumeCheckRequest{
VolumeId: t.volumeID,
})
if err != nil {
return fmt.Errorf("failed to check volume vacuum status: %v", err)
}
garbageRatio = resp.GarbageRatio
return nil
})
return garbageRatio, err
}
// performVacuum runs the three-phase vacuum protocol that mirrors
// master built-in vacuum (topology.vacuumOneVolumeId):
//
// Phase 1 (Compact): build the new .cpd/.cpx files on every target.
// If any replica fails, roll back by Cleanup'ing the .cp* temp files
// on every target and abort — no replica has yet swapped its active
// files, so no replica is committed.
//
// Phase 2 (Commit): swap each target's active files with its .cp*
// files. Best-effort, matching batchVacuumVolumeCommit: per-replica
// errors are logged and surfaced together, but once any replica has
// swapped there is no clean rollback for the others, so we do not
// retry or undo. An operator must reconcile a partial commit
// failure.
//
// Phase 3 (Mark Writable): re-notify the master per replica so the
// volume re-enters the writable set, the worker analog of
// batchVacuumVolumeCommit's per-replica SetVolumeAvailable. Skipped
// when a replica came back read-only. Best-effort; never fails the
// task.
//
// Interleaving Compact→Commit→Cleanup per replica (the prior behavior)
// could leave a committed first replica beside an uncompacted second
// replica when Compact on the second failed — replica divergence with
// no automatic recovery.
func (t *VacuumTask) performVacuum(ctx context.Context) error {
// Phase 1: Compact all targets.
for _, server := range t.vacuumTargets {
if err := t.compactOne(ctx, server); err != nil {
t.cleanupAll(ctx)
return fmt.Errorf("vacuum compact on %s: %w", server, err)
}
}
// Phase 2: Commit all targets, tracking whether any replica is still
// read-only after the swap.
var commitErrors []error
anyReadOnly := false
for _, server := range t.vacuumTargets {
resp, err := t.commitOne(ctx, server)
if err != nil {
glog.Errorf("vacuum commit on %s for volume %d: %v", server, t.volumeID, err)
commitErrors = append(commitErrors, fmt.Errorf("%s: %w", server, err))
continue
}
if resp.GetIsReadOnly() {
anyReadOnly = true
}
}
if len(commitErrors) > 0 {
return fmt.Errorf("vacuum commit failed on %d/%d replicas: %v",
len(commitErrors), len(t.vacuumTargets), commitErrors)
}
// Phase 3: re-notify the master so the volume re-enters the writable
// set. The worker's only lever is VolumeMarkWritable, which clears the
// read-only flag and runs notifyMasterVolumeReadonly(false). Gate on
// the commit's IsReadOnly exactly as SetVolumeAvailable does: a replica
// still read-only (operator-set, EIO-quarantined, or disk-space-low)
// must stay out, and recovers on its own via the next ReadOnly=false
// heartbeat — force-clearing the flag here would override that. The
// worker is not told the master's size limit, so the isFullCapacity
// guard is left to the next capacity heartbeat. Best-effort: the vacuum
// itself already succeeded.
if anyReadOnly {
glog.V(0).Infof("post-vacuum: volume %d still read-only on a replica, leaving it out of writables", t.volumeID)
return nil
}
for _, server := range t.vacuumTargets {
if err := t.markWritableOne(ctx, server); err != nil {
glog.Warningf("post-vacuum mark writable on %s for volume %d: %v", server, t.volumeID, err)
continue
}
glog.V(0).Infof("post-vacuum marked volume %d writable on %s", t.volumeID, server)
}
return nil
}
func (t *VacuumTask) compactOne(ctx context.Context, server string) error {
return operation.WithVolumeServerClient(false, pb.ServerAddress(server), t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
t.GetLogger().Info("Compacting volume on %s", server)
compactCtx, cancel := context.WithTimeout(ctx, t.vacuumTimeout(3*time.Minute))
defer cancel()
stream, err := client.VacuumVolumeCompact(compactCtx, &volume_server_pb.VacuumVolumeCompactRequest{
VolumeId: t.volumeID,
})
if err != nil {
return fmt.Errorf("vacuum compact start: %v", err)
}
for {
resp, recvErr := stream.Recv()
if recvErr != nil {
if recvErr == io.EOF {
break
}
return fmt.Errorf("vacuum compact stream: %v", recvErr)
}
glog.V(2).Infof("Volume %d on %s compact progress: %d bytes", t.volumeID, server, resp.ProcessedBytes)
}
return nil
})
}
func (t *VacuumTask) commitOne(ctx context.Context, server string) (*volume_server_pb.VacuumVolumeCommitResponse, error) {
var resp *volume_server_pb.VacuumVolumeCommitResponse
err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
t.GetLogger().Info("Committing vacuum on %s", server)
commitCtx, cancel := context.WithTimeout(ctx, t.vacuumTimeout(time.Minute))
defer cancel()
var err error
resp, err = client.VacuumVolumeCommit(commitCtx, &volume_server_pb.VacuumVolumeCommitRequest{
VolumeId: t.volumeID,
})
if err != nil {
return fmt.Errorf("vacuum commit: %v", err)
}
return nil
})
return resp, err
}
func (t *VacuumTask) cleanupOne(ctx context.Context, server string) error {
return operation.WithVolumeServerClient(false, pb.ServerAddress(server), t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
cleanupCtx, cancel := context.WithTimeout(ctx, t.vacuumTimeout(time.Minute))
defer cancel()
_, err := client.VacuumVolumeCleanup(cleanupCtx, &volume_server_pb.VacuumVolumeCleanupRequest{
VolumeId: t.volumeID,
})
return err
})
}
func (t *VacuumTask) markWritableOne(ctx context.Context, server string) error {
return operation.WithVolumeServerClient(false, pb.ServerAddress(server), t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
// VolumeMarkWritable is a metadata RPC (reopen idx + flags +
// notifyMasterVolumeReadonly heartbeat) — millisecond-scale and
// independent of volume size. A flat 1m cap prevents an
// unresponsive replica from blocking Phase 3 for hours on a
// TB-scale volume where vacuumTimeout() would balloon.
markCtx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
_, err := client.VolumeMarkWritable(markCtx, &volume_server_pb.VolumeMarkWritableRequest{
VolumeId: t.volumeID,
})
return err
})
}
// cleanupAll removes the .cpd/.cpx/.cpldb temp files on every target.
// Used to roll back when Compact fails on one replica after others
// have already created their temp files. Per-target failures are
// logged but never bubble up — the rollback is best-effort.
func (t *VacuumTask) cleanupAll(ctx context.Context) {
for _, server := range t.vacuumTargets {
if err := t.cleanupOne(ctx, server); err != nil {
glog.Warningf("rollback cleanup on %s for volume %d: %v", server, t.volumeID, err)
}
}
}
// verifyVacuumResults checks each target replica's post-vacuum garbage
// ratio. Failures are logged at WARN — the task does not fail because the
// vacuum itself already succeeded.
func (t *VacuumTask) verifyVacuumResults(ctx context.Context) error {
for _, server := range t.vacuumTargets {
err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), t.grpcDialOption,
func(client volume_server_pb.VolumeServerClient) error {
verifyCtx, cancel := context.WithTimeout(ctx, t.vacuumTimeout(time.Minute))
defer cancel()
resp, err := client.VacuumVolumeCheck(verifyCtx, &volume_server_pb.VacuumVolumeCheckRequest{
VolumeId: t.volumeID,
})
if err != nil {
return fmt.Errorf("failed to verify vacuum results: %v", err)
}
glog.V(1).Infof("Volume %d on %s post-vacuum garbage ratio: %.2f%%",
t.volumeID, server, resp.GarbageRatio*100)
return nil
})
if err != nil {
glog.Warningf("post-vacuum verify on %s: %v", server, err)
}
}
return nil
}
// dedupePreserveOrder returns servers with duplicates removed, keeping the
// first occurrence's position. Detection sometimes hands the same node
// address in multiple Sources (e.g. EC variants); we coalesce them so each
// physical replica is vacuumed exactly once.
func dedupePreserveOrder(servers []string) []string {
seen := make(map[string]struct{}, len(servers))
out := make([]string, 0, len(servers))
for _, s := range servers {
if s == "" {
continue
}
if _, ok := seen[s]; ok {
continue
}
seen[s] = struct{}{}
out = append(out, s)
}
return out
}