snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,190 @@
package maintenance
import (
"github.com/seaweedfs/seaweedfs/weed/admin/config"
)
// Type aliases for backward compatibility
type ConfigFieldType = config.FieldType
type ConfigFieldUnit = config.FieldUnit
type ConfigField = config.Field
// Constant aliases for backward compatibility
const (
FieldTypeBool = config.FieldTypeBool
FieldTypeInt = config.FieldTypeInt
FieldTypeDuration = config.FieldTypeDuration
FieldTypeInterval = config.FieldTypeInterval
FieldTypeString = config.FieldTypeString
FieldTypeFloat = config.FieldTypeFloat
)
const (
UnitSeconds = config.UnitSeconds
UnitMinutes = config.UnitMinutes
UnitHours = config.UnitHours
UnitDays = config.UnitDays
UnitCount = config.UnitCount
UnitNone = config.UnitNone
)
// Function aliases for backward compatibility
var (
SecondsToIntervalValueUnit = config.SecondsToIntervalValueUnit
IntervalValueUnitToSeconds = config.IntervalValueUnitToSeconds
)
// MaintenanceConfigSchema defines the schema for maintenance configuration
type MaintenanceConfigSchema struct {
config.Schema // Embed common schema functionality
}
// GetMaintenanceConfigSchema returns the schema for maintenance configuration
func GetMaintenanceConfigSchema() *MaintenanceConfigSchema {
return &MaintenanceConfigSchema{
Schema: config.Schema{
Fields: []*config.Field{
{
Name: "enabled",
JSONName: "enabled",
Type: config.FieldTypeBool,
DefaultValue: true,
Required: false,
DisplayName: "Enable Maintenance System",
Description: "When enabled, the system will automatically scan for and execute maintenance tasks",
HelpText: "Toggle this to enable or disable the entire maintenance system",
InputType: "checkbox",
CSSClasses: "form-check-input",
},
{
Name: "scan_interval_seconds",
JSONName: "scan_interval_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 30 * 60, // 30 minutes in seconds
MinValue: 1 * 60, // 1 minute
MaxValue: 24 * 60 * 60, // 24 hours
Required: true,
DisplayName: "Scan Interval",
Description: "How often to scan for maintenance tasks",
HelpText: "The system will check for new maintenance tasks at this interval",
Placeholder: "30",
Unit: config.UnitMinutes,
InputType: "interval",
CSSClasses: "form-control",
},
{
Name: "worker_timeout_seconds",
JSONName: "worker_timeout_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 5 * 60, // 5 minutes
MinValue: 1 * 60, // 1 minute
MaxValue: 60 * 60, // 1 hour
Required: true,
DisplayName: "Worker Timeout",
Description: "How long to wait for worker heartbeat before considering it inactive",
HelpText: "Workers that don't send heartbeats within this time are considered offline",
Placeholder: "5",
Unit: config.UnitMinutes,
InputType: "interval",
CSSClasses: "form-control",
},
{
Name: "task_timeout_seconds",
JSONName: "task_timeout_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 2 * 60 * 60, // 2 hours
MinValue: 10 * 60, // 10 minutes
MaxValue: 24 * 60 * 60, // 24 hours
Required: true,
DisplayName: "Task Timeout",
Description: "Maximum time allowed for a task to complete",
HelpText: "Tasks that exceed this duration will be marked as failed",
Placeholder: "2",
Unit: config.UnitHours,
InputType: "interval",
CSSClasses: "form-control",
},
{
Name: "retry_delay_seconds",
JSONName: "retry_delay_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 15 * 60, // 15 minutes
MinValue: 1 * 60, // 1 minute
MaxValue: 24 * 60 * 60, // 24 hours
Required: true,
DisplayName: "Retry Delay",
Description: "How long to wait before retrying a failed task",
HelpText: "Failed tasks will be retried after this delay",
Placeholder: "15",
Unit: config.UnitMinutes,
InputType: "interval",
CSSClasses: "form-control",
},
{
Name: "max_retries",
JSONName: "max_retries",
Type: config.FieldTypeInt,
DefaultValue: 3,
MinValue: 0,
MaxValue: 10,
Required: true,
DisplayName: "Max Retries",
Description: "Maximum number of times to retry a failed task",
HelpText: "Tasks that fail more than this many times will be marked as permanently failed",
Placeholder: "3",
Unit: config.UnitCount,
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "cleanup_interval_seconds",
JSONName: "cleanup_interval_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 24 * 60 * 60, // 24 hours
MinValue: 1 * 60 * 60, // 1 hour
MaxValue: 7 * 24 * 60 * 60, // 7 days
Required: true,
DisplayName: "Cleanup Interval",
Description: "How often to run maintenance cleanup operations",
HelpText: "Removes old task records and temporary files at this interval",
Placeholder: "24",
Unit: config.UnitHours,
InputType: "interval",
CSSClasses: "form-control",
},
{
Name: "task_retention_seconds",
JSONName: "task_retention_seconds",
Type: config.FieldTypeInterval,
DefaultValue: 7 * 24 * 60 * 60, // 7 days
MinValue: 1 * 24 * 60 * 60, // 1 day
MaxValue: 30 * 24 * 60 * 60, // 30 days
Required: true,
DisplayName: "Task Retention",
Description: "How long to keep completed task records",
HelpText: "Task history older than this duration will be automatically deleted",
Placeholder: "7",
Unit: config.UnitDays,
InputType: "interval",
CSSClasses: "form-control",
},
{
Name: "global_max_concurrent",
JSONName: "global_max_concurrent",
Type: config.FieldTypeInt,
DefaultValue: 10,
MinValue: 1,
MaxValue: 100,
Required: true,
DisplayName: "Global Max Concurrent Tasks",
Description: "Maximum number of maintenance tasks that can run simultaneously across all workers",
HelpText: "Limits the total number of maintenance operations to control system load",
Placeholder: "10",
Unit: config.UnitCount,
InputType: "number",
CSSClasses: "form-control",
},
},
},
}
}

View File

@@ -0,0 +1,22 @@
package maintenance
import (
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"google.golang.org/protobuf/proto"
)
// DefaultMaintenanceConfigProto returns default configuration as protobuf
func DefaultMaintenanceConfigProto() *worker_pb.MaintenanceConfig {
return &worker_pb.MaintenanceConfig{
Enabled: proto.Bool(true),
ScanIntervalSeconds: 30 * 60, // 30 minutes
WorkerTimeoutSeconds: 5 * 60, // 5 minutes
TaskTimeoutSeconds: 2 * 60 * 60, // 2 hours
RetryDelaySeconds: 15 * 60, // 15 minutes
MaxRetries: 3,
CleanupIntervalSeconds: 24 * 60 * 60, // 24 hours
TaskRetentionSeconds: 7 * 24 * 60 * 60, // 7 days
// Policy field will be populated dynamically from separate task configuration files
Policy: nil,
}
}

View File

@@ -0,0 +1,620 @@
package maintenance
import (
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// MaintenanceIntegration bridges the task system with existing maintenance
type MaintenanceIntegration struct {
taskRegistry *types.TaskRegistry
uiRegistry *types.UIRegistry
// Bridge to existing system
maintenanceQueue *MaintenanceQueue
maintenancePolicy *MaintenancePolicy
// Pending operations tracker
pendingOperations *PendingOperations
// Active topology for task detection and target selection
activeTopology *topology.ActiveTopology
// Master's default replication, refreshed by the scanner each cycle and
// passed to detectors as the replica-placement fallback (matches the shell).
defaultReplicaPlacement string
// Type conversion maps
taskTypeMap map[types.TaskType]MaintenanceTaskType
revTaskTypeMap map[MaintenanceTaskType]types.TaskType
priorityMap map[types.TaskPriority]MaintenanceTaskPriority
revPriorityMap map[MaintenanceTaskPriority]types.TaskPriority
}
// NewMaintenanceIntegration creates the integration bridge
func NewMaintenanceIntegration(queue *MaintenanceQueue, policy *MaintenancePolicy) *MaintenanceIntegration {
integration := &MaintenanceIntegration{
taskRegistry: tasks.GetGlobalTypesRegistry(), // Use global types registry with auto-registered tasks
uiRegistry: tasks.GetGlobalUIRegistry(), // Use global UI registry with auto-registered UI providers
maintenanceQueue: queue,
maintenancePolicy: policy,
pendingOperations: NewPendingOperations(),
}
// Initialize active topology with 10 second recent task window
integration.activeTopology = topology.NewActiveTopology(10)
// Initialize type conversion maps
integration.initializeTypeMaps()
// Register all tasks
integration.registerAllTasks()
return integration
}
// initializeTypeMaps creates the type conversion maps for dynamic conversion
func (s *MaintenanceIntegration) initializeTypeMaps() {
// Initialize empty maps
s.taskTypeMap = make(map[types.TaskType]MaintenanceTaskType)
s.revTaskTypeMap = make(map[MaintenanceTaskType]types.TaskType)
// Build task type mappings dynamically from registered tasks after registration
// This will be called from registerAllTasks() after all tasks are registered
// Priority mappings (these are static and don't depend on registered tasks)
s.priorityMap = map[types.TaskPriority]MaintenanceTaskPriority{
types.TaskPriorityLow: PriorityLow,
types.TaskPriorityNormal: PriorityNormal,
types.TaskPriorityHigh: PriorityHigh,
}
// Reverse priority mappings
s.revPriorityMap = map[MaintenanceTaskPriority]types.TaskPriority{
PriorityLow: types.TaskPriorityLow,
PriorityNormal: types.TaskPriorityNormal,
PriorityHigh: types.TaskPriorityHigh,
PriorityCritical: types.TaskPriorityHigh, // Map critical to high
}
}
// buildTaskTypeMappings dynamically builds task type mappings from registered tasks
func (s *MaintenanceIntegration) buildTaskTypeMappings() {
// Clear existing mappings
s.taskTypeMap = make(map[types.TaskType]MaintenanceTaskType)
s.revTaskTypeMap = make(map[MaintenanceTaskType]types.TaskType)
// Build mappings from registered detectors
for workerTaskType := range s.taskRegistry.GetAllDetectors() {
// Convert types.TaskType to MaintenanceTaskType by string conversion
maintenanceTaskType := MaintenanceTaskType(string(workerTaskType))
s.taskTypeMap[workerTaskType] = maintenanceTaskType
s.revTaskTypeMap[maintenanceTaskType] = workerTaskType
glog.V(3).Infof("Dynamically mapped task type: %s <-> %s", workerTaskType, maintenanceTaskType)
}
glog.V(2).Infof("Built %d dynamic task type mappings", len(s.taskTypeMap))
}
// registerAllTasks registers all available tasks
func (s *MaintenanceIntegration) registerAllTasks() {
// Tasks are already auto-registered via import statements
// No manual registration needed
// Build dynamic type mappings from registered tasks
s.buildTaskTypeMappings()
// Configure tasks from policy
s.ConfigureTasksFromPolicy()
registeredTaskTypes := make([]string, 0, len(s.taskTypeMap))
for _, maintenanceTaskType := range s.taskTypeMap {
registeredTaskTypes = append(registeredTaskTypes, string(maintenanceTaskType))
}
glog.V(1).Infof("Registered tasks: %v", registeredTaskTypes)
}
// SetPolicy replaces the maintenance policy the integration configures tasks from and
// applies it immediately. Without this the integration kept the policy it was built with,
// so a policy updated at runtime reached the queue but never the detectors that decide
// which task types are scanned for.
//
// Not safe to call concurrently with a running scan. ConfigureTasksFromPolicy writes
// TaskDefinition.Config and TaskDefinition.MaxConcurrent, which ScanWithTaskDetectors reads
// through detector.IsEnabled(); nothing synchronises the two. Today every caller runs during
// admin server startup, before the scan loop exists. Anything that wires this to an HTTP
// handler has to add that synchronisation first - the same applies to
// tasks.ConfigUpdateRegistry.UpdateAllConfigs, which replaces TaskDefinition.Config outright.
func (s *MaintenanceIntegration) SetPolicy(policy *MaintenancePolicy) {
s.maintenancePolicy = policy
s.ConfigureTasksFromPolicy()
}
// ConfigureTasksFromPolicy dynamically configures all registered tasks based on the maintenance policy
func (s *MaintenanceIntegration) ConfigureTasksFromPolicy() {
if s.maintenancePolicy == nil {
return
}
// Configure all registered detectors and schedulers dynamically using policy configuration
configuredCount := 0
// Get all registered task types from the registry
for taskType, detector := range s.taskRegistry.GetAllDetectors() {
// Configure detector using policy-based configuration
s.configureDetectorFromPolicy(taskType, detector)
configuredCount++
}
for taskType, scheduler := range s.taskRegistry.GetAllSchedulers() {
// Configure scheduler using policy-based configuration
s.configureSchedulerFromPolicy(taskType, scheduler)
}
glog.V(1).Infof("Dynamically configured %d task types from maintenance policy", configuredCount)
}
// configureDetectorFromPolicy configures a detector using policy-based configuration
func (s *MaintenanceIntegration) configureDetectorFromPolicy(taskType types.TaskType, detector types.TaskDetector) {
// Try to configure using PolicyConfigurableDetector interface if supported
if configurableDetector, ok := detector.(types.PolicyConfigurableDetector); ok {
configurableDetector.ConfigureFromPolicy(s.maintenancePolicy)
glog.V(2).Infof("Configured detector %s using policy interface", taskType)
return
}
// Convert task system type to maintenance task type for policy lookup
maintenanceTaskType, exists := s.taskTypeMap[taskType]
if !exists {
glog.V(3).Infof("No maintenance task type mapping for %s, skipping configuration", taskType)
return
}
// A task type the policy says nothing about is left alone. IsTaskEnabled reports
// false for a missing entry, so applying it unconditionally would silently disable
// every task the policy does not list - which is how ec_balance would have been
// switched off the moment SetEnabled started working.
if GetTaskPolicy(s.maintenancePolicy, maintenanceTaskType) == nil {
glog.V(2).Infof("Maintenance policy has no entry for %s, leaving its detector configuration untouched", taskType)
return
}
// Apply basic configuration that all detectors should support
if basicDetector, ok := detector.(interface{ SetEnabled(bool) }); ok {
enabled := IsTaskEnabled(s.maintenancePolicy, maintenanceTaskType)
basicDetector.SetEnabled(enabled)
glog.V(3).Infof("Set enabled=%v for detector %s", enabled, taskType)
} else {
// For detectors that don't implement PolicyConfigurableDetector interface,
// they should be updated to implement it for full policy-based configuration
glog.V(2).Infof("Detector %s supports neither PolicyConfigurableDetector nor SetEnabled, its policy is ignored", taskType)
}
}
// configureSchedulerFromPolicy configures a scheduler using policy-based configuration
func (s *MaintenanceIntegration) configureSchedulerFromPolicy(taskType types.TaskType, scheduler types.TaskScheduler) {
// Try to configure using PolicyConfigurableScheduler interface if supported
if configurableScheduler, ok := scheduler.(types.PolicyConfigurableScheduler); ok {
configurableScheduler.ConfigureFromPolicy(s.maintenancePolicy)
glog.V(2).Infof("Configured scheduler %s using policy interface", taskType)
return
}
// Apply basic configuration that all schedulers should support
maintenanceTaskType, exists := s.taskTypeMap[taskType]
if !exists {
glog.V(3).Infof("No maintenance task type mapping for %s, skipping configuration", taskType)
return
}
// Same guard as on the detector side: no policy entry means no opinion, not disabled.
if GetTaskPolicy(s.maintenancePolicy, maintenanceTaskType) == nil {
glog.V(2).Infof("Maintenance policy has no entry for %s, leaving its scheduler configuration untouched", taskType)
return
}
// Set enabled status if scheduler supports it
if enableableScheduler, ok := scheduler.(interface{ SetEnabled(bool) }); ok {
enabled := IsTaskEnabled(s.maintenancePolicy, maintenanceTaskType)
enableableScheduler.SetEnabled(enabled)
glog.V(3).Infof("Set enabled=%v for scheduler %s", enabled, taskType)
} else {
// For schedulers that don't implement PolicyConfigurableScheduler interface,
// they should be updated to implement it for full policy-based configuration
glog.V(2).Infof("Scheduler %s supports neither PolicyConfigurableScheduler nor SetEnabled, its policy is ignored", taskType)
}
// Set max concurrent if scheduler supports it
if concurrentScheduler, ok := scheduler.(interface{ SetMaxConcurrent(int) }); ok {
maxConcurrent := GetMaxConcurrent(s.maintenancePolicy, maintenanceTaskType)
if maxConcurrent > 0 {
concurrentScheduler.SetMaxConcurrent(maxConcurrent)
glog.V(3).Infof("Set max concurrent=%d for scheduler %s", maxConcurrent, taskType)
}
}
}
// ScanWithTaskDetectors performs a scan using the task system
func (s *MaintenanceIntegration) ScanWithTaskDetectors(volumeMetrics []*types.VolumeHealthMetrics) ([]*TaskDetectionResult, error) {
// Note: ActiveTopology gets updated from topology info instead of volume metrics
glog.V(2).Infof("Processed %d volume metrics for task detection", len(volumeMetrics))
// Filter out volumes with pending operations to avoid duplicates
filteredMetrics := s.pendingOperations.FilterVolumeMetricsExcludingPending(volumeMetrics)
glog.V(1).Infof("Scanning %d volumes (filtered from %d) excluding pending operations",
len(filteredMetrics), len(volumeMetrics))
var allResults []*TaskDetectionResult
// Create cluster info
clusterInfo := &types.ClusterInfo{
TotalVolumes: len(filteredMetrics),
LastUpdated: time.Now(),
ActiveTopology: s.activeTopology, // Provide ActiveTopology for destination planning
DefaultReplicaPlacement: s.defaultReplicaPlacement,
}
// Run detection for each registered task type
for taskType, detector := range s.taskRegistry.GetAllDetectors() {
if !detector.IsEnabled() {
continue
}
// Cancel stale pending tasks for this type before re-detection
maintenanceType := s.taskTypeMap[taskType]
if cancelled := s.maintenanceQueue.CancelPendingTasksByType(maintenanceType); cancelled > 0 {
glog.Infof("Cancelled %d stale pending %s tasks before re-detection", cancelled, taskType)
}
glog.V(2).Infof("Running detection for task type: %s", taskType)
results, err := detector.ScanForTasks(filteredMetrics, clusterInfo)
if err != nil {
glog.Errorf("Failed to scan for %s tasks: %v", taskType, err)
continue
}
// Convert results to existing system format and check for conflicts
for _, result := range results {
existingResult := s.convertToExistingFormat(result)
if existingResult != nil {
// Double-check for conflicts with pending operations
opType := s.mapMaintenanceTaskTypeToPendingOperationType(existingResult.TaskType)
if !s.pendingOperations.WouldConflictWithPending(existingResult.VolumeID, opType) {
// All task types should now have TypedParams populated during detection phase
if existingResult.TypedParams == nil {
glog.Warningf("Task %s for volume %d has no typed parameters - skipping (task parameter creation may have failed)",
existingResult.TaskType, existingResult.VolumeID)
continue
}
allResults = append(allResults, existingResult)
} else {
glog.V(2).Infof("Skipping task %s for volume %d due to conflict with pending operation",
existingResult.TaskType, existingResult.VolumeID)
}
}
}
glog.V(2).Infof("Found %d %s tasks", len(results), taskType)
}
return allResults, nil
}
// SetDefaultReplicaPlacement records the master's default replication so detectors
// can use it as the replica-placement fallback (matching the shell).
func (s *MaintenanceIntegration) SetDefaultReplicaPlacement(replicaPlacement string) {
s.defaultReplicaPlacement = replicaPlacement
}
// UpdateTopologyInfo updates the volume shard tracker with topology information for empty servers
func (s *MaintenanceIntegration) UpdateTopologyInfo(topologyInfo *master_pb.TopologyInfo) error {
// Log topology details before update for diagnostics
if topologyInfo != nil {
dcCount, nodeCount, diskCount := topology.CountTopologyResources(topologyInfo)
glog.V(2).Infof("UpdateTopologyInfo: received topology with %d datacenters, %d nodes, %d disks",
dcCount, nodeCount, diskCount)
} else {
glog.Warningf("UpdateTopologyInfo: received nil topologyInfo")
}
err := s.activeTopology.UpdateTopology(topologyInfo)
if err != nil {
glog.Errorf("UpdateTopologyInfo: topology update failed: %v", err)
} else {
// Log success with current disk count
currentDiskCount := s.activeTopology.GetDiskCount()
glog.V(1).Infof("UpdateTopologyInfo: topology update successful, active topology now has %d disks", currentDiskCount)
}
return err
}
// convertToExistingFormat converts task results to existing system format using dynamic mapping
func (s *MaintenanceIntegration) convertToExistingFormat(result *types.TaskDetectionResult) *TaskDetectionResult {
// Convert types using mapping tables
existingType, exists := s.taskTypeMap[result.TaskType]
if !exists {
glog.Warningf("Unknown task type %s, skipping conversion", result.TaskType)
// Return nil to indicate conversion failed - caller should handle this
return nil
}
existingPriority, exists := s.priorityMap[result.Priority]
if !exists {
glog.Warningf("Unknown priority %s, defaulting to normal", result.Priority)
existingPriority = PriorityNormal
}
return &TaskDetectionResult{
TaskID: result.TaskID,
TaskType: existingType,
VolumeID: result.VolumeID,
Server: result.Server,
Collection: result.Collection,
Priority: existingPriority,
Reason: result.Reason,
TypedParams: result.TypedParams,
ScheduleAt: result.ScheduleAt,
}
}
// CanScheduleWithTaskSchedulers determines if a task can be scheduled using task schedulers with dynamic type conversion
func (s *MaintenanceIntegration) CanScheduleWithTaskSchedulers(task *MaintenanceTask, runningTasks []*MaintenanceTask, availableWorkers []*MaintenanceWorker) bool {
// Convert existing types to task types using mapping
taskType, exists := s.revTaskTypeMap[task.Type]
if !exists {
return false // Fallback to existing logic for unknown types
}
// Convert task objects
taskObject := s.convertTaskToTaskSystem(task)
if taskObject == nil {
return false
}
runningTaskObjects := s.convertTasksToTaskSystem(runningTasks)
workerObjects := s.convertWorkersToTaskSystem(availableWorkers)
// Get the appropriate scheduler
scheduler := s.taskRegistry.GetScheduler(taskType)
if scheduler == nil {
return false
}
canSchedule := scheduler.CanScheduleNow(taskObject, runningTaskObjects, workerObjects)
return canSchedule
}
// convertTaskToTaskSystem converts existing task to task system format using dynamic mapping
func (s *MaintenanceIntegration) convertTaskToTaskSystem(task *MaintenanceTask) *types.TaskInput {
// Convert task type using mapping
taskType, exists := s.revTaskTypeMap[task.Type]
if !exists {
glog.Errorf("Unknown task type %s in conversion, cannot convert task", task.Type)
// Return nil to indicate conversion failed
return nil
}
// Convert priority using mapping
priority, exists := s.revPriorityMap[task.Priority]
if !exists {
glog.Warningf("Unknown priority %d in conversion, defaulting to normal", task.Priority)
priority = types.TaskPriorityNormal
}
return &types.TaskInput{
ID: task.ID,
Type: taskType,
Priority: priority,
VolumeID: task.VolumeID,
Server: task.Server,
Collection: task.Collection,
TypedParams: task.TypedParams,
CreatedAt: task.CreatedAt,
}
}
// convertTasksToTaskSystem converts multiple tasks
func (s *MaintenanceIntegration) convertTasksToTaskSystem(tasks []*MaintenanceTask) []*types.TaskInput {
var result []*types.TaskInput
for _, task := range tasks {
converted := s.convertTaskToTaskSystem(task)
if converted != nil {
result = append(result, converted)
}
}
return result
}
// convertWorkersToTaskSystem converts workers to task system format using dynamic mapping
func (s *MaintenanceIntegration) convertWorkersToTaskSystem(workers []*MaintenanceWorker) []*types.WorkerData {
var result []*types.WorkerData
for _, worker := range workers {
capabilities := make([]types.TaskType, 0, len(worker.Capabilities))
for _, cap := range worker.Capabilities {
// Convert capability using mapping
taskType, exists := s.revTaskTypeMap[cap]
if exists {
capabilities = append(capabilities, taskType)
} else {
glog.V(3).Infof("Unknown capability %s for worker %s, skipping", cap, worker.ID)
}
}
result = append(result, &types.WorkerData{
ID: worker.ID,
Address: worker.Address,
Capabilities: capabilities,
MaxConcurrent: worker.MaxConcurrent,
CurrentLoad: worker.CurrentLoad,
})
}
return result
}
// GetTaskScheduler returns the scheduler for a task type using dynamic mapping
func (s *MaintenanceIntegration) GetTaskScheduler(taskType MaintenanceTaskType) types.TaskScheduler {
// Convert task type using mapping
taskSystemType, exists := s.revTaskTypeMap[taskType]
if !exists {
glog.V(3).Infof("Unknown task type %s for scheduler", taskType)
return nil
}
return s.taskRegistry.GetScheduler(taskSystemType)
}
// GetUIProvider returns the UI provider for a task type using dynamic mapping
func (s *MaintenanceIntegration) GetUIProvider(taskType MaintenanceTaskType) types.TaskUIProvider {
// Convert task type using mapping
taskSystemType, exists := s.revTaskTypeMap[taskType]
if !exists {
glog.V(3).Infof("Unknown task type %s for UI provider", taskType)
return nil
}
return s.uiRegistry.GetProvider(taskSystemType)
}
// GetAllTaskStats returns stats for all registered tasks
func (s *MaintenanceIntegration) GetAllTaskStats() []*types.TaskStats {
var stats []*types.TaskStats
for taskType, detector := range s.taskRegistry.GetAllDetectors() {
uiProvider := s.uiRegistry.GetProvider(taskType)
if uiProvider == nil {
continue
}
stat := &types.TaskStats{
TaskType: taskType,
DisplayName: uiProvider.GetDisplayName(),
Enabled: detector.IsEnabled(),
LastScan: time.Now().Add(-detector.ScanInterval()),
NextScan: time.Now().Add(detector.ScanInterval()),
ScanInterval: detector.ScanInterval(),
MaxConcurrent: s.taskRegistry.GetScheduler(taskType).GetMaxConcurrent(),
// Would need to get these from actual queue/stats
PendingTasks: 0,
RunningTasks: 0,
CompletedToday: 0,
FailedToday: 0,
}
stats = append(stats, stat)
}
return stats
}
// mapMaintenanceTaskTypeToPendingOperationType converts a maintenance task type to a pending operation type
func (s *MaintenanceIntegration) mapMaintenanceTaskTypeToPendingOperationType(taskType MaintenanceTaskType) PendingOperationType {
switch taskType {
case MaintenanceTaskType("balance"):
return OpTypeVolumeBalance
case MaintenanceTaskType("erasure_coding"):
return OpTypeErasureCoding
case MaintenanceTaskType("vacuum"):
return OpTypeVacuum
case MaintenanceTaskType("replication"):
return OpTypeReplication
default:
// For other task types, assume they're volume operations
return OpTypeVolumeMove
}
}
// GetPendingOperations returns the pending operations tracker
func (s *MaintenanceIntegration) GetPendingOperations() *PendingOperations {
return s.pendingOperations
}
// GetActiveTopology returns the active topology for task detection
func (s *MaintenanceIntegration) GetActiveTopology() *topology.ActiveTopology {
return s.activeTopology
}
// SyncTask synchronizes a maintenance task with the active topology for capacity tracking
func (s *MaintenanceIntegration) SyncTask(task *MaintenanceTask) {
if s.activeTopology == nil {
return
}
// Convert task type
taskType, exists := s.revTaskTypeMap[task.Type]
if !exists {
return
}
// Convert status
var status topology.TaskStatus
switch task.Status {
case TaskStatusPending:
status = topology.TaskStatusPending
case TaskStatusAssigned, TaskStatusInProgress:
status = topology.TaskStatusInProgress
default:
return // Don't sync completed/failed/cancelled tasks
}
// Extract sources and destinations from TypedParams
var sources []topology.TaskSource
var destinations []topology.TaskDestination
var estimatedSize int64
if task.TypedParams != nil {
// Calculate storage impact for this task type
// Volume size is not currently used for Balance/Vacuum impact and is not stored in MaintenanceTask
sourceImpact, targetImpact := topology.CalculateTaskStorageImpact(topology.TaskType(string(taskType)), 0)
// Use unified sources and targets from TaskParams.
// Task protos store ServerAddresses (with gRPC port, e.g., "host:port.grpcPort")
// but the topology indexes disks by NodeId (e.g., "host:port").
// Strip the gRPC port suffix via ToHttpAddress() to match the topology key.
for _, src := range task.TypedParams.Sources {
resolvedSrc := pb.ServerAddress(src.Node).ToHttpAddress()
glog.V(2).Infof("SyncTask %s: source proto Node=%q resolved to %q, diskId=%d", task.ID, src.Node, resolvedSrc, src.DiskId)
sources = append(sources, topology.TaskSource{
SourceServer: resolvedSrc,
SourceDisk: src.DiskId,
StorageChange: sourceImpact,
})
// Sum estimated size from all sources
estimatedSize += int64(src.EstimatedSize)
}
for _, target := range task.TypedParams.Targets {
resolvedTarget := pb.ServerAddress(target.Node).ToHttpAddress()
glog.V(2).Infof("SyncTask %s: target proto Node=%q resolved to %q, diskId=%d", task.ID, target.Node, resolvedTarget, target.DiskId)
destinations = append(destinations, topology.TaskDestination{
TargetServer: resolvedTarget,
TargetDisk: target.DiskId,
StorageChange: targetImpact,
})
}
// Handle type-specific params for additional task-specific sync logic
if vacuumParams := task.TypedParams.GetVacuumParams(); vacuumParams != nil {
// TODO: Add vacuum-specific sync logic if necessary
} else if ecParams := task.TypedParams.GetErasureCodingParams(); ecParams != nil {
// TODO: Add EC-specific sync logic if necessary
} else if balanceParams := task.TypedParams.GetBalanceParams(); balanceParams != nil {
// TODO: Add balance-specific sync logic if necessary
}
}
// Restore into topology
s.activeTopology.RestoreMaintenanceTask(task.ID, task.VolumeID, topology.TaskType(string(taskType)), status, sources, destinations, estimatedSize)
}

View File

@@ -0,0 +1,675 @@
package maintenance
import (
"fmt"
"strings"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/ec_balance"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// BuildPolicyFromTaskConfigs loads each registered task's configuration and builds the
// MaintenancePolicy the maintenance system runs on.
//
// Every entry is produced by the task's own ToTaskPolicy(), so a policy entry always carries
// exactly what that task's config holds. Hand-copying the fields here instead made this a
// second, silently diverging definition of every task's policy: it dropped the erasure
// coding preferred tags and replica placement and the balance IO rate limit outright.
//
// Every task registered through base.RegisterTask needs an entry, because a task type the
// policy does not list has no enabled flag and no concurrency limit of its own -
// IsTaskEnabled reports false for a missing entry.
//
// configPersistence is duck-typed as interface{} because weed/admin/dash already imports this
// package, so importing *dash.ConfigPersistence back here would create an import cycle. It must be
// a value implementing the LoadXTaskPolicy() accessors the task loaders assert on; passing nil (or
// anything else) makes every task fall back to its compiled-in defaults.
func BuildPolicyFromTaskConfigs(configPersistence interface{}) *worker_pb.MaintenancePolicy {
policy := &worker_pb.MaintenancePolicy{
GlobalMaxConcurrent: 4,
DefaultRepeatIntervalSeconds: 6 * 3600, // 6 hours in seconds
DefaultCheckIntervalSeconds: 12 * 3600, // 12 hours in seconds
TaskPolicies: make(map[string]*worker_pb.TaskPolicy),
}
if vacuumConfig := vacuum.LoadConfigFromPersistence(configPersistence); vacuumConfig != nil {
policy.TaskPolicies[string(types.TaskTypeVacuum)] = vacuumConfig.ToTaskPolicy()
}
if ecConfig := erasure_coding.LoadConfigFromPersistence(configPersistence); ecConfig != nil {
policy.TaskPolicies[string(types.TaskTypeErasureCoding)] = ecConfig.ToTaskPolicy()
}
if balanceConfig := balance.LoadConfigFromPersistence(configPersistence); balanceConfig != nil {
policy.TaskPolicies[string(types.TaskTypeBalance)] = balanceConfig.ToTaskPolicy()
}
if ecBalanceConfig := ec_balance.LoadConfigFromPersistence(configPersistence); ecBalanceConfig != nil {
policy.TaskPolicies[string(types.TaskTypeECBalance)] = ecBalanceConfig.ToTaskPolicy()
}
glog.V(1).Infof("Built maintenance policy from separate task configs - %d task policies loaded", len(policy.TaskPolicies))
return policy
}
// MaintenanceManager coordinates the maintenance system
type MaintenanceManager struct {
config *MaintenanceConfig
scanner *MaintenanceScanner
queue *MaintenanceQueue
adminClient AdminClient
running bool
stopChan chan struct{}
// Error handling and backoff
errorCount int
lastError error
lastErrorTime time.Time
backoffDelay time.Duration
mutex sync.RWMutex
scanInProgress bool
}
// NewMaintenanceManager creates a new maintenance manager.
//
// configPersistence is the config store to read persisted task configs from when the policy has to
// be built here. See BuildPolicyFromTaskConfigs for why it is duck-typed; pass nil when no config
// store is available.
func NewMaintenanceManager(adminClient AdminClient, config *MaintenanceConfig, configPersistence interface{}) *MaintenanceManager {
if config == nil {
config = DefaultMaintenanceConfig()
}
// Use the policy from the config (which is populated from separate task files in LoadMaintenanceConfig)
policy := config.Policy
if policy == nil {
// Fallback: build policy from separate task configuration files if not already populated
policy = BuildPolicyFromTaskConfigs(configPersistence)
}
queue := NewMaintenanceQueue(policy)
scanner := NewMaintenanceScanner(adminClient, policy, queue)
return &MaintenanceManager{
config: config,
scanner: scanner,
queue: queue,
adminClient: adminClient,
stopChan: make(chan struct{}),
backoffDelay: time.Second, // Start with 1 second backoff
}
}
// Start begins the maintenance manager
func (mm *MaintenanceManager) Start() error {
if !mm.config.GetEnabled() {
glog.V(1).Infof("Maintenance system is disabled")
return nil
}
// Validate configuration durations to prevent ticker panics
if err := mm.validateConfig(); err != nil {
return fmt.Errorf("invalid maintenance configuration: %w", err)
}
mm.mutex.Lock()
mm.running = true
mm.mutex.Unlock()
// Start background processes
go mm.scanLoop()
go mm.cleanupLoop()
go mm.topologyStatusLoop() // Periodic diagnostic logging
glog.Infof("Maintenance manager started with scan interval %ds", mm.config.ScanIntervalSeconds)
return nil
}
// validateConfig validates the maintenance configuration durations
func (mm *MaintenanceManager) validateConfig() error {
if mm.config.ScanIntervalSeconds <= 0 {
glog.Warningf("Invalid scan interval %ds, using default 30m", mm.config.ScanIntervalSeconds)
mm.config.ScanIntervalSeconds = 30 * 60 // 30 minutes in seconds
}
if mm.config.CleanupIntervalSeconds <= 0 {
glog.Warningf("Invalid cleanup interval %ds, using default 24h", mm.config.CleanupIntervalSeconds)
mm.config.CleanupIntervalSeconds = 24 * 60 * 60 // 24 hours in seconds
}
if mm.config.WorkerTimeoutSeconds <= 0 {
glog.Warningf("Invalid worker timeout %ds, using default 5m", mm.config.WorkerTimeoutSeconds)
mm.config.WorkerTimeoutSeconds = 5 * 60 // 5 minutes in seconds
}
if mm.config.TaskTimeoutSeconds <= 0 {
glog.Warningf("Invalid task timeout %ds, using default 2h", mm.config.TaskTimeoutSeconds)
mm.config.TaskTimeoutSeconds = 2 * 60 * 60 // 2 hours in seconds
}
if mm.config.RetryDelaySeconds <= 0 {
glog.Warningf("Invalid retry delay %ds, using default 15m", mm.config.RetryDelaySeconds)
mm.config.RetryDelaySeconds = 15 * 60 // 15 minutes in seconds
}
if mm.config.TaskRetentionSeconds <= 0 {
glog.Warningf("Invalid task retention %ds, using default 168h", mm.config.TaskRetentionSeconds)
mm.config.TaskRetentionSeconds = 7 * 24 * 60 * 60 // 7 days in seconds
}
return nil
}
// IsRunning returns whether the maintenance manager is currently running.
// running is guarded by mm.mutex because the background loops read it on every
// iteration while Start and Stop are called from the admin server's goroutines.
func (mm *MaintenanceManager) IsRunning() bool {
mm.mutex.RLock()
defer mm.mutex.RUnlock()
return mm.running
}
// Stop terminates the maintenance manager. It is a no-op when the manager is not
// running, so a second call cannot close the already closed stop channel.
func (mm *MaintenanceManager) Stop() {
mm.mutex.Lock()
if !mm.running {
mm.mutex.Unlock()
return
}
mm.running = false
close(mm.stopChan)
mm.mutex.Unlock()
glog.Infof("Maintenance manager stopped")
}
// scanLoop periodically scans for maintenance tasks with adaptive timing
func (mm *MaintenanceManager) scanLoop() {
scanInterval := time.Duration(mm.config.ScanIntervalSeconds) * time.Second
// activeInterval is the interval the ticker is actually running at right now. It has
// to be tracked separately from the configured scanInterval, because the error backoff
// replaces the ticker with a much shorter one. Comparing the target against the
// configured interval instead never restores the normal cadence: once the errors stop,
// getScanInterval returns scanInterval again, the comparison comes out false, and the
// ticker is left at the backoff delay. A single transient scan failure therefore pinned
// the scanner to one scan per second forever.
activeInterval := scanInterval
ticker := time.NewTicker(activeInterval)
// Wrapped in a closure so the replacement ticker is stopped, not the one that happened
// to be current when the defer was registered.
defer func() { ticker.Stop() }()
for mm.IsRunning() {
select {
case <-mm.stopChan:
return
case <-ticker.C:
glog.V(1).Infof("Performing maintenance scan every %v", activeInterval)
// Use the same synchronization as TriggerScan to prevent concurrent scans
if err := mm.triggerScanInternal(false); err != nil {
glog.V(1).Infof("Scheduled scan skipped: %v", err)
}
// Adjust ticker interval based on error state (read error state safely)
currentInterval := mm.getScanInterval(scanInterval)
// Reset ticker whenever the target differs from what the ticker is running at,
// which covers both entering the backoff and returning to the normal cadence.
if currentInterval != activeInterval {
ticker.Stop()
ticker = time.NewTicker(currentInterval)
glog.V(1).Infof("Maintenance scan cadence changed from %v to %v", activeInterval, currentInterval)
activeInterval = currentInterval
}
}
}
}
// getScanInterval safely reads the current scan interval with error backoff
func (mm *MaintenanceManager) getScanInterval(baseInterval time.Duration) time.Duration {
mm.mutex.RLock()
defer mm.mutex.RUnlock()
if mm.errorCount > 0 {
// Use backoff delay when there are errors
currentInterval := mm.backoffDelay
if currentInterval > baseInterval {
// Don't make it longer than the configured interval * 10
maxInterval := baseInterval * 10
if currentInterval > maxInterval {
currentInterval = maxInterval
}
}
return currentInterval
}
return baseInterval
}
// cleanupLoop periodically cleans up old tasks and stale workers
func (mm *MaintenanceManager) cleanupLoop() {
cleanupInterval := time.Duration(mm.config.CleanupIntervalSeconds) * time.Second
ticker := time.NewTicker(cleanupInterval)
defer ticker.Stop()
for mm.IsRunning() {
select {
case <-mm.stopChan:
return
case <-ticker.C:
mm.performCleanup()
}
}
}
// topologyStatusLoop periodically logs topology status for diagnostics
func (mm *MaintenanceManager) topologyStatusLoop() {
// Log topology status every 5 minutes for diagnostic purposes
statusInterval := 5 * time.Minute
ticker := time.NewTicker(statusInterval)
defer ticker.Stop()
for mm.IsRunning() {
select {
case <-mm.stopChan:
return
case <-ticker.C:
mm.logTopologyStatus()
}
}
}
// logTopologyStatus logs current topology and worker status for diagnostics
func (mm *MaintenanceManager) logTopologyStatus() {
if mm.scanner == nil || mm.scanner.integration == nil {
glog.V(2).Infof("Topology status: scanner/integration not available")
return
}
activeTopology := mm.scanner.integration.GetActiveTopology()
if activeTopology == nil {
glog.V(1).Infof("Topology status: ActiveTopology is nil")
return
}
diskCount := activeTopology.GetDiskCount()
nodeCount := len(activeTopology.GetAllNodes())
// Get queue stats
stats := mm.queue.GetStats()
workerCount := len(mm.queue.GetWorkers())
mm.mutex.RLock()
errorCount := mm.errorCount
mm.mutex.RUnlock()
glog.V(1).Infof("Topology status: %d nodes, %d disks, %d workers, %d pending tasks, %d running tasks, errors: %d",
nodeCount, diskCount, workerCount,
stats.TasksByStatus[TaskStatusPending],
stats.TasksByStatus[TaskStatusInProgress]+stats.TasksByStatus[TaskStatusAssigned],
errorCount)
}
// performScan executes a maintenance scan with error handling and backoff
func (mm *MaintenanceManager) performScan() {
defer func() {
// Always reset scan in progress flag when done
mm.mutex.Lock()
mm.scanInProgress = false
mm.mutex.Unlock()
}()
glog.Infof("Starting maintenance scan...")
results, err := mm.scanner.ScanForMaintenanceTasks()
stats_collect.AdminMaintenanceLastScanTimestampSeconds.SetToCurrentTime()
if err != nil {
// Handle scan error
mm.mutex.Lock()
mm.handleScanError(err)
mm.mutex.Unlock()
glog.Warningf("Maintenance scan failed: %v", err)
return
}
// Scan succeeded - update state and process results
mm.handleScanSuccess(results)
}
// handleScanSuccess processes successful scan results with proper lock management
func (mm *MaintenanceManager) handleScanSuccess(results []*TaskDetectionResult) {
// Update manager state first
mm.mutex.Lock()
mm.resetErrorTracking()
taskCount := len(results)
mm.mutex.Unlock()
if taskCount > 0 {
// Count tasks by type for logging (outside of lock)
taskCounts := make(map[MaintenanceTaskType]int)
for _, result := range results {
taskCounts[result.TaskType]++
}
// Add tasks to queue (no manager lock held)
mm.queue.AddTasksFromResults(results)
// Log detailed scan results
glog.Infof("Maintenance scan completed: found %d tasks", taskCount)
for taskType, count := range taskCounts {
glog.Infof(" - %s: %d tasks", taskType, count)
}
} else {
glog.Infof("Maintenance scan completed: no maintenance tasks needed")
}
}
// handleScanError handles scan errors with exponential backoff and reduced logging
func (mm *MaintenanceManager) handleScanError(err error) {
now := time.Now()
mm.errorCount++
mm.lastError = err
mm.lastErrorTime = now
// Use exponential backoff with jitter
if mm.errorCount > 1 {
mm.backoffDelay = mm.backoffDelay * 2
if mm.backoffDelay > 5*time.Minute {
mm.backoffDelay = 5 * time.Minute // Cap at 5 minutes
}
}
// Reduce log frequency based on error count and time
shouldLog := false
if mm.errorCount <= 3 {
// Log first 3 errors immediately
shouldLog = true
} else if mm.errorCount <= 10 && mm.errorCount%3 == 0 {
// Log every 3rd error for errors 4-10
shouldLog = true
} else if mm.errorCount%10 == 0 {
// Log every 10th error after that
shouldLog = true
}
if shouldLog {
// Check if it's a connection error to provide better messaging
if isConnectionError(err) {
if mm.errorCount == 1 {
glog.Errorf("Maintenance scan failed: %v (will retry with backoff)", err)
} else {
glog.Errorf("Maintenance scan still failing after %d attempts: %v (backoff: %v)",
mm.errorCount, err, mm.backoffDelay)
}
} else {
glog.Errorf("Maintenance scan failed: %v", err)
}
} else {
// Use debug level for suppressed errors
glog.V(3).Infof("Maintenance scan failed (error #%d, suppressed): %v", mm.errorCount, err)
}
}
// resetErrorTracking resets error tracking when scan succeeds
func (mm *MaintenanceManager) resetErrorTracking() {
if mm.errorCount > 0 {
glog.V(1).Infof("Maintenance scan recovered after %d failed attempts", mm.errorCount)
mm.errorCount = 0
mm.lastError = nil
mm.backoffDelay = time.Second // Reset to initial delay
}
}
// isConnectionError checks if the error is a connection-related error
func isConnectionError(err error) bool {
if err == nil {
return false
}
errStr := err.Error()
return strings.Contains(errStr, "connection refused") ||
strings.Contains(errStr, "connection error") ||
strings.Contains(errStr, "dial tcp") ||
strings.Contains(errStr, "connection timeout") ||
strings.Contains(errStr, "no route to host") ||
strings.Contains(errStr, "network unreachable")
}
// performCleanup cleans up old tasks and stale workers
func (mm *MaintenanceManager) performCleanup() {
glog.V(2).Infof("Starting maintenance cleanup")
taskRetention := time.Duration(mm.config.TaskRetentionSeconds) * time.Second
workerTimeout := time.Duration(mm.config.WorkerTimeoutSeconds) * time.Second
removedTasks := mm.queue.CleanupOldTasks(taskRetention)
removedWorkers := mm.queue.RemoveStaleWorkers(workerTimeout)
// Clean up stale pending operations (operations running for more than 4 hours)
staleOperationTimeout := 4 * time.Hour
removedOperations := 0
if mm.scanner != nil && mm.scanner.integration != nil {
pendingOps := mm.scanner.integration.GetPendingOperations()
if pendingOps != nil {
removedOperations = pendingOps.CleanupStaleOperations(staleOperationTimeout)
}
}
if removedTasks > 0 || removedWorkers > 0 || removedOperations > 0 {
glog.V(1).Infof("Cleanup completed: removed %d old tasks, %d stale workers, and %d stale operations",
removedTasks, removedWorkers, removedOperations)
}
}
// GetQueue returns the maintenance queue
func (mm *MaintenanceManager) GetQueue() *MaintenanceQueue {
return mm.queue
}
// GetConfig returns the maintenance configuration
func (mm *MaintenanceManager) GetConfig() *MaintenanceConfig {
return mm.config
}
// GetStats returns maintenance statistics
func (mm *MaintenanceManager) GetStats() *MaintenanceStats {
stats := mm.queue.GetStats()
mm.mutex.RLock()
defer mm.mutex.RUnlock()
stats.LastScanTime = time.Now() // Would need to track this properly
// Calculate next scan time based on current error state
scanInterval := time.Duration(mm.config.ScanIntervalSeconds) * time.Second
nextScanInterval := scanInterval
if mm.errorCount > 0 {
nextScanInterval = mm.backoffDelay
maxInterval := scanInterval * 10
if nextScanInterval > maxInterval {
nextScanInterval = maxInterval
}
}
stats.NextScanTime = time.Now().Add(nextScanInterval)
return stats
}
// ReloadTaskConfigurations reloads task configurations from the current policy
func (mm *MaintenanceManager) ReloadTaskConfigurations() error {
mm.mutex.Lock()
defer mm.mutex.Unlock()
// Trigger configuration reload in the integration layer
if mm.scanner != nil && mm.scanner.integration != nil {
mm.scanner.integration.ConfigureTasksFromPolicy()
glog.V(1).Infof("Task configurations reloaded from policy")
return nil
}
return fmt.Errorf("integration not available for configuration reload")
}
// GetErrorState returns the current error state for monitoring
func (mm *MaintenanceManager) GetErrorState() (errorCount int, lastError error, backoffDelay time.Duration) {
mm.mutex.RLock()
defer mm.mutex.RUnlock()
return mm.errorCount, mm.lastError, mm.backoffDelay
}
// GetTasks returns tasks with filtering
func (mm *MaintenanceManager) GetTasks(status MaintenanceTaskStatus, taskType MaintenanceTaskType, limit int) []*MaintenanceTask {
return mm.queue.GetTasks(status, taskType, limit)
}
// GetWorkers returns all registered workers
func (mm *MaintenanceManager) GetWorkers() []*MaintenanceWorker {
return mm.queue.GetWorkers()
}
// GetWorkerSlots returns used/max task slots per worker ID.
func (mm *MaintenanceManager) GetWorkerSlots() map[string]WorkerSlots {
return mm.queue.GetWorkerSlots()
}
// TriggerScan manually triggers a maintenance scan
func (mm *MaintenanceManager) TriggerScan() error {
return mm.triggerScanInternal(true)
}
// triggerScanInternal handles both manual and automatic scan triggers
func (mm *MaintenanceManager) triggerScanInternal(isManual bool) error {
// running and scanInProgress are checked under one lock so a Stop that lands
// between the two checks cannot leave a scan running after shutdown.
mm.mutex.Lock()
if !mm.running {
mm.mutex.Unlock()
return fmt.Errorf("maintenance manager is not running")
}
// Prevent multiple concurrent scans
if mm.scanInProgress {
mm.mutex.Unlock()
if isManual {
glog.V(1).Infof("Manual scan already in progress, ignoring trigger request")
} else {
glog.V(2).Infof("Automatic scan already in progress, ignoring scheduled scan")
}
return fmt.Errorf("scan already in progress")
}
mm.scanInProgress = true
mm.mutex.Unlock()
go mm.performScan()
return nil
}
// UpdateConfig updates the maintenance configuration
func (mm *MaintenanceManager) UpdateConfig(config *MaintenanceConfig) error {
if config == nil {
return fmt.Errorf("config cannot be nil")
}
mm.config = config
mm.queue.policy = config.Policy
mm.scanner.policy = config.Policy
// Propagate global policy changes to individual task configuration files
if config.Policy != nil {
mm.saveTaskConfigsFromPolicy(config.Policy)
}
// The integration holds its own reference to the policy and is what pushes the
// enabled flag and the concurrency limit into the registered detectors and
// schedulers. Without this the queue and the scanner saw the new policy while the
// detectors kept scanning under the old one until the next restart.
if mm.scanner != nil && mm.scanner.integration != nil {
mm.scanner.integration.SetPolicy(config.Policy)
}
glog.V(1).Infof("Maintenance configuration updated")
return nil
}
// saveTaskConfigsFromPolicy propagates global policy settings to separate task configuration files
func (mm *MaintenanceManager) saveTaskConfigsFromPolicy(policy *worker_pb.MaintenancePolicy) {
if mm.queue.persistence == nil || policy == nil {
return
}
glog.V(1).Infof("Propagating maintenance policy changes to separate task configs")
for taskType, taskPolicy := range policy.TaskPolicies {
if err := mm.queue.persistence.SaveTaskPolicy(taskType, taskPolicy); err != nil {
glog.Errorf("Failed to save task policy for %s: %v", taskType, err)
}
}
}
// CancelTask cancels a pending task
func (mm *MaintenanceManager) CancelTask(taskID string) error {
mm.queue.mutex.Lock()
defer mm.queue.mutex.Unlock()
task, exists := mm.queue.tasks[taskID]
if !exists {
return fmt.Errorf("task %s not found", taskID)
}
if task.Status == TaskStatusPending {
task.Status = TaskStatusCancelled
task.CompletedAt = &[]time.Time{time.Now()}[0]
// Remove from pending tasks
for i, pendingTask := range mm.queue.pendingTasks {
if pendingTask.ID == taskID {
mm.queue.pendingTasks = append(mm.queue.pendingTasks[:i], mm.queue.pendingTasks[i+1:]...)
break
}
}
// Notify ActiveTopology to release capacity
if mm.scanner != nil && mm.scanner.integration != nil {
if at := mm.scanner.integration.GetActiveTopology(); at != nil {
_ = at.CompleteTask(taskID)
}
}
glog.V(2).Infof("Cancelled task %s", taskID)
return nil
}
return fmt.Errorf("task %s cannot be cancelled (status: %s)", taskID, task.Status)
}
// RegisterWorker registers a new worker
func (mm *MaintenanceManager) RegisterWorker(worker *MaintenanceWorker) {
mm.queue.RegisterWorker(worker)
}
// GetNextTask returns the next task for a worker
func (mm *MaintenanceManager) GetNextTask(workerID string, capabilities []MaintenanceTaskType) *MaintenanceTask {
return mm.queue.GetNextTask(workerID, capabilities)
}
// CompleteTask marks a task as completed
func (mm *MaintenanceManager) CompleteTask(taskID string, error string) {
mm.queue.CompleteTask(taskID, error)
}
// UpdateTaskProgress updates task progress
func (mm *MaintenanceManager) UpdateTaskProgress(taskID string, progress float64) {
mm.queue.UpdateTaskProgress(taskID, progress)
}
// UpdateWorkerHeartbeat updates worker heartbeat
func (mm *MaintenanceManager) UpdateWorkerHeartbeat(workerID string) {
mm.queue.UpdateWorkerHeartbeat(workerID)
}

View File

@@ -0,0 +1,140 @@
package maintenance
import (
"errors"
"testing"
"time"
)
func TestMaintenanceManager_ErrorHandling(t *testing.T) {
config := DefaultMaintenanceConfig()
config.ScanIntervalSeconds = 1 // Short interval for testing (1 second)
manager := NewMaintenanceManager(nil, config, nil)
// Test initial state
if manager.errorCount != 0 {
t.Errorf("Expected initial error count to be 0, got %d", manager.errorCount)
}
if manager.backoffDelay != time.Second {
t.Errorf("Expected initial backoff delay to be 1s, got %v", manager.backoffDelay)
}
// Test error handling
err := errors.New("dial tcp [::1]:19333: connect: connection refused")
manager.handleScanError(err)
if manager.errorCount != 1 {
t.Errorf("Expected error count to be 1, got %d", manager.errorCount)
}
if manager.lastError != err {
t.Errorf("Expected last error to be set")
}
// Test exponential backoff
initialDelay := manager.backoffDelay
manager.handleScanError(err)
if manager.backoffDelay != initialDelay*2 {
t.Errorf("Expected backoff delay to double, got %v", manager.backoffDelay)
}
if manager.errorCount != 2 {
t.Errorf("Expected error count to be 2, got %d", manager.errorCount)
}
// Test backoff cap
for i := 0; i < 10; i++ {
manager.handleScanError(err)
}
if manager.backoffDelay > 5*time.Minute {
t.Errorf("Expected backoff delay to be capped at 5 minutes, got %v", manager.backoffDelay)
}
// Test error reset
manager.resetErrorTracking()
if manager.errorCount != 0 {
t.Errorf("Expected error count to be reset to 0, got %d", manager.errorCount)
}
if manager.backoffDelay != time.Second {
t.Errorf("Expected backoff delay to be reset to 1s, got %v", manager.backoffDelay)
}
if manager.lastError != nil {
t.Errorf("Expected last error to be reset to nil")
}
}
func TestIsConnectionError(t *testing.T) {
tests := []struct {
err error
expected bool
}{
{nil, false},
{errors.New("connection refused"), true},
{errors.New("dial tcp [::1]:19333: connect: connection refused"), true},
{errors.New("connection error: desc = \"transport: Error while dialing\""), true},
{errors.New("connection timeout"), true},
{errors.New("no route to host"), true},
{errors.New("network unreachable"), true},
{errors.New("some other error"), false},
{errors.New("invalid argument"), false},
}
for _, test := range tests {
result := isConnectionError(test.err)
if result != test.expected {
t.Errorf("For error %v, expected %v, got %v", test.err, test.expected, result)
}
}
}
func TestMaintenanceManager_GetErrorState(t *testing.T) {
config := DefaultMaintenanceConfig()
manager := NewMaintenanceManager(nil, config, nil)
// Test initial state
errorCount, lastError, backoffDelay := manager.GetErrorState()
if errorCount != 0 || lastError != nil || backoffDelay != time.Second {
t.Errorf("Expected initial state to be clean")
}
// Add some errors
err := errors.New("test error")
manager.handleScanError(err)
manager.handleScanError(err)
errorCount, lastError, backoffDelay = manager.GetErrorState()
if errorCount != 2 || lastError != err || backoffDelay != 2*time.Second {
t.Errorf("Expected error state to be tracked correctly: count=%d, err=%v, delay=%v",
errorCount, lastError, backoffDelay)
}
}
func TestMaintenanceManager_LogThrottling(t *testing.T) {
config := DefaultMaintenanceConfig()
manager := NewMaintenanceManager(nil, config, nil)
// This is a basic test to ensure the error handling doesn't panic
// In practice, you'd want to capture log output to verify throttling
err := errors.New("test error")
// Generate many errors to test throttling
for i := 0; i < 25; i++ {
manager.handleScanError(err)
}
// Should not panic and should have capped backoff
if manager.backoffDelay > 5*time.Minute {
t.Errorf("Expected backoff to be capped at 5 minutes")
}
if manager.errorCount != 25 {
t.Errorf("Expected error count to be 25, got %d", manager.errorCount)
}
}

View File

@@ -0,0 +1,133 @@
package maintenance
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
)
// stubConfigPersistence implements the LoadXTaskPolicy accessors that the task config loaders
// type-assert on. The real implementation is *dash.ConfigPersistence, which this package cannot
// import: weed/admin/dash already imports weed/admin/maintenance, so the dependency only runs one
// way and the persistence argument has to stay duck-typed.
type stubConfigPersistence struct {
vacuum *worker_pb.TaskPolicy
ec *worker_pb.TaskPolicy
balance *worker_pb.TaskPolicy
ecBalance *worker_pb.TaskPolicy
}
func (s *stubConfigPersistence) LoadVacuumTaskPolicy() (*worker_pb.TaskPolicy, error) {
return s.vacuum, nil
}
func (s *stubConfigPersistence) LoadErasureCodingTaskPolicy() (*worker_pb.TaskPolicy, error) {
return s.ec, nil
}
func (s *stubConfigPersistence) LoadBalanceTaskPolicy() (*worker_pb.TaskPolicy, error) {
return s.balance, nil
}
func (s *stubConfigPersistence) LoadEcBalanceTaskPolicy() (*worker_pb.TaskPolicy, error) {
return s.ecBalance, nil
}
func disabledStub() *stubConfigPersistence {
return &stubConfigPersistence{
vacuum: &worker_pb.TaskPolicy{
Enabled: false,
MaxConcurrent: 2,
RepeatIntervalSeconds: 2 * 3600,
TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
VacuumConfig: &worker_pb.VacuumTaskConfig{GarbageThreshold: 0.3, MinVolumeAgeHours: 24},
},
},
ec: &worker_pb.TaskPolicy{
Enabled: false,
MaxConcurrent: 1,
RepeatIntervalSeconds: 3600,
TaskConfig: &worker_pb.TaskPolicy_ErasureCodingConfig{
ErasureCodingConfig: &worker_pb.ErasureCodingTaskConfig{FullnessRatio: 0.95, QuietForSeconds: 3600, MinVolumeSizeMb: 30},
},
},
balance: &worker_pb.TaskPolicy{
Enabled: false,
MaxConcurrent: 1,
RepeatIntervalSeconds: 30 * 60,
TaskConfig: &worker_pb.TaskPolicy_BalanceConfig{
BalanceConfig: &worker_pb.BalanceTaskConfig{ImbalanceThreshold: 0.2, MinServerCount: 7},
},
},
ecBalance: &worker_pb.TaskPolicy{
Enabled: false,
MaxConcurrent: 1,
RepeatIntervalSeconds: 60 * 60,
TaskConfig: &worker_pb.TaskPolicy_EcBalanceConfig{
EcBalanceConfig: &worker_pb.EcBalanceTaskConfig{ImbalanceThreshold: 0.2, MinServerCount: 5},
},
},
}
}
// TestBuildPolicyFromTaskConfigsUsesPersistence covers a reported bug: the persistence argument used to be a
// literal nil, which no type assertion can satisfy, so a task disabled on disk came back enabled.
func TestBuildPolicyFromTaskConfigsUsesPersistence(t *testing.T) {
policy := BuildPolicyFromTaskConfigs(disabledStub())
for _, taskType := range []string{"vacuum", "erasure_coding", "balance", "ec_balance"} {
taskPolicy := policy.TaskPolicies[taskType]
if taskPolicy == nil {
t.Fatalf("no %s task policy built", taskType)
}
if taskPolicy.Enabled {
t.Errorf("%s enabled = true, want false from the persisted config", taskType)
}
}
if got := policy.TaskPolicies["balance"].GetBalanceConfig().GetMinServerCount(); got != 7 {
t.Errorf("balance min server count = %d, want persisted 7", got)
}
}
// TestBuildPolicyFromTaskConfigsWithoutPersistence keeps the documented fallback: with no config
// store there is nothing to read, so the compiled-in defaults apply.
func TestBuildPolicyFromTaskConfigsWithoutPersistence(t *testing.T) {
policy := BuildPolicyFromTaskConfigs(nil)
for _, taskType := range []string{"vacuum", "erasure_coding", "balance", "ec_balance"} {
taskPolicy := policy.TaskPolicies[taskType]
if taskPolicy == nil {
t.Fatalf("no %s task policy built", taskType)
}
if !taskPolicy.Enabled {
t.Errorf("%s enabled = false, want the compiled-in default true", taskType)
}
}
}
// TestNewMaintenanceManagerUsesPersistenceForPolicyFallback covers the path the admin server takes
// when no maintenance.pb has been written yet: the config carries no policy, so the manager builds
// one itself and must read the persisted task configs to do it.
func TestNewMaintenanceManagerUsesPersistenceForPolicyFallback(t *testing.T) {
config := DefaultMaintenanceConfig()
if config.Policy != nil {
t.Fatal("default config unexpectedly carries a policy, test no longer covers the fallback")
}
manager := NewMaintenanceManager(nil, config, disabledStub())
policy := manager.queue.policy
if policy == nil {
t.Fatal("queue policy is nil")
}
if IsTaskEnabled(policy, MaintenanceTaskType("balance")) {
t.Error("balance enabled = true, want false from the persisted config")
}
if IsTaskEnabled(policy, MaintenanceTaskType("vacuum")) {
t.Error("vacuum enabled = true, want false from the persisted config")
}
if manager.scanner.policy != policy {
t.Error("scanner and queue disagree on the policy")
}
}

View File

@@ -0,0 +1,209 @@
package maintenance
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// The detectors and schedulers live in a process-global registry, so these tests restore
// whatever they change. Otherwise a test that disables a task leaks that state into every
// later test in the package.
func snapshotDetectorState(t *testing.T) {
t.Helper()
registry := tasks.GetGlobalTypesRegistry()
enabled := make(map[types.TaskType]bool)
maxConcurrent := make(map[types.TaskType]int)
for taskType, detector := range registry.GetAllDetectors() {
enabled[taskType] = detector.IsEnabled()
}
for taskType, scheduler := range registry.GetAllSchedulers() {
maxConcurrent[taskType] = scheduler.GetMaxConcurrent()
}
t.Cleanup(func() {
for taskType, detector := range registry.GetAllDetectors() {
if setter, ok := detector.(interface{ SetEnabled(bool) }); ok {
setter.SetEnabled(enabled[taskType])
}
}
for taskType, scheduler := range registry.GetAllSchedulers() {
if setter, ok := scheduler.(interface{ SetMaxConcurrent(int) }); ok {
setter.SetMaxConcurrent(maxConcurrent[taskType])
}
}
})
}
// TestPolicyReachesRegisteredDetectors is the regression test for the half of the scan-flood bug
// that a corrected policy alone did not fix: MaintenanceIntegration pushes the policy into
// detectors and schedulers through interface{ SetEnabled(bool) } and
// interface{ SetMaxConcurrent(int) } type assertions, but every task is backed by
// base.GenericDetector/base.GenericScheduler, which implemented neither. The assertions
// failed silently for every task on every startup, so the policy never reached
// detector.IsEnabled() - which is what ScanWithTaskDetectors gates scanning on.
func TestPolicyReachesRegisteredDetectors(t *testing.T) {
snapshotDetectorState(t)
registry := tasks.GetGlobalTypesRegistry()
if len(registry.GetAllDetectors()) == 0 {
t.Fatal("no detectors registered, the test cannot prove anything")
}
// Enable everything first so the disabling below cannot pass by accident.
policy := policyWithAllTasks(t, true)
NewMaintenanceIntegration(NewMaintenanceQueue(policy), policy)
for taskType, detector := range registry.GetAllDetectors() {
if !detector.IsEnabled() {
t.Fatalf("detector %s is disabled after an all-enabled policy was applied", taskType)
}
}
// Now disable everything through the policy and check it lands on the detectors.
policy = policyWithAllTasks(t, false)
NewMaintenanceIntegration(NewMaintenanceQueue(policy), policy)
for taskType, detector := range registry.GetAllDetectors() {
if detector.IsEnabled() {
t.Errorf("detector %s still reports enabled after the policy disabled it; "+
"the policy is not reaching the flag ScanWithTaskDetectors gates on", taskType)
}
}
for taskType, scheduler := range registry.GetAllSchedulers() {
if scheduler.IsEnabled() {
t.Errorf("scheduler %s still reports enabled after the policy disabled it", taskType)
}
}
}
// TestPolicyMaxConcurrentReachesSchedulers covers the SetMaxConcurrent half of the same
// wiring. GetMaxConcurrent is what MaintenanceQueue.getMaxConcurrentForTaskType asks before
// starting another task of a type.
func TestPolicyMaxConcurrentReachesSchedulers(t *testing.T) {
snapshotDetectorState(t)
const wantMaxConcurrent = 7
policy := policyWithAllTasks(t, true)
for _, taskPolicy := range policy.TaskPolicies {
taskPolicy.MaxConcurrent = wantMaxConcurrent
}
NewMaintenanceIntegration(NewMaintenanceQueue(policy), policy)
registry := tasks.GetGlobalTypesRegistry()
for taskType, scheduler := range registry.GetAllSchedulers() {
if _, covered := policy.TaskPolicies[string(taskType)]; !covered {
continue
}
if got := scheduler.GetMaxConcurrent(); got != wantMaxConcurrent {
t.Errorf("scheduler %s max concurrent = %d, want %d from the policy", taskType, got, wantMaxConcurrent)
}
}
}
// TestPolicyWithoutEntryLeavesTaskAlone guards the direction that would have been a silent
// outage: IsTaskEnabled reports false for a task type the policy does not list, so applying
// it unconditionally would disable every task the policy has no entry for.
func TestPolicyWithoutEntryLeavesTaskAlone(t *testing.T) {
snapshotDetectorState(t)
registry := tasks.GetGlobalTypesRegistry()
// Start from a policy that enables everything.
enabling := policyWithAllTasks(t, true)
NewMaintenanceIntegration(NewMaintenanceQueue(enabling), enabling)
// An empty policy has an entry for nothing at all.
empty := &MaintenancePolicy{TaskPolicies: make(map[string]*worker_pb.TaskPolicy)}
NewMaintenanceIntegration(NewMaintenanceQueue(empty), empty)
for taskType, detector := range registry.GetAllDetectors() {
if !detector.IsEnabled() {
t.Errorf("detector %s was disabled by a policy that has no entry for it; "+
"a missing entry means no opinion, not disabled", taskType)
}
}
}
// TestBuildPolicyCoversEveryRegisteredTask keeps BuildPolicyFromTaskConfigs and the task
// registry in step. A registered task with no policy entry has no enabled flag and no
// concurrency limit of its own, which is the state ec_balance was in.
func TestBuildPolicyCoversEveryRegisteredTask(t *testing.T) {
policy := BuildPolicyFromTaskConfigs(nil)
for taskType := range tasks.GetGlobalTypesRegistry().GetAllDetectors() {
if _, ok := policy.TaskPolicies[string(taskType)]; !ok {
t.Errorf("task %s is registered as a detector but BuildPolicyFromTaskConfigs "+
"builds no policy entry for it, so IsTaskEnabled reports false for it", taskType)
}
}
}
// TestBuildPolicyIncludesEcBalance pins the specific gap above.
func TestBuildPolicyIncludesEcBalance(t *testing.T) {
policy := BuildPolicyFromTaskConfigs(nil)
ecBalance := policy.TaskPolicies[string(types.TaskTypeECBalance)]
if ecBalance == nil {
t.Fatal("no ec_balance entry in the built maintenance policy")
}
if !IsTaskEnabled(policy, MaintenanceTaskType(types.TaskTypeECBalance)) {
t.Error("ec_balance reports disabled with no persisted config, want the compiled-in default of enabled")
}
if ecBalance.GetEcBalanceConfig() == nil {
t.Error("ec_balance policy entry carries no EcBalanceConfig")
}
}
// policyWithAllTasks builds a policy that has an entry for every registered task type,
// all sharing the same enabled flag.
func policyWithAllTasks(t *testing.T, enabled bool) *MaintenancePolicy {
t.Helper()
policy := &MaintenancePolicy{
GlobalMaxConcurrent: 4,
TaskPolicies: make(map[string]*worker_pb.TaskPolicy),
}
for taskType := range tasks.GetGlobalTypesRegistry().GetAllDetectors() {
policy.TaskPolicies[string(taskType)] = &worker_pb.TaskPolicy{
Enabled: enabled,
MaxConcurrent: 1,
RepeatIntervalSeconds: 3600,
}
}
return policy
}
// TestPolicyHelpersTolerateNilPolicy pins the nil handling in the exported policy helpers.
// MaintenanceConfig.Policy is nil until something builds one, and UpdateConfig accepts a
// config that has none, so these are reachable with a nil policy - GetTaskPolicy used to
// dereference it and take the admin process down.
func TestPolicyHelpersTolerateNilPolicy(t *testing.T) {
const taskType = MaintenanceTaskType("balance")
if got := GetTaskPolicy(nil, taskType); got != nil {
t.Errorf("GetTaskPolicy(nil) = %v, want nil", got)
}
if IsTaskEnabled(nil, taskType) {
t.Error("IsTaskEnabled(nil) = true, want false")
}
if got := GetMaxConcurrent(nil, taskType); got != 1 {
t.Errorf("GetMaxConcurrent(nil) = %d, want the safe default 1", got)
}
if got := GetRepeatInterval(nil, taskType); got != 0 {
t.Errorf("GetRepeatInterval(nil) = %d, want 0 so callers fall back to their own default", got)
}
// A policy that exists but lists nothing must behave the same way.
empty := &MaintenancePolicy{}
if got := GetTaskPolicy(empty, taskType); got != nil {
t.Errorf("GetTaskPolicy(empty) = %v, want nil", got)
}
if IsTaskEnabled(empty, taskType) {
t.Error("IsTaskEnabled(empty) = true, want false")
}
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,168 @@
package maintenance
import (
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
// scanCadenceClient is a minimal AdminClient whose first call blocks until the test
// releases it. Blocking the first scan is what makes the cadence test deterministic:
// the scan loop evaluates the error state right after triggering a scan, and that scan
// runs in its own goroutine, so without the block the loop could observe the error
// counter either before or after the scan finished.
type scanCadenceClient struct {
mu sync.Mutex
calls int
release chan struct{}
}
func (c *scanCadenceClient) WithMasterClient(fn func(client master_pb.SeaweedClient) error) error {
c.mu.Lock()
c.calls++
first := c.calls == 1
c.mu.Unlock()
if first {
<-c.release
}
// Returning nil without invoking fn yields an empty, successful scan.
return nil
}
func (c *scanCadenceClient) callCount() int {
c.mu.Lock()
defer c.mu.Unlock()
return c.calls
}
// TestScanLoopRestoresIntervalAfterBackoff is the regression test for the scan flood. The loop shortens its ticker to the
// error backoff delay after a failed scan. It used to compare the target interval against
// the *configured* scan interval rather than against the interval the ticker was actually
// running at, so once the errors stopped the comparison came out false and the ticker was
// never restored: one transient failure pinned the scanner to one scan per second for as
// long as the process lived.
//
// The test seeds the error state, lets the loop drop into the 1s backoff, then lets a scan
// succeed and asserts the loop goes back to the configured 3s cadence instead of keeping
// the 1s one.
func TestScanLoopRestoresIntervalAfterBackoff(t *testing.T) {
const baseInterval = 3 * time.Second
client := &scanCadenceClient{release: make(chan struct{})}
config := DefaultMaintenanceConfig()
config.ScanIntervalSeconds = int32(baseInterval / time.Second)
manager := NewMaintenanceManager(client, config, nil)
// Seed a failed scan so the very first cadence decision drops to the backoff delay.
// backoffDelay is what getScanInterval returns while errorCount > 0.
manager.mutex.Lock()
manager.errorCount = 1
manager.backoffDelay = time.Second
manager.running = true
manager.mutex.Unlock()
go manager.scanLoop()
defer manager.Stop()
// First tick at ~3s: the scan blocks in the client, so the loop still sees errorCount == 1
// and switches the ticker to the 1s backoff.
deadline := time.Now().Add(baseInterval + 2*time.Second)
for client.callCount() == 0 {
if time.Now().After(deadline) {
t.Fatal("scan loop never triggered its first scan")
}
time.Sleep(20 * time.Millisecond)
}
// Let the blocked scan complete successfully, which clears the error state.
close(client.release)
// Wait for the error state to clear so the next cadence decision is unambiguous.
deadline = time.Now().Add(2 * time.Second)
for {
errorCount, _, _ := manager.GetErrorState()
if errorCount == 0 {
break
}
if time.Now().After(deadline) {
t.Fatalf("error tracking never reset, errorCount=%d", errorCount)
}
time.Sleep(20 * time.Millisecond)
}
// The ticker is at 1s now. Wait for the next tick, where the loop must notice the
// recovery and restore the 3s cadence.
time.Sleep(1500 * time.Millisecond)
before := client.callCount()
// Observe a window that a 1s cadence would fill with scans and a 3s cadence would not.
const window = 5 * time.Second
time.Sleep(window)
callsInWindow := client.callCount() - before
// One scan makes two master-client calls. 3s cadence: at most 2 scans (4 calls) in 5s,
// even if a stall shifts the window. 1s cadence: about 5 scans (10 calls).
if callsInWindow > 4 {
t.Errorf("scan loop made %d master-client calls in %v after recovering from a failed scan; "+
"the ticker was left at the %v backoff instead of returning to the configured %v",
callsInWindow, window, time.Second, baseInterval)
}
if callsInWindow == 0 {
t.Errorf("scan loop ran no scans in %v, expected the %v cadence to fire at least once", window, baseInterval)
}
}
// TestScanLoopEntersBackoffOnError checks the other half of the cadence logic: a failing
// scan still has to shorten the ticker, so the fix above did not simply pin the loop to
// the configured interval.
func TestScanLoopEntersBackoffOnError(t *testing.T) {
config := DefaultMaintenanceConfig()
config.ScanIntervalSeconds = 600
manager := NewMaintenanceManager(nil, config, nil)
baseInterval := time.Duration(config.ScanIntervalSeconds) * time.Second
if got := manager.getScanInterval(baseInterval); got != baseInterval {
t.Errorf("healthy scan interval = %v, want the configured %v", got, baseInterval)
}
manager.mutex.Lock()
manager.errorCount = 1
manager.backoffDelay = time.Second
manager.mutex.Unlock()
if got := manager.getScanInterval(baseInterval); got != time.Second {
t.Errorf("scan interval while failing = %v, want the %v backoff", got, time.Second)
}
manager.mutex.Lock()
manager.resetErrorTracking()
manager.mutex.Unlock()
if got := manager.getScanInterval(baseInterval); got != baseInterval {
t.Errorf("scan interval after recovery = %v, want the configured %v", got, baseInterval)
}
}
// TestStopIsIdempotent guards the stop channel against a double close, which used to panic
// when StopMaintenanceManager ran twice (for example on a shutdown path that also runs on
// a signal handler).
func TestStopIsIdempotent(t *testing.T) {
manager := NewMaintenanceManager(nil, DefaultMaintenanceConfig(), nil)
manager.mutex.Lock()
manager.running = true
manager.mutex.Unlock()
manager.Stop()
manager.Stop()
if manager.IsRunning() {
t.Error("manager still reports running after Stop")
}
}

View File

@@ -0,0 +1,224 @@
package maintenance
import (
"context"
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// NewMaintenanceScanner creates a new maintenance scanner
func NewMaintenanceScanner(adminClient AdminClient, policy *MaintenancePolicy, queue *MaintenanceQueue) *MaintenanceScanner {
scanner := &MaintenanceScanner{
adminClient: adminClient,
policy: policy,
queue: queue,
lastScan: make(map[MaintenanceTaskType]time.Time),
}
// Initialize integration
scanner.integration = NewMaintenanceIntegration(queue, policy)
// Set up bidirectional relationship
queue.SetIntegration(scanner.integration)
glog.V(1).Infof("Initialized maintenance scanner with task system")
return scanner
}
// ScanForMaintenanceTasks analyzes the cluster and generates maintenance tasks
func (ms *MaintenanceScanner) ScanForMaintenanceTasks() ([]*TaskDetectionResult, error) {
// Get volume health metrics directly in task-system format, along with topology info
taskMetrics, topologyInfo, err := ms.getVolumeHealthMetrics()
if err != nil {
return nil, fmt.Errorf("failed to get volume health metrics: %w", err)
}
// Use task system for all task types
if ms.integration != nil {
// Update topology information for complete cluster view (including empty servers)
// This must happen before task detection to ensure EC placement can consider all servers
if topologyInfo != nil {
if err := ms.integration.UpdateTopologyInfo(topologyInfo); err != nil {
glog.Errorf("Failed to update topology info for empty servers: %v", err)
// Don't fail the scan - continue with just volume-bearing servers
} else {
glog.V(1).Infof("Updated topology info for complete cluster view including empty servers")
}
}
// Refresh the master's default replication so detectors can use it as the
// replica-placement fallback (matches the shell ec.balance default).
ms.integration.SetDefaultReplicaPlacement(ms.getDefaultReplicaPlacement())
// Use task detection system with complete cluster information
results, err := ms.integration.ScanWithTaskDetectors(taskMetrics)
if err != nil {
glog.Errorf("Task scanning failed: %v", err)
return nil, err
}
glog.V(1).Infof("Maintenance scan completed: found %d tasks", len(results))
return results, nil
}
// No integration available
glog.Warningf("No integration available, no tasks will be scheduled")
return []*TaskDetectionResult{}, nil
}
// getDefaultReplicaPlacement reads the master's configured default replication,
// used by detectors as the replica-placement fallback. Returns "" on error so
// detectors fall back to even spread rather than failing the scan.
func (ms *MaintenanceScanner) getDefaultReplicaPlacement() string {
var replicaPlacement string
err := ms.adminClient.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
if err != nil {
return err
}
replicaPlacement = resp.DefaultReplication
return nil
})
if err != nil {
glog.V(1).Infof("could not fetch master default replication: %v", err)
return ""
}
return replicaPlacement
}
// getVolumeHealthMetrics collects health information for all volumes.
// Returns metrics in task-system format directly (no intermediate copy) and
// the topology info for updating the active topology.
func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*types.VolumeHealthMetrics, *master_pb.TopologyInfo, error) {
var metrics []*types.VolumeHealthMetrics
var topologyInfo *master_pb.TopologyInfo
glog.V(1).Infof("Collecting volume health metrics from master")
err := ms.adminClient.WithMasterClient(func(client master_pb.SeaweedClient) error {
// Streamed, so the master never builds the whole listing to send it.
resp, err := pb.CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{})
if err != nil {
return err
}
if resp.TopologyInfo == nil {
glog.Warningf("No topology info received from master")
return nil
}
volumeSizeLimitBytes := uint64(resp.VolumeSizeLimitMb) * 1024 * 1024 // Convert MB to bytes
// Track node counts for summary logging (avoid accumulating full ID slices)
var totalNodes, nodesWithVolumes, nodesWithoutVolumes int
for _, dc := range resp.TopologyInfo.DataCenterInfos {
glog.V(3).Infof("Processing datacenter: %s", dc.Id)
for _, rack := range dc.RackInfos {
glog.V(3).Infof("Processing rack: %s in datacenter: %s", rack.Id, dc.Id)
for _, node := range rack.DataNodeInfos {
totalNodes++
glog.V(3).Infof("Found volume server in topology: %s (disks: %d)", node.Id, len(node.DiskInfos))
hasVolumes := false
// Process each disk on this node
for diskType, diskInfo := range node.DiskInfos {
if len(diskInfo.VolumeInfos) > 0 {
hasVolumes = true
glog.V(3).Infof("Volume server %s disk %s has %d volumes", node.Id, diskType, len(diskInfo.VolumeInfos))
}
// Process volumes on this specific disk
for _, volInfo := range diskInfo.VolumeInfos {
metric := &types.VolumeHealthMetrics{
VolumeID: volInfo.Id,
Server: node.Id,
ServerAddress: node.Address,
DiskType: diskType, // Track which disk this volume is on
DiskId: volInfo.DiskId, // Use disk ID from volume info
DataCenter: dc.Id, // Data center from current loop
Rack: rack.Id, // Rack from current loop
Collection: volInfo.Collection,
Size: volInfo.Size,
DeletedBytes: volInfo.DeletedByteCount,
LastModified: time.Unix(int64(volInfo.ModifiedAtSecond), 0),
IsReadOnly: volInfo.ReadOnly,
HasRemoteCopy: volInfo.RemoteStorageName != "",
IsECVolume: false, // Will be determined from volume structure
ReplicaCount: 1, // Will be counted
ExpectedReplicas: int(volInfo.ReplicaPlacement),
}
// Calculate derived metrics
if metric.Size > 0 {
metric.GarbageRatio = float64(metric.DeletedBytes) / float64(metric.Size)
// Calculate fullness ratio using actual volume size limit from master
metric.FullnessRatio = float64(metric.Size) / float64(volumeSizeLimitBytes)
}
metric.Age = time.Since(metric.LastModified)
glog.V(4).Infof("Volume %d on %s:%s (ID %d): size=%d, limit=%d, fullness=%.2f",
metric.VolumeID, metric.Server, metric.DiskType, metric.DiskId, metric.Size, volumeSizeLimitBytes, metric.FullnessRatio)
metrics = append(metrics, metric)
}
}
if hasVolumes {
nodesWithVolumes++
} else {
nodesWithoutVolumes++
glog.V(1).Infof("Volume server %s found in topology but has no volumes", node.Id)
}
}
}
}
glog.Infof("Topology discovery: %d volume servers (%d with volumes, %d without)",
totalNodes, nodesWithVolumes, nodesWithoutVolumes)
// Return topology info as a local value (not retained on the scanner struct)
topologyInfo = resp.TopologyInfo
return nil
})
if err != nil {
glog.Errorf("Failed to get volume health metrics: %v", err)
return nil, nil, err
}
glog.V(1).Infof("Successfully collected metrics for %d actual volumes with disk ID information", len(metrics))
// Count actual replicas and identify EC volumes
ms.enrichVolumeMetrics(metrics)
return metrics, topologyInfo, nil
}
// enrichVolumeMetrics adds additional information like replica counts
func (ms *MaintenanceScanner) enrichVolumeMetrics(metrics []*types.VolumeHealthMetrics) {
// Group volumes by ID to count replicas
volumeGroups := make(map[uint32][]*types.VolumeHealthMetrics)
for _, metric := range metrics {
volumeGroups[metric.VolumeID] = append(volumeGroups[metric.VolumeID], metric)
}
// Update replica counts for actual volumes
for volumeID, replicas := range volumeGroups {
replicaCount := len(replicas)
for _, replica := range replicas {
replica.ReplicaCount = replicaCount
}
glog.V(4).Infof("Volume %d has %d replicas", volumeID, replicaCount)
}
// TODO: Identify EC volumes by checking volume structure
// This would require querying volume servers for EC shard information
}

View File

@@ -0,0 +1,361 @@
package maintenance
import (
"html/template"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
)
// AdminClient interface defines what the maintenance system needs from the admin server
type AdminClient interface {
WithMasterClient(fn func(client master_pb.SeaweedClient) error) error
}
// MaintenanceTaskType represents different types of maintenance operations
type MaintenanceTaskType string
// MaintenanceTaskPriority represents task execution priority
type MaintenanceTaskPriority int
const (
PriorityLow MaintenanceTaskPriority = iota
PriorityNormal
PriorityHigh
PriorityCritical
)
// MaintenanceTaskStatus represents the current status of a task
type MaintenanceTaskStatus string
const (
TaskStatusPending MaintenanceTaskStatus = "pending"
TaskStatusAssigned MaintenanceTaskStatus = "assigned"
TaskStatusInProgress MaintenanceTaskStatus = "in_progress"
TaskStatusCompleted MaintenanceTaskStatus = "completed"
TaskStatusFailed MaintenanceTaskStatus = "failed"
TaskStatusCancelled MaintenanceTaskStatus = "cancelled"
)
// MaintenanceTask represents a single maintenance operation
type MaintenanceTask struct {
ID string `json:"id"`
Type MaintenanceTaskType `json:"type"`
Priority MaintenanceTaskPriority `json:"priority"`
Status MaintenanceTaskStatus `json:"status"`
VolumeID uint32 `json:"volume_id,omitempty"`
Server string `json:"server,omitempty"`
Collection string `json:"collection,omitempty"`
TypedParams *worker_pb.TaskParams `json:"typed_params,omitempty"`
Reason string `json:"reason"`
CreatedAt time.Time `json:"created_at"`
ScheduledAt time.Time `json:"scheduled_at"`
StartedAt *time.Time `json:"started_at,omitempty"`
CompletedAt *time.Time `json:"completed_at,omitempty"`
WorkerID string `json:"worker_id,omitempty"`
Error string `json:"error,omitempty"`
Progress float64 `json:"progress"` // 0-100
RetryCount int `json:"retry_count"`
MaxRetries int `json:"max_retries"`
// Enhanced fields for detailed task tracking
CreatedBy string `json:"created_by,omitempty"` // Who/what created this task
CreationContext string `json:"creation_context,omitempty"` // Additional context about creation
AssignmentHistory []*TaskAssignmentRecord `json:"assignment_history,omitempty"` // History of worker assignments
DetailedReason string `json:"detailed_reason,omitempty"` // More detailed explanation than Reason
Tags map[string]string `json:"tags,omitempty"` // Additional metadata tags
}
// TaskAssignmentRecord tracks when a task was assigned to a worker
type TaskAssignmentRecord struct {
WorkerID string `json:"worker_id"`
WorkerAddress string `json:"worker_address"`
AssignedAt time.Time `json:"assigned_at"`
UnassignedAt *time.Time `json:"unassigned_at,omitempty"`
Reason string `json:"reason"` // Why was it assigned/unassigned
}
// TaskExecutionLog represents a log entry from task execution
type TaskExecutionLog struct {
Timestamp time.Time `json:"timestamp"`
Level string `json:"level"` // "info", "warn", "error", "debug"
Message string `json:"message"`
Source string `json:"source"` // Which component logged this
TaskID string `json:"task_id"`
WorkerID string `json:"worker_id"`
// Optional structured fields carried from worker logs
Fields map[string]string `json:"fields,omitempty"`
// Optional progress/status carried from worker logs
Progress *float64 `json:"progress,omitempty"`
Status string `json:"status,omitempty"`
}
// TaskDetailData represents comprehensive information about a task for the detail view
type TaskDetailData struct {
Task *MaintenanceTask `json:"task"`
AssignmentHistory []*TaskAssignmentRecord `json:"assignment_history"`
ExecutionLogs []*TaskExecutionLog `json:"execution_logs"`
RelatedTasks []*MaintenanceTask `json:"related_tasks,omitempty"` // Other tasks on same volume/server
WorkerInfo *MaintenanceWorker `json:"worker_info,omitempty"` // Current or last assigned worker
CreationMetrics *TaskCreationMetrics `json:"creation_metrics,omitempty"` // Metrics that led to task creation
LastUpdated time.Time `json:"last_updated"`
}
// TaskCreationMetrics holds metrics that led to the task being created
type TaskCreationMetrics struct {
TriggerMetric string `json:"trigger_metric"` // What metric triggered this task
MetricValue float64 `json:"metric_value"` // Value of the trigger metric
Threshold float64 `json:"threshold"` // Threshold that was exceeded
VolumeMetrics *VolumeHealthMetrics `json:"volume_metrics,omitempty"`
AdditionalData map[string]interface{} `json:"additional_data,omitempty"`
}
// MaintenanceConfig holds configuration for the maintenance system
// DEPRECATED: Use worker_pb.MaintenanceConfig instead
type MaintenanceConfig = worker_pb.MaintenanceConfig
// MaintenancePolicy defines policies for maintenance operations
// DEPRECATED: Use worker_pb.MaintenancePolicy instead
type MaintenancePolicy = worker_pb.MaintenancePolicy
// TaskPolicy represents configuration for a specific task type
// DEPRECATED: Use worker_pb.TaskPolicy instead
type TaskPolicy = worker_pb.TaskPolicy
// TaskPersistence interface for task state persistence
type TaskPersistence interface {
SaveTaskState(task *MaintenanceTask) error
LoadTaskState(taskID string) (*MaintenanceTask, error)
LoadAllTaskStates() ([]*MaintenanceTask, error)
DeleteTaskState(taskID string) error
DeleteAllTaskStates() error
CleanupCompletedTasks() error
// Policy persistence
SaveTaskPolicy(taskType string, policy *TaskPolicy) error
}
// Default configuration values
func DefaultMaintenanceConfig() *MaintenanceConfig {
return DefaultMaintenanceConfigProto()
}
// Policy helper functions (since we can't add methods to type aliases)
// GetTaskPolicy returns the policy for a specific task type, or nil when the maintenance
// policy has no entry for it.
//
// A nil maintenance policy is a legitimate state, not a programming error:
// MaintenanceConfig.Policy is unset until something builds one, and UpdateConfig accepts a
// config that carries none. Dereferencing it here panicked the whole admin process.
func GetTaskPolicy(mp *MaintenancePolicy, taskType MaintenanceTaskType) *TaskPolicy {
if mp == nil || mp.TaskPolicies == nil {
return nil
}
return mp.TaskPolicies[string(taskType)]
}
// IsTaskEnabled returns whether a task type is enabled
func IsTaskEnabled(mp *MaintenancePolicy, taskType MaintenanceTaskType) bool {
policy := GetTaskPolicy(mp, taskType)
if policy == nil {
return false
}
return policy.Enabled
}
// GetMaxConcurrent returns the max concurrent limit for a task type
func GetMaxConcurrent(mp *MaintenancePolicy, taskType MaintenanceTaskType) int {
policy := GetTaskPolicy(mp, taskType)
if policy == nil {
return 1
}
return int(policy.MaxConcurrent)
}
// GetRepeatInterval returns the repeat interval for a task type
func GetRepeatInterval(mp *MaintenancePolicy, taskType MaintenanceTaskType) int {
policy := GetTaskPolicy(mp, taskType)
if policy == nil {
if mp == nil {
return 0
}
return int(mp.DefaultRepeatIntervalSeconds)
}
return int(policy.RepeatIntervalSeconds)
}
// SetTaskConfig sets a configuration value for a task type (legacy method - use typed setters above)
// Note: SetTaskConfig was removed - use typed setters: SetVacuumTaskConfig, SetErasureCodingTaskConfig, SetBalanceTaskConfig, or SetReplicationTaskConfig
// MaintenanceWorker represents a worker instance
type MaintenanceWorker struct {
ID string `json:"id"`
Address string `json:"address"`
LastHeartbeat time.Time `json:"last_heartbeat"`
Status string `json:"status"` // active, inactive, busy
CurrentTask *MaintenanceTask `json:"current_task,omitempty"`
Capabilities []MaintenanceTaskType `json:"capabilities"`
MaxConcurrent int `json:"max_concurrent"`
CurrentLoad int `json:"current_load"`
}
// WorkerSlots is a point-in-time snapshot of one worker's task slot usage,
// taken under the queue lock so callers never read live worker fields.
type WorkerSlots struct {
Used int `json:"used"`
Max int `json:"max"`
}
// MaintenanceQueue manages the task queue and worker coordination
type MaintenanceQueue struct {
tasks map[string]*MaintenanceTask
workers map[string]*MaintenanceWorker
pendingTasks []*MaintenanceTask
mutex sync.RWMutex
policy *MaintenancePolicy
integration *MaintenanceIntegration
persistence TaskPersistence // Interface for task persistence
}
// MaintenanceScanner analyzes the cluster and generates maintenance tasks
type MaintenanceScanner struct {
adminClient AdminClient
policy *MaintenancePolicy
queue *MaintenanceQueue
lastScan map[MaintenanceTaskType]time.Time
integration *MaintenanceIntegration
}
// TaskDetectionResult represents the result of scanning for maintenance needs
type TaskDetectionResult struct {
TaskID string `json:"task_id"`
TaskType MaintenanceTaskType `json:"task_type"`
VolumeID uint32 `json:"volume_id,omitempty"`
Server string `json:"server,omitempty"`
Collection string `json:"collection,omitempty"`
Priority MaintenanceTaskPriority `json:"priority"`
Reason string `json:"reason"`
TypedParams *worker_pb.TaskParams `json:"typed_params,omitempty"`
ScheduleAt time.Time `json:"schedule_at"`
}
// VolumeHealthMetrics represents the health metrics for a volume
type VolumeHealthMetrics struct {
VolumeID uint32 `json:"volume_id"`
Server string `json:"server"`
ServerAddress string `json:"server_address"`
DiskType string `json:"disk_type"` // Disk type (e.g., "hdd", "ssd") or disk path (e.g., "/data1")
DiskId uint32 `json:"disk_id"` // ID of the disk in Store.Locations array
DataCenter string `json:"data_center"` // Data center of the server
Rack string `json:"rack"` // Rack of the server
Collection string `json:"collection"`
Size uint64 `json:"size"`
DeletedBytes uint64 `json:"deleted_bytes"`
GarbageRatio float64 `json:"garbage_ratio"`
LastModified time.Time `json:"last_modified"`
Age time.Duration `json:"age"`
ReplicaCount int `json:"replica_count"`
ExpectedReplicas int `json:"expected_replicas"`
IsReadOnly bool `json:"is_read_only"`
HasRemoteCopy bool `json:"has_remote_copy"`
IsECVolume bool `json:"is_ec_volume"`
FullnessRatio float64 `json:"fullness_ratio"`
}
// MaintenanceStats provides statistics about maintenance operations
type MaintenanceStats struct {
TotalTasks int `json:"total_tasks"`
TasksByStatus map[MaintenanceTaskStatus]int `json:"tasks_by_status"`
TasksByType map[MaintenanceTaskType]int `json:"tasks_by_type"`
ActiveWorkers int `json:"active_workers"`
CompletedToday int `json:"completed_today"`
FailedToday int `json:"failed_today"`
AverageTaskTime time.Duration `json:"average_task_time"`
LastScanTime time.Time `json:"last_scan_time"`
NextScanTime time.Time `json:"next_scan_time"`
}
// MaintenanceQueueData represents data for the queue visualization UI
type MaintenanceQueueData struct {
Tasks []*MaintenanceTask `json:"tasks"`
Workers []*MaintenanceWorker `json:"workers"`
Stats *QueueStats `json:"stats"`
LastUpdated time.Time `json:"last_updated"`
}
// QueueStats provides statistics for the queue UI
type QueueStats struct {
PendingTasks int `json:"pending_tasks"`
RunningTasks int `json:"running_tasks"`
CompletedToday int `json:"completed_today"`
FailedToday int `json:"failed_today"`
TotalTasks int `json:"total_tasks"`
}
// MaintenanceConfigData represents configuration data for the UI
type MaintenanceConfigData struct {
Config *MaintenanceConfig `json:"config"`
IsEnabled bool `json:"is_enabled"`
LastScanTime time.Time `json:"last_scan_time"`
NextScanTime time.Time `json:"next_scan_time"`
SystemStats *MaintenanceStats `json:"system_stats"`
MenuItems []*MaintenanceMenuItem `json:"menu_items"`
}
// MaintenanceMenuItem represents a menu item for task configuration
type MaintenanceMenuItem struct {
TaskType MaintenanceTaskType `json:"task_type"`
DisplayName string `json:"display_name"`
Description string `json:"description"`
Icon string `json:"icon"`
IsEnabled bool `json:"is_enabled"`
Path string `json:"path"`
}
// WorkerDetailsData represents detailed worker information
type WorkerDetailsData struct {
Worker *MaintenanceWorker `json:"worker"`
CurrentTasks []*MaintenanceTask `json:"current_tasks"`
RecentTasks []*MaintenanceTask `json:"recent_tasks"`
Performance *WorkerPerformance `json:"performance"`
LastUpdated time.Time `json:"last_updated"`
}
// WorkerPerformance tracks worker performance metrics
type WorkerPerformance struct {
TasksCompleted int `json:"tasks_completed"`
TasksFailed int `json:"tasks_failed"`
AverageTaskTime time.Duration `json:"average_task_time"`
Uptime time.Duration `json:"uptime"`
SuccessRate float64 `json:"success_rate"`
}
// TaskConfigData represents data for individual task configuration page
type TaskConfigData struct {
TaskType MaintenanceTaskType `json:"task_type"`
TaskName string `json:"task_name"`
TaskIcon string `json:"task_icon"`
Description string `json:"description"`
ConfigFormHTML template.HTML `json:"config_form_html"`
}
// ClusterReplicationTask represents a cluster replication task parameters
type ClusterReplicationTask struct {
SourcePath string `json:"source_path"`
TargetCluster string `json:"target_cluster"`
TargetPath string `json:"target_path"`
ReplicationMode string `json:"replication_mode"` // "sync", "async", "backup"
Priority int `json:"priority"`
Checksum string `json:"checksum,omitempty"`
FileSize int64 `json:"file_size"`
CreatedAt time.Time `json:"created_at"`
Metadata map[string]string `json:"metadata,omitempty"`
}
// Helper functions to extract configuration fields
// Note: Removed getVacuumConfigField, getErasureCodingConfigField, getBalanceConfigField, getReplicationConfigField
// These were orphaned after removing GetTaskConfig - use typed getters instead

View File

@@ -0,0 +1,311 @@
package maintenance
import (
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
// PendingOperationType represents the type of pending operation
type PendingOperationType string
const (
OpTypeVolumeMove PendingOperationType = "volume_move"
OpTypeVolumeBalance PendingOperationType = "volume_balance"
OpTypeErasureCoding PendingOperationType = "erasure_coding"
OpTypeVacuum PendingOperationType = "vacuum"
OpTypeReplication PendingOperationType = "replication"
)
// PendingOperation represents a pending volume/shard operation
type PendingOperation struct {
VolumeID uint32 `json:"volume_id"`
OperationType PendingOperationType `json:"operation_type"`
SourceNode string `json:"source_node"`
DestNode string `json:"dest_node,omitempty"` // Empty for non-movement operations
TaskID string `json:"task_id"`
StartTime time.Time `json:"start_time"`
EstimatedSize uint64 `json:"estimated_size"` // Bytes
Collection string `json:"collection"`
Status string `json:"status"` // "assigned", "in_progress", "completing"
}
// PendingOperations tracks all pending volume/shard operations
type PendingOperations struct {
// Operations by volume ID for conflict detection
byVolumeID map[uint32]*PendingOperation
// Operations by task ID for updates
byTaskID map[string]*PendingOperation
// Operations by node for capacity calculations
bySourceNode map[string][]*PendingOperation
byDestNode map[string][]*PendingOperation
mutex sync.RWMutex
}
// NewPendingOperations creates a new pending operations tracker
func NewPendingOperations() *PendingOperations {
return &PendingOperations{
byVolumeID: make(map[uint32]*PendingOperation),
byTaskID: make(map[string]*PendingOperation),
bySourceNode: make(map[string][]*PendingOperation),
byDestNode: make(map[string][]*PendingOperation),
}
}
// AddOperation adds a pending operation
func (po *PendingOperations) AddOperation(op *PendingOperation) {
po.mutex.Lock()
defer po.mutex.Unlock()
// Check for existing operation on this volume
if existing, exists := po.byVolumeID[op.VolumeID]; exists {
glog.V(1).Infof("Replacing existing pending operation on volume %d: %s -> %s",
op.VolumeID, existing.TaskID, op.TaskID)
po.removeOperationUnlocked(existing)
}
// Add new operation
po.byVolumeID[op.VolumeID] = op
po.byTaskID[op.TaskID] = op
// Add to node indexes
po.bySourceNode[op.SourceNode] = append(po.bySourceNode[op.SourceNode], op)
if op.DestNode != "" {
po.byDestNode[op.DestNode] = append(po.byDestNode[op.DestNode], op)
}
glog.V(2).Infof("Added pending operation: volume %d, type %s, task %s, %s -> %s",
op.VolumeID, op.OperationType, op.TaskID, op.SourceNode, op.DestNode)
}
// RemoveOperation removes a completed operation
func (po *PendingOperations) RemoveOperation(taskID string) {
po.mutex.Lock()
defer po.mutex.Unlock()
if op, exists := po.byTaskID[taskID]; exists {
po.removeOperationUnlocked(op)
glog.V(2).Infof("Removed completed operation: volume %d, task %s", op.VolumeID, taskID)
}
}
// removeOperationUnlocked removes an operation (must hold lock)
func (po *PendingOperations) removeOperationUnlocked(op *PendingOperation) {
delete(po.byVolumeID, op.VolumeID)
delete(po.byTaskID, op.TaskID)
// Remove from source node list
if ops, exists := po.bySourceNode[op.SourceNode]; exists {
for i, other := range ops {
if other.TaskID == op.TaskID {
po.bySourceNode[op.SourceNode] = append(ops[:i], ops[i+1:]...)
break
}
}
}
// Remove from dest node list
if op.DestNode != "" {
if ops, exists := po.byDestNode[op.DestNode]; exists {
for i, other := range ops {
if other.TaskID == op.TaskID {
po.byDestNode[op.DestNode] = append(ops[:i], ops[i+1:]...)
break
}
}
}
}
}
// HasPendingOperationOnVolume checks if a volume has a pending operation
func (po *PendingOperations) HasPendingOperationOnVolume(volumeID uint32) bool {
po.mutex.RLock()
defer po.mutex.RUnlock()
_, exists := po.byVolumeID[volumeID]
return exists
}
// GetPendingOperationOnVolume returns the pending operation on a volume
func (po *PendingOperations) GetPendingOperationOnVolume(volumeID uint32) *PendingOperation {
po.mutex.RLock()
defer po.mutex.RUnlock()
return po.byVolumeID[volumeID]
}
// WouldConflictWithPending checks if a new operation would conflict with pending ones
func (po *PendingOperations) WouldConflictWithPending(volumeID uint32, opType PendingOperationType) bool {
po.mutex.RLock()
defer po.mutex.RUnlock()
if existing, exists := po.byVolumeID[volumeID]; exists {
// Volume already has a pending operation
glog.V(3).Infof("Volume %d conflict: already has %s operation (task %s)",
volumeID, existing.OperationType, existing.TaskID)
return true
}
return false
}
// GetPendingCapacityImpactForNode calculates pending capacity changes for a node
func (po *PendingOperations) GetPendingCapacityImpactForNode(nodeID string) (incoming uint64, outgoing uint64) {
po.mutex.RLock()
defer po.mutex.RUnlock()
// Calculate outgoing capacity (volumes leaving this node)
if ops, exists := po.bySourceNode[nodeID]; exists {
for _, op := range ops {
// Only count movement operations
if op.DestNode != "" {
outgoing += op.EstimatedSize
}
}
}
// Calculate incoming capacity (volumes coming to this node)
if ops, exists := po.byDestNode[nodeID]; exists {
for _, op := range ops {
incoming += op.EstimatedSize
}
}
return incoming, outgoing
}
// FilterVolumeMetricsExcludingPending filters out volumes with pending operations
func (po *PendingOperations) FilterVolumeMetricsExcludingPending(metrics []*types.VolumeHealthMetrics) []*types.VolumeHealthMetrics {
po.mutex.RLock()
defer po.mutex.RUnlock()
var filtered []*types.VolumeHealthMetrics
excludedCount := 0
for _, metric := range metrics {
if _, hasPending := po.byVolumeID[metric.VolumeID]; !hasPending {
filtered = append(filtered, metric)
} else {
excludedCount++
glog.V(3).Infof("Excluding volume %d from scan due to pending operation", metric.VolumeID)
}
}
if excludedCount > 0 {
glog.V(1).Infof("Filtered out %d volumes with pending operations from %d total volumes",
excludedCount, len(metrics))
}
return filtered
}
// GetNodeCapacityProjection calculates projected capacity for a node
func (po *PendingOperations) GetNodeCapacityProjection(nodeID string, currentUsed uint64, totalCapacity uint64) NodeCapacityProjection {
incoming, outgoing := po.GetPendingCapacityImpactForNode(nodeID)
projectedUsed := currentUsed + incoming - outgoing
projectedFree := totalCapacity - projectedUsed
return NodeCapacityProjection{
NodeID: nodeID,
CurrentUsed: currentUsed,
TotalCapacity: totalCapacity,
PendingIncoming: incoming,
PendingOutgoing: outgoing,
ProjectedUsed: projectedUsed,
ProjectedFree: projectedFree,
}
}
// GetAllPendingOperations returns all pending operations
func (po *PendingOperations) GetAllPendingOperations() []*PendingOperation {
po.mutex.RLock()
defer po.mutex.RUnlock()
var operations []*PendingOperation
for _, op := range po.byVolumeID {
operations = append(operations, op)
}
return operations
}
// UpdateOperationStatus updates the status of a pending operation
func (po *PendingOperations) UpdateOperationStatus(taskID string, status string) {
po.mutex.Lock()
defer po.mutex.Unlock()
if op, exists := po.byTaskID[taskID]; exists {
op.Status = status
glog.V(3).Infof("Updated operation status: task %s, volume %d -> %s", taskID, op.VolumeID, status)
}
}
// CleanupStaleOperations removes operations that have been running too long
func (po *PendingOperations) CleanupStaleOperations(maxAge time.Duration) int {
po.mutex.Lock()
defer po.mutex.Unlock()
cutoff := time.Now().Add(-maxAge)
var staleOps []*PendingOperation
for _, op := range po.byVolumeID {
if op.StartTime.Before(cutoff) {
staleOps = append(staleOps, op)
}
}
for _, op := range staleOps {
po.removeOperationUnlocked(op)
glog.Warningf("Removed stale pending operation: volume %d, task %s, age %v",
op.VolumeID, op.TaskID, time.Since(op.StartTime))
}
return len(staleOps)
}
// NodeCapacityProjection represents projected capacity for a node
type NodeCapacityProjection struct {
NodeID string `json:"node_id"`
CurrentUsed uint64 `json:"current_used"`
TotalCapacity uint64 `json:"total_capacity"`
PendingIncoming uint64 `json:"pending_incoming"`
PendingOutgoing uint64 `json:"pending_outgoing"`
ProjectedUsed uint64 `json:"projected_used"`
ProjectedFree uint64 `json:"projected_free"`
}
// GetStats returns statistics about pending operations
func (po *PendingOperations) GetStats() PendingOperationsStats {
po.mutex.RLock()
defer po.mutex.RUnlock()
stats := PendingOperationsStats{
TotalOperations: len(po.byVolumeID),
ByType: make(map[PendingOperationType]int),
ByStatus: make(map[string]int),
}
var totalSize uint64
for _, op := range po.byVolumeID {
stats.ByType[op.OperationType]++
stats.ByStatus[op.Status]++
totalSize += op.EstimatedSize
}
stats.TotalEstimatedSize = totalSize
return stats
}
// PendingOperationsStats provides statistics about pending operations
type PendingOperationsStats struct {
TotalOperations int `json:"total_operations"`
ByType map[PendingOperationType]int `json:"by_type"`
ByStatus map[string]int `json:"by_status"`
TotalEstimatedSize uint64 `json:"total_estimated_size"`
}

View File

@@ -0,0 +1,250 @@
package maintenance
import (
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
func TestPendingOperations_ConflictDetection(t *testing.T) {
pendingOps := NewPendingOperations()
// Add a pending erasure coding operation on volume 123
op := &PendingOperation{
VolumeID: 123,
OperationType: OpTypeErasureCoding,
SourceNode: "node1",
TaskID: "task-001",
StartTime: time.Now(),
EstimatedSize: 1024 * 1024 * 1024, // 1GB
Collection: "test",
Status: "assigned",
}
pendingOps.AddOperation(op)
// Test conflict detection
if !pendingOps.HasPendingOperationOnVolume(123) {
t.Errorf("Expected volume 123 to have pending operation")
}
if !pendingOps.WouldConflictWithPending(123, OpTypeVacuum) {
t.Errorf("Expected conflict when trying to add vacuum operation on volume 123")
}
if pendingOps.HasPendingOperationOnVolume(124) {
t.Errorf("Expected volume 124 to have no pending operation")
}
if pendingOps.WouldConflictWithPending(124, OpTypeVacuum) {
t.Errorf("Expected no conflict for volume 124")
}
}
func TestPendingOperations_CapacityProjection(t *testing.T) {
pendingOps := NewPendingOperations()
// Add operation moving volume from node1 to node2
op1 := &PendingOperation{
VolumeID: 100,
OperationType: OpTypeVolumeMove,
SourceNode: "node1",
DestNode: "node2",
TaskID: "task-001",
StartTime: time.Now(),
EstimatedSize: 2 * 1024 * 1024 * 1024, // 2GB
Collection: "test",
Status: "in_progress",
}
// Add operation moving volume from node3 to node1
op2 := &PendingOperation{
VolumeID: 101,
OperationType: OpTypeVolumeMove,
SourceNode: "node3",
DestNode: "node1",
TaskID: "task-002",
StartTime: time.Now(),
EstimatedSize: 1 * 1024 * 1024 * 1024, // 1GB
Collection: "test",
Status: "assigned",
}
pendingOps.AddOperation(op1)
pendingOps.AddOperation(op2)
// Test capacity impact for node1
incoming, outgoing := pendingOps.GetPendingCapacityImpactForNode("node1")
expectedIncoming := uint64(1 * 1024 * 1024 * 1024) // 1GB incoming
expectedOutgoing := uint64(2 * 1024 * 1024 * 1024) // 2GB outgoing
if incoming != expectedIncoming {
t.Errorf("Expected incoming capacity %d, got %d", expectedIncoming, incoming)
}
if outgoing != expectedOutgoing {
t.Errorf("Expected outgoing capacity %d, got %d", expectedOutgoing, outgoing)
}
// Test projection for node1
currentUsed := uint64(10 * 1024 * 1024 * 1024) // 10GB current
totalCapacity := uint64(50 * 1024 * 1024 * 1024) // 50GB total
projection := pendingOps.GetNodeCapacityProjection("node1", currentUsed, totalCapacity)
expectedProjectedUsed := currentUsed + incoming - outgoing // 10 + 1 - 2 = 9GB
expectedProjectedFree := totalCapacity - expectedProjectedUsed // 50 - 9 = 41GB
if projection.ProjectedUsed != expectedProjectedUsed {
t.Errorf("Expected projected used %d, got %d", expectedProjectedUsed, projection.ProjectedUsed)
}
if projection.ProjectedFree != expectedProjectedFree {
t.Errorf("Expected projected free %d, got %d", expectedProjectedFree, projection.ProjectedFree)
}
}
func TestPendingOperations_VolumeFiltering(t *testing.T) {
pendingOps := NewPendingOperations()
// Create volume metrics
metrics := []*types.VolumeHealthMetrics{
{VolumeID: 100, Server: "node1", ServerAddress: "192.168.1.1:8080"},
{VolumeID: 101, Server: "node2", ServerAddress: "192.168.1.2:8080"},
{VolumeID: 102, Server: "node3", ServerAddress: "192.168.1.3:8080"},
{VolumeID: 103, Server: "node1", ServerAddress: "192.168.1.1:8080"},
}
// Add pending operations on volumes 101 and 103
op1 := &PendingOperation{
VolumeID: 101,
OperationType: OpTypeVacuum,
SourceNode: "node2",
TaskID: "task-001",
StartTime: time.Now(),
EstimatedSize: 1024 * 1024 * 1024,
Status: "in_progress",
}
op2 := &PendingOperation{
VolumeID: 103,
OperationType: OpTypeErasureCoding,
SourceNode: "node1",
TaskID: "task-002",
StartTime: time.Now(),
EstimatedSize: 2 * 1024 * 1024 * 1024,
Status: "assigned",
}
pendingOps.AddOperation(op1)
pendingOps.AddOperation(op2)
// Filter metrics
filtered := pendingOps.FilterVolumeMetricsExcludingPending(metrics)
// Should only have volumes 100 and 102 (101 and 103 are filtered out)
if len(filtered) != 2 {
t.Errorf("Expected 2 filtered metrics, got %d", len(filtered))
}
// Check that correct volumes remain
foundVolumes := make(map[uint32]bool)
for _, metric := range filtered {
foundVolumes[metric.VolumeID] = true
}
if !foundVolumes[100] || !foundVolumes[102] {
t.Errorf("Expected volumes 100 and 102 to remain after filtering")
}
if foundVolumes[101] || foundVolumes[103] {
t.Errorf("Expected volumes 101 and 103 to be filtered out")
}
}
func TestPendingOperations_OperationLifecycle(t *testing.T) {
pendingOps := NewPendingOperations()
// Add operation
op := &PendingOperation{
VolumeID: 200,
OperationType: OpTypeVolumeBalance,
SourceNode: "node1",
DestNode: "node2",
TaskID: "task-balance-001",
StartTime: time.Now(),
EstimatedSize: 1024 * 1024 * 1024,
Status: "assigned",
}
pendingOps.AddOperation(op)
// Check it exists
if !pendingOps.HasPendingOperationOnVolume(200) {
t.Errorf("Expected volume 200 to have pending operation")
}
// Update status
pendingOps.UpdateOperationStatus("task-balance-001", "in_progress")
retrievedOp := pendingOps.GetPendingOperationOnVolume(200)
if retrievedOp == nil {
t.Errorf("Expected to retrieve pending operation for volume 200")
} else if retrievedOp.Status != "in_progress" {
t.Errorf("Expected operation status to be 'in_progress', got '%s'", retrievedOp.Status)
}
// Complete operation
pendingOps.RemoveOperation("task-balance-001")
if pendingOps.HasPendingOperationOnVolume(200) {
t.Errorf("Expected volume 200 to have no pending operation after removal")
}
}
func TestPendingOperations_StaleCleanup(t *testing.T) {
pendingOps := NewPendingOperations()
// Add recent operation
recentOp := &PendingOperation{
VolumeID: 300,
OperationType: OpTypeVacuum,
SourceNode: "node1",
TaskID: "task-recent",
StartTime: time.Now(),
EstimatedSize: 1024 * 1024 * 1024,
Status: "in_progress",
}
// Add stale operation (24 hours ago)
staleOp := &PendingOperation{
VolumeID: 301,
OperationType: OpTypeErasureCoding,
SourceNode: "node2",
TaskID: "task-stale",
StartTime: time.Now().Add(-24 * time.Hour),
EstimatedSize: 2 * 1024 * 1024 * 1024,
Status: "in_progress",
}
pendingOps.AddOperation(recentOp)
pendingOps.AddOperation(staleOp)
// Clean up operations older than 1 hour
removedCount := pendingOps.CleanupStaleOperations(1 * time.Hour)
if removedCount != 1 {
t.Errorf("Expected to remove 1 stale operation, removed %d", removedCount)
}
// Recent operation should still exist
if !pendingOps.HasPendingOperationOnVolume(300) {
t.Errorf("Expected recent operation on volume 300 to still exist")
}
// Stale operation should be removed
if pendingOps.HasPendingOperationOnVolume(301) {
t.Errorf("Expected stale operation on volume 301 to be removed")
}
}