snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/admin/maintenance/maintenance_manager.go

@@ -0,0 +1,675 @@
+package maintenance
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/ec_balance"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
+	"github.com/seaweedfs/seaweedfs/weed/worker/types"
+)
+
+// BuildPolicyFromTaskConfigs loads each registered task's configuration and builds the
+// MaintenancePolicy the maintenance system runs on.
+//
+// Every entry is produced by the task's own ToTaskPolicy(), so a policy entry always carries
+// exactly what that task's config holds. Hand-copying the fields here instead made this a
+// second, silently diverging definition of every task's policy: it dropped the erasure
+// coding preferred tags and replica placement and the balance IO rate limit outright.
+//
+// Every task registered through base.RegisterTask needs an entry, because a task type the
+// policy does not list has no enabled flag and no concurrency limit of its own -
+// IsTaskEnabled reports false for a missing entry.
+//
+// configPersistence is duck-typed as interface{} because weed/admin/dash already imports this
+// package, so importing *dash.ConfigPersistence back here would create an import cycle. It must be
+// a value implementing the LoadXTaskPolicy() accessors the task loaders assert on; passing nil (or
+// anything else) makes every task fall back to its compiled-in defaults.
+func BuildPolicyFromTaskConfigs(configPersistence interface{}) *worker_pb.MaintenancePolicy {
+	policy := &worker_pb.MaintenancePolicy{
+		GlobalMaxConcurrent:          4,
+		DefaultRepeatIntervalSeconds: 6 * 3600,  // 6 hours in seconds
+		DefaultCheckIntervalSeconds:  12 * 3600, // 12 hours in seconds
+		TaskPolicies:                 make(map[string]*worker_pb.TaskPolicy),
+	}
+
+	if vacuumConfig := vacuum.LoadConfigFromPersistence(configPersistence); vacuumConfig != nil {
+		policy.TaskPolicies[string(types.TaskTypeVacuum)] = vacuumConfig.ToTaskPolicy()
+	}
+
+	if ecConfig := erasure_coding.LoadConfigFromPersistence(configPersistence); ecConfig != nil {
+		policy.TaskPolicies[string(types.TaskTypeErasureCoding)] = ecConfig.ToTaskPolicy()
+	}
+
+	if balanceConfig := balance.LoadConfigFromPersistence(configPersistence); balanceConfig != nil {
+		policy.TaskPolicies[string(types.TaskTypeBalance)] = balanceConfig.ToTaskPolicy()
+	}
+
+	if ecBalanceConfig := ec_balance.LoadConfigFromPersistence(configPersistence); ecBalanceConfig != nil {
+		policy.TaskPolicies[string(types.TaskTypeECBalance)] = ecBalanceConfig.ToTaskPolicy()
+	}
+
+	glog.V(1).Infof("Built maintenance policy from separate task configs - %d task policies loaded", len(policy.TaskPolicies))
+	return policy
+}
+
+// MaintenanceManager coordinates the maintenance system
+type MaintenanceManager struct {
+	config      *MaintenanceConfig
+	scanner     *MaintenanceScanner
+	queue       *MaintenanceQueue
+	adminClient AdminClient
+	running     bool
+	stopChan    chan struct{}
+	// Error handling and backoff
+	errorCount     int
+	lastError      error
+	lastErrorTime  time.Time
+	backoffDelay   time.Duration
+	mutex          sync.RWMutex
+	scanInProgress bool
+}
+
+// NewMaintenanceManager creates a new maintenance manager.
+//
+// configPersistence is the config store to read persisted task configs from when the policy has to
+// be built here. See BuildPolicyFromTaskConfigs for why it is duck-typed; pass nil when no config
+// store is available.
+func NewMaintenanceManager(adminClient AdminClient, config *MaintenanceConfig, configPersistence interface{}) *MaintenanceManager {
+	if config == nil {
+		config = DefaultMaintenanceConfig()
+	}
+
+	// Use the policy from the config (which is populated from separate task files in LoadMaintenanceConfig)
+	policy := config.Policy
+	if policy == nil {
+		// Fallback: build policy from separate task configuration files if not already populated
+		policy = BuildPolicyFromTaskConfigs(configPersistence)
+	}
+
+	queue := NewMaintenanceQueue(policy)
+	scanner := NewMaintenanceScanner(adminClient, policy, queue)
+
+	return &MaintenanceManager{
+		config:       config,
+		scanner:      scanner,
+		queue:        queue,
+		adminClient:  adminClient,
+		stopChan:     make(chan struct{}),
+		backoffDelay: time.Second, // Start with 1 second backoff
+	}
+}
+
+// Start begins the maintenance manager
+func (mm *MaintenanceManager) Start() error {
+	if !mm.config.GetEnabled() {
+		glog.V(1).Infof("Maintenance system is disabled")
+		return nil
+	}
+
+	// Validate configuration durations to prevent ticker panics
+	if err := mm.validateConfig(); err != nil {
+		return fmt.Errorf("invalid maintenance configuration: %w", err)
+	}
+
+	mm.mutex.Lock()
+	mm.running = true
+	mm.mutex.Unlock()
+
+	// Start background processes
+	go mm.scanLoop()
+	go mm.cleanupLoop()
+	go mm.topologyStatusLoop() // Periodic diagnostic logging
+
+	glog.Infof("Maintenance manager started with scan interval %ds", mm.config.ScanIntervalSeconds)
+	return nil
+}
+
+// validateConfig validates the maintenance configuration durations
+func (mm *MaintenanceManager) validateConfig() error {
+	if mm.config.ScanIntervalSeconds <= 0 {
+		glog.Warningf("Invalid scan interval %ds, using default 30m", mm.config.ScanIntervalSeconds)
+		mm.config.ScanIntervalSeconds = 30 * 60 // 30 minutes in seconds
+	}
+
+	if mm.config.CleanupIntervalSeconds <= 0 {
+		glog.Warningf("Invalid cleanup interval %ds, using default 24h", mm.config.CleanupIntervalSeconds)
+		mm.config.CleanupIntervalSeconds = 24 * 60 * 60 // 24 hours in seconds
+	}
+
+	if mm.config.WorkerTimeoutSeconds <= 0 {
+		glog.Warningf("Invalid worker timeout %ds, using default 5m", mm.config.WorkerTimeoutSeconds)
+		mm.config.WorkerTimeoutSeconds = 5 * 60 // 5 minutes in seconds
+	}
+
+	if mm.config.TaskTimeoutSeconds <= 0 {
+		glog.Warningf("Invalid task timeout %ds, using default 2h", mm.config.TaskTimeoutSeconds)
+		mm.config.TaskTimeoutSeconds = 2 * 60 * 60 // 2 hours in seconds
+	}
+
+	if mm.config.RetryDelaySeconds <= 0 {
+		glog.Warningf("Invalid retry delay %ds, using default 15m", mm.config.RetryDelaySeconds)
+		mm.config.RetryDelaySeconds = 15 * 60 // 15 minutes in seconds
+	}
+
+	if mm.config.TaskRetentionSeconds <= 0 {
+		glog.Warningf("Invalid task retention %ds, using default 168h", mm.config.TaskRetentionSeconds)
+		mm.config.TaskRetentionSeconds = 7 * 24 * 60 * 60 // 7 days in seconds
+	}
+
+	return nil
+}
+
+// IsRunning returns whether the maintenance manager is currently running.
+// running is guarded by mm.mutex because the background loops read it on every
+// iteration while Start and Stop are called from the admin server's goroutines.
+func (mm *MaintenanceManager) IsRunning() bool {
+	mm.mutex.RLock()
+	defer mm.mutex.RUnlock()
+	return mm.running
+}
+
+// Stop terminates the maintenance manager. It is a no-op when the manager is not
+// running, so a second call cannot close the already closed stop channel.
+func (mm *MaintenanceManager) Stop() {
+	mm.mutex.Lock()
+	if !mm.running {
+		mm.mutex.Unlock()
+		return
+	}
+	mm.running = false
+	close(mm.stopChan)
+	mm.mutex.Unlock()
+
+	glog.Infof("Maintenance manager stopped")
+}
+
+// scanLoop periodically scans for maintenance tasks with adaptive timing
+func (mm *MaintenanceManager) scanLoop() {
+	scanInterval := time.Duration(mm.config.ScanIntervalSeconds) * time.Second
+
+	// activeInterval is the interval the ticker is actually running at right now. It has
+	// to be tracked separately from the configured scanInterval, because the error backoff
+	// replaces the ticker with a much shorter one. Comparing the target against the
+	// configured interval instead never restores the normal cadence: once the errors stop,
+	// getScanInterval returns scanInterval again, the comparison comes out false, and the
+	// ticker is left at the backoff delay. A single transient scan failure therefore pinned
+	// the scanner to one scan per second forever.
+	activeInterval := scanInterval
+	ticker := time.NewTicker(activeInterval)
+	// Wrapped in a closure so the replacement ticker is stopped, not the one that happened
+	// to be current when the defer was registered.
+	defer func() { ticker.Stop() }()
+
+	for mm.IsRunning() {
+		select {
+		case <-mm.stopChan:
+			return
+		case <-ticker.C:
+			glog.V(1).Infof("Performing maintenance scan every %v", activeInterval)
+
+			// Use the same synchronization as TriggerScan to prevent concurrent scans
+			if err := mm.triggerScanInternal(false); err != nil {
+				glog.V(1).Infof("Scheduled scan skipped: %v", err)
+			}
+
+			// Adjust ticker interval based on error state (read error state safely)
+			currentInterval := mm.getScanInterval(scanInterval)
+
+			// Reset ticker whenever the target differs from what the ticker is running at,
+			// which covers both entering the backoff and returning to the normal cadence.
+			if currentInterval != activeInterval {
+				ticker.Stop()
+				ticker = time.NewTicker(currentInterval)
+				glog.V(1).Infof("Maintenance scan cadence changed from %v to %v", activeInterval, currentInterval)
+				activeInterval = currentInterval
+			}
+		}
+	}
+}
+
+// getScanInterval safely reads the current scan interval with error backoff
+func (mm *MaintenanceManager) getScanInterval(baseInterval time.Duration) time.Duration {
+	mm.mutex.RLock()
+	defer mm.mutex.RUnlock()
+
+	if mm.errorCount > 0 {
+		// Use backoff delay when there are errors
+		currentInterval := mm.backoffDelay
+		if currentInterval > baseInterval {
+			// Don't make it longer than the configured interval * 10
+			maxInterval := baseInterval * 10
+			if currentInterval > maxInterval {
+				currentInterval = maxInterval
+			}
+		}
+		return currentInterval
+	}
+	return baseInterval
+}
+
+// cleanupLoop periodically cleans up old tasks and stale workers
+func (mm *MaintenanceManager) cleanupLoop() {
+	cleanupInterval := time.Duration(mm.config.CleanupIntervalSeconds) * time.Second
+	ticker := time.NewTicker(cleanupInterval)
+	defer ticker.Stop()
+
+	for mm.IsRunning() {
+		select {
+		case <-mm.stopChan:
+			return
+		case <-ticker.C:
+			mm.performCleanup()
+		}
+	}
+}
+
+// topologyStatusLoop periodically logs topology status for diagnostics
+func (mm *MaintenanceManager) topologyStatusLoop() {
+	// Log topology status every 5 minutes for diagnostic purposes
+	statusInterval := 5 * time.Minute
+	ticker := time.NewTicker(statusInterval)
+	defer ticker.Stop()
+
+	for mm.IsRunning() {
+		select {
+		case <-mm.stopChan:
+			return
+		case <-ticker.C:
+			mm.logTopologyStatus()
+		}
+	}
+}
+
+// logTopologyStatus logs current topology and worker status for diagnostics
+func (mm *MaintenanceManager) logTopologyStatus() {
+	if mm.scanner == nil || mm.scanner.integration == nil {
+		glog.V(2).Infof("Topology status: scanner/integration not available")
+		return
+	}
+
+	activeTopology := mm.scanner.integration.GetActiveTopology()
+	if activeTopology == nil {
+		glog.V(1).Infof("Topology status: ActiveTopology is nil")
+		return
+	}
+
+	diskCount := activeTopology.GetDiskCount()
+	nodeCount := len(activeTopology.GetAllNodes())
+
+	// Get queue stats
+	stats := mm.queue.GetStats()
+	workerCount := len(mm.queue.GetWorkers())
+
+	mm.mutex.RLock()
+	errorCount := mm.errorCount
+	mm.mutex.RUnlock()
+
+	glog.V(1).Infof("Topology status: %d nodes, %d disks, %d workers, %d pending tasks, %d running tasks, errors: %d",
+		nodeCount, diskCount, workerCount,
+		stats.TasksByStatus[TaskStatusPending],
+		stats.TasksByStatus[TaskStatusInProgress]+stats.TasksByStatus[TaskStatusAssigned],
+		errorCount)
+}
+
+// performScan executes a maintenance scan with error handling and backoff
+func (mm *MaintenanceManager) performScan() {
+	defer func() {
+		// Always reset scan in progress flag when done
+		mm.mutex.Lock()
+		mm.scanInProgress = false
+		mm.mutex.Unlock()
+	}()
+
+	glog.Infof("Starting maintenance scan...")
+
+	results, err := mm.scanner.ScanForMaintenanceTasks()
+	stats_collect.AdminMaintenanceLastScanTimestampSeconds.SetToCurrentTime()
+	if err != nil {
+		// Handle scan error
+		mm.mutex.Lock()
+		mm.handleScanError(err)
+		mm.mutex.Unlock()
+		glog.Warningf("Maintenance scan failed: %v", err)
+		return
+	}
+
+	// Scan succeeded - update state and process results
+	mm.handleScanSuccess(results)
+}
+
+// handleScanSuccess processes successful scan results with proper lock management
+func (mm *MaintenanceManager) handleScanSuccess(results []*TaskDetectionResult) {
+	// Update manager state first
+	mm.mutex.Lock()
+	mm.resetErrorTracking()
+	taskCount := len(results)
+	mm.mutex.Unlock()
+
+	if taskCount > 0 {
+		// Count tasks by type for logging (outside of lock)
+		taskCounts := make(map[MaintenanceTaskType]int)
+		for _, result := range results {
+			taskCounts[result.TaskType]++
+		}
+
+		// Add tasks to queue (no manager lock held)
+		mm.queue.AddTasksFromResults(results)
+
+		// Log detailed scan results
+		glog.Infof("Maintenance scan completed: found %d tasks", taskCount)
+		for taskType, count := range taskCounts {
+			glog.Infof("  - %s: %d tasks", taskType, count)
+		}
+	} else {
+		glog.Infof("Maintenance scan completed: no maintenance tasks needed")
+	}
+}
+
+// handleScanError handles scan errors with exponential backoff and reduced logging
+func (mm *MaintenanceManager) handleScanError(err error) {
+	now := time.Now()
+	mm.errorCount++
+	mm.lastError = err
+	mm.lastErrorTime = now
+
+	// Use exponential backoff with jitter
+	if mm.errorCount > 1 {
+		mm.backoffDelay = mm.backoffDelay * 2
+		if mm.backoffDelay > 5*time.Minute {
+			mm.backoffDelay = 5 * time.Minute // Cap at 5 minutes
+		}
+	}
+
+	// Reduce log frequency based on error count and time
+	shouldLog := false
+	if mm.errorCount <= 3 {
+		// Log first 3 errors immediately
+		shouldLog = true
+	} else if mm.errorCount <= 10 && mm.errorCount%3 == 0 {
+		// Log every 3rd error for errors 4-10
+		shouldLog = true
+	} else if mm.errorCount%10 == 0 {
+		// Log every 10th error after that
+		shouldLog = true
+	}
+
+	if shouldLog {
+		// Check if it's a connection error to provide better messaging
+		if isConnectionError(err) {
+			if mm.errorCount == 1 {
+				glog.Errorf("Maintenance scan failed: %v (will retry with backoff)", err)
+			} else {
+				glog.Errorf("Maintenance scan still failing after %d attempts: %v (backoff: %v)",
+					mm.errorCount, err, mm.backoffDelay)
+			}
+		} else {
+			glog.Errorf("Maintenance scan failed: %v", err)
+		}
+	} else {
+		// Use debug level for suppressed errors
+		glog.V(3).Infof("Maintenance scan failed (error #%d, suppressed): %v", mm.errorCount, err)
+	}
+}
+
+// resetErrorTracking resets error tracking when scan succeeds
+func (mm *MaintenanceManager) resetErrorTracking() {
+	if mm.errorCount > 0 {
+		glog.V(1).Infof("Maintenance scan recovered after %d failed attempts", mm.errorCount)
+		mm.errorCount = 0
+		mm.lastError = nil
+		mm.backoffDelay = time.Second // Reset to initial delay
+	}
+}
+
+// isConnectionError checks if the error is a connection-related error
+func isConnectionError(err error) bool {
+	if err == nil {
+		return false
+	}
+	errStr := err.Error()
+	return strings.Contains(errStr, "connection refused") ||
+		strings.Contains(errStr, "connection error") ||
+		strings.Contains(errStr, "dial tcp") ||
+		strings.Contains(errStr, "connection timeout") ||
+		strings.Contains(errStr, "no route to host") ||
+		strings.Contains(errStr, "network unreachable")
+}
+
+// performCleanup cleans up old tasks and stale workers
+func (mm *MaintenanceManager) performCleanup() {
+	glog.V(2).Infof("Starting maintenance cleanup")
+
+	taskRetention := time.Duration(mm.config.TaskRetentionSeconds) * time.Second
+	workerTimeout := time.Duration(mm.config.WorkerTimeoutSeconds) * time.Second
+
+	removedTasks := mm.queue.CleanupOldTasks(taskRetention)
+	removedWorkers := mm.queue.RemoveStaleWorkers(workerTimeout)
+
+	// Clean up stale pending operations (operations running for more than 4 hours)
+	staleOperationTimeout := 4 * time.Hour
+	removedOperations := 0
+	if mm.scanner != nil && mm.scanner.integration != nil {
+		pendingOps := mm.scanner.integration.GetPendingOperations()
+		if pendingOps != nil {
+			removedOperations = pendingOps.CleanupStaleOperations(staleOperationTimeout)
+		}
+	}
+
+	if removedTasks > 0 || removedWorkers > 0 || removedOperations > 0 {
+		glog.V(1).Infof("Cleanup completed: removed %d old tasks, %d stale workers, and %d stale operations",
+			removedTasks, removedWorkers, removedOperations)
+	}
+}
+
+// GetQueue returns the maintenance queue
+func (mm *MaintenanceManager) GetQueue() *MaintenanceQueue {
+	return mm.queue
+}
+
+// GetConfig returns the maintenance configuration
+func (mm *MaintenanceManager) GetConfig() *MaintenanceConfig {
+	return mm.config
+}
+
+// GetStats returns maintenance statistics
+func (mm *MaintenanceManager) GetStats() *MaintenanceStats {
+	stats := mm.queue.GetStats()
+
+	mm.mutex.RLock()
+	defer mm.mutex.RUnlock()
+
+	stats.LastScanTime = time.Now() // Would need to track this properly
+
+	// Calculate next scan time based on current error state
+	scanInterval := time.Duration(mm.config.ScanIntervalSeconds) * time.Second
+	nextScanInterval := scanInterval
+	if mm.errorCount > 0 {
+		nextScanInterval = mm.backoffDelay
+		maxInterval := scanInterval * 10
+		if nextScanInterval > maxInterval {
+			nextScanInterval = maxInterval
+		}
+	}
+	stats.NextScanTime = time.Now().Add(nextScanInterval)
+
+	return stats
+}
+
+// ReloadTaskConfigurations reloads task configurations from the current policy
+func (mm *MaintenanceManager) ReloadTaskConfigurations() error {
+	mm.mutex.Lock()
+	defer mm.mutex.Unlock()
+
+	// Trigger configuration reload in the integration layer
+	if mm.scanner != nil && mm.scanner.integration != nil {
+		mm.scanner.integration.ConfigureTasksFromPolicy()
+		glog.V(1).Infof("Task configurations reloaded from policy")
+		return nil
+	}
+
+	return fmt.Errorf("integration not available for configuration reload")
+}
+
+// GetErrorState returns the current error state for monitoring
+func (mm *MaintenanceManager) GetErrorState() (errorCount int, lastError error, backoffDelay time.Duration) {
+	mm.mutex.RLock()
+	defer mm.mutex.RUnlock()
+	return mm.errorCount, mm.lastError, mm.backoffDelay
+}
+
+// GetTasks returns tasks with filtering
+func (mm *MaintenanceManager) GetTasks(status MaintenanceTaskStatus, taskType MaintenanceTaskType, limit int) []*MaintenanceTask {
+	return mm.queue.GetTasks(status, taskType, limit)
+}
+
+// GetWorkers returns all registered workers
+func (mm *MaintenanceManager) GetWorkers() []*MaintenanceWorker {
+	return mm.queue.GetWorkers()
+}
+
+// GetWorkerSlots returns used/max task slots per worker ID.
+func (mm *MaintenanceManager) GetWorkerSlots() map[string]WorkerSlots {
+	return mm.queue.GetWorkerSlots()
+}
+
+// TriggerScan manually triggers a maintenance scan
+func (mm *MaintenanceManager) TriggerScan() error {
+	return mm.triggerScanInternal(true)
+}
+
+// triggerScanInternal handles both manual and automatic scan triggers
+func (mm *MaintenanceManager) triggerScanInternal(isManual bool) error {
+	// running and scanInProgress are checked under one lock so a Stop that lands
+	// between the two checks cannot leave a scan running after shutdown.
+	mm.mutex.Lock()
+	if !mm.running {
+		mm.mutex.Unlock()
+		return fmt.Errorf("maintenance manager is not running")
+	}
+
+	// Prevent multiple concurrent scans
+	if mm.scanInProgress {
+		mm.mutex.Unlock()
+		if isManual {
+			glog.V(1).Infof("Manual scan already in progress, ignoring trigger request")
+		} else {
+			glog.V(2).Infof("Automatic scan already in progress, ignoring scheduled scan")
+		}
+		return fmt.Errorf("scan already in progress")
+	}
+	mm.scanInProgress = true
+	mm.mutex.Unlock()
+
+	go mm.performScan()
+	return nil
+}
+
+// UpdateConfig updates the maintenance configuration
+func (mm *MaintenanceManager) UpdateConfig(config *MaintenanceConfig) error {
+	if config == nil {
+		return fmt.Errorf("config cannot be nil")
+	}
+
+	mm.config = config
+	mm.queue.policy = config.Policy
+	mm.scanner.policy = config.Policy
+
+	// Propagate global policy changes to individual task configuration files
+	if config.Policy != nil {
+		mm.saveTaskConfigsFromPolicy(config.Policy)
+	}
+
+	// The integration holds its own reference to the policy and is what pushes the
+	// enabled flag and the concurrency limit into the registered detectors and
+	// schedulers. Without this the queue and the scanner saw the new policy while the
+	// detectors kept scanning under the old one until the next restart.
+	if mm.scanner != nil && mm.scanner.integration != nil {
+		mm.scanner.integration.SetPolicy(config.Policy)
+	}
+
+	glog.V(1).Infof("Maintenance configuration updated")
+	return nil
+}
+
+// saveTaskConfigsFromPolicy propagates global policy settings to separate task configuration files
+func (mm *MaintenanceManager) saveTaskConfigsFromPolicy(policy *worker_pb.MaintenancePolicy) {
+	if mm.queue.persistence == nil || policy == nil {
+		return
+	}
+
+	glog.V(1).Infof("Propagating maintenance policy changes to separate task configs")
+	for taskType, taskPolicy := range policy.TaskPolicies {
+		if err := mm.queue.persistence.SaveTaskPolicy(taskType, taskPolicy); err != nil {
+			glog.Errorf("Failed to save task policy for %s: %v", taskType, err)
+		}
+	}
+}
+
+// CancelTask cancels a pending task
+func (mm *MaintenanceManager) CancelTask(taskID string) error {
+	mm.queue.mutex.Lock()
+	defer mm.queue.mutex.Unlock()
+
+	task, exists := mm.queue.tasks[taskID]
+	if !exists {
+		return fmt.Errorf("task %s not found", taskID)
+	}
+
+	if task.Status == TaskStatusPending {
+		task.Status = TaskStatusCancelled
+		task.CompletedAt = &[]time.Time{time.Now()}[0]
+
+		// Remove from pending tasks
+		for i, pendingTask := range mm.queue.pendingTasks {
+			if pendingTask.ID == taskID {
+				mm.queue.pendingTasks = append(mm.queue.pendingTasks[:i], mm.queue.pendingTasks[i+1:]...)
+				break
+			}
+		}
+
+		// Notify ActiveTopology to release capacity
+		if mm.scanner != nil && mm.scanner.integration != nil {
+			if at := mm.scanner.integration.GetActiveTopology(); at != nil {
+				_ = at.CompleteTask(taskID)
+			}
+		}
+
+		glog.V(2).Infof("Cancelled task %s", taskID)
+		return nil
+	}
+
+	return fmt.Errorf("task %s cannot be cancelled (status: %s)", taskID, task.Status)
+}
+
+// RegisterWorker registers a new worker
+func (mm *MaintenanceManager) RegisterWorker(worker *MaintenanceWorker) {
+	mm.queue.RegisterWorker(worker)
+}
+
+// GetNextTask returns the next task for a worker
+func (mm *MaintenanceManager) GetNextTask(workerID string, capabilities []MaintenanceTaskType) *MaintenanceTask {
+	return mm.queue.GetNextTask(workerID, capabilities)
+}
+
+// CompleteTask marks a task as completed
+func (mm *MaintenanceManager) CompleteTask(taskID string, error string) {
+	mm.queue.CompleteTask(taskID, error)
+}
+
+// UpdateTaskProgress updates task progress
+func (mm *MaintenanceManager) UpdateTaskProgress(taskID string, progress float64) {
+	mm.queue.UpdateTaskProgress(taskID, progress)
+}
+
+// UpdateWorkerHeartbeat updates worker heartbeat
+func (mm *MaintenanceManager) UpdateWorkerHeartbeat(workerID string) {
+	mm.queue.UpdateWorkerHeartbeat(workerID)
+}