snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
190
weed/admin/maintenance/config_schema.go
Normal file
190
weed/admin/maintenance/config_schema.go
Normal file
@@ -0,0 +1,190 @@
|
||||
package maintenance
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/config"
|
||||
)
|
||||
|
||||
// Type aliases for backward compatibility
|
||||
type ConfigFieldType = config.FieldType
|
||||
type ConfigFieldUnit = config.FieldUnit
|
||||
type ConfigField = config.Field
|
||||
|
||||
// Constant aliases for backward compatibility
|
||||
const (
|
||||
FieldTypeBool = config.FieldTypeBool
|
||||
FieldTypeInt = config.FieldTypeInt
|
||||
FieldTypeDuration = config.FieldTypeDuration
|
||||
FieldTypeInterval = config.FieldTypeInterval
|
||||
FieldTypeString = config.FieldTypeString
|
||||
FieldTypeFloat = config.FieldTypeFloat
|
||||
)
|
||||
|
||||
const (
|
||||
UnitSeconds = config.UnitSeconds
|
||||
UnitMinutes = config.UnitMinutes
|
||||
UnitHours = config.UnitHours
|
||||
UnitDays = config.UnitDays
|
||||
UnitCount = config.UnitCount
|
||||
UnitNone = config.UnitNone
|
||||
)
|
||||
|
||||
// Function aliases for backward compatibility
|
||||
var (
|
||||
SecondsToIntervalValueUnit = config.SecondsToIntervalValueUnit
|
||||
IntervalValueUnitToSeconds = config.IntervalValueUnitToSeconds
|
||||
)
|
||||
|
||||
// MaintenanceConfigSchema defines the schema for maintenance configuration
|
||||
type MaintenanceConfigSchema struct {
|
||||
config.Schema // Embed common schema functionality
|
||||
}
|
||||
|
||||
// GetMaintenanceConfigSchema returns the schema for maintenance configuration
|
||||
func GetMaintenanceConfigSchema() *MaintenanceConfigSchema {
|
||||
return &MaintenanceConfigSchema{
|
||||
Schema: config.Schema{
|
||||
Fields: []*config.Field{
|
||||
{
|
||||
Name: "enabled",
|
||||
JSONName: "enabled",
|
||||
Type: config.FieldTypeBool,
|
||||
DefaultValue: true,
|
||||
Required: false,
|
||||
DisplayName: "Enable Maintenance System",
|
||||
Description: "When enabled, the system will automatically scan for and execute maintenance tasks",
|
||||
HelpText: "Toggle this to enable or disable the entire maintenance system",
|
||||
InputType: "checkbox",
|
||||
CSSClasses: "form-check-input",
|
||||
},
|
||||
{
|
||||
Name: "scan_interval_seconds",
|
||||
JSONName: "scan_interval_seconds",
|
||||
Type: config.FieldTypeInterval,
|
||||
DefaultValue: 30 * 60, // 30 minutes in seconds
|
||||
MinValue: 1 * 60, // 1 minute
|
||||
MaxValue: 24 * 60 * 60, // 24 hours
|
||||
Required: true,
|
||||
DisplayName: "Scan Interval",
|
||||
Description: "How often to scan for maintenance tasks",
|
||||
HelpText: "The system will check for new maintenance tasks at this interval",
|
||||
Placeholder: "30",
|
||||
Unit: config.UnitMinutes,
|
||||
InputType: "interval",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "worker_timeout_seconds",
|
||||
JSONName: "worker_timeout_seconds",
|
||||
Type: config.FieldTypeInterval,
|
||||
DefaultValue: 5 * 60, // 5 minutes
|
||||
MinValue: 1 * 60, // 1 minute
|
||||
MaxValue: 60 * 60, // 1 hour
|
||||
Required: true,
|
||||
DisplayName: "Worker Timeout",
|
||||
Description: "How long to wait for worker heartbeat before considering it inactive",
|
||||
HelpText: "Workers that don't send heartbeats within this time are considered offline",
|
||||
Placeholder: "5",
|
||||
Unit: config.UnitMinutes,
|
||||
InputType: "interval",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "task_timeout_seconds",
|
||||
JSONName: "task_timeout_seconds",
|
||||
Type: config.FieldTypeInterval,
|
||||
DefaultValue: 2 * 60 * 60, // 2 hours
|
||||
MinValue: 10 * 60, // 10 minutes
|
||||
MaxValue: 24 * 60 * 60, // 24 hours
|
||||
Required: true,
|
||||
DisplayName: "Task Timeout",
|
||||
Description: "Maximum time allowed for a task to complete",
|
||||
HelpText: "Tasks that exceed this duration will be marked as failed",
|
||||
Placeholder: "2",
|
||||
Unit: config.UnitHours,
|
||||
InputType: "interval",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "retry_delay_seconds",
|
||||
JSONName: "retry_delay_seconds",
|
||||
Type: config.FieldTypeInterval,
|
||||
DefaultValue: 15 * 60, // 15 minutes
|
||||
MinValue: 1 * 60, // 1 minute
|
||||
MaxValue: 24 * 60 * 60, // 24 hours
|
||||
Required: true,
|
||||
DisplayName: "Retry Delay",
|
||||
Description: "How long to wait before retrying a failed task",
|
||||
HelpText: "Failed tasks will be retried after this delay",
|
||||
Placeholder: "15",
|
||||
Unit: config.UnitMinutes,
|
||||
InputType: "interval",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "max_retries",
|
||||
JSONName: "max_retries",
|
||||
Type: config.FieldTypeInt,
|
||||
DefaultValue: 3,
|
||||
MinValue: 0,
|
||||
MaxValue: 10,
|
||||
Required: true,
|
||||
DisplayName: "Max Retries",
|
||||
Description: "Maximum number of times to retry a failed task",
|
||||
HelpText: "Tasks that fail more than this many times will be marked as permanently failed",
|
||||
Placeholder: "3",
|
||||
Unit: config.UnitCount,
|
||||
InputType: "number",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "cleanup_interval_seconds",
|
||||
JSONName: "cleanup_interval_seconds",
|
||||
Type: config.FieldTypeInterval,
|
||||
DefaultValue: 24 * 60 * 60, // 24 hours
|
||||
MinValue: 1 * 60 * 60, // 1 hour
|
||||
MaxValue: 7 * 24 * 60 * 60, // 7 days
|
||||
Required: true,
|
||||
DisplayName: "Cleanup Interval",
|
||||
Description: "How often to run maintenance cleanup operations",
|
||||
HelpText: "Removes old task records and temporary files at this interval",
|
||||
Placeholder: "24",
|
||||
Unit: config.UnitHours,
|
||||
InputType: "interval",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "task_retention_seconds",
|
||||
JSONName: "task_retention_seconds",
|
||||
Type: config.FieldTypeInterval,
|
||||
DefaultValue: 7 * 24 * 60 * 60, // 7 days
|
||||
MinValue: 1 * 24 * 60 * 60, // 1 day
|
||||
MaxValue: 30 * 24 * 60 * 60, // 30 days
|
||||
Required: true,
|
||||
DisplayName: "Task Retention",
|
||||
Description: "How long to keep completed task records",
|
||||
HelpText: "Task history older than this duration will be automatically deleted",
|
||||
Placeholder: "7",
|
||||
Unit: config.UnitDays,
|
||||
InputType: "interval",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
{
|
||||
Name: "global_max_concurrent",
|
||||
JSONName: "global_max_concurrent",
|
||||
Type: config.FieldTypeInt,
|
||||
DefaultValue: 10,
|
||||
MinValue: 1,
|
||||
MaxValue: 100,
|
||||
Required: true,
|
||||
DisplayName: "Global Max Concurrent Tasks",
|
||||
Description: "Maximum number of maintenance tasks that can run simultaneously across all workers",
|
||||
HelpText: "Limits the total number of maintenance operations to control system load",
|
||||
Placeholder: "10",
|
||||
Unit: config.UnitCount,
|
||||
InputType: "number",
|
||||
CSSClasses: "form-control",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
22
weed/admin/maintenance/maintenance_config_proto.go
Normal file
22
weed/admin/maintenance/maintenance_config_proto.go
Normal file
@@ -0,0 +1,22 @@
|
||||
package maintenance
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
// DefaultMaintenanceConfigProto returns default configuration as protobuf
|
||||
func DefaultMaintenanceConfigProto() *worker_pb.MaintenanceConfig {
|
||||
return &worker_pb.MaintenanceConfig{
|
||||
Enabled: proto.Bool(true),
|
||||
ScanIntervalSeconds: 30 * 60, // 30 minutes
|
||||
WorkerTimeoutSeconds: 5 * 60, // 5 minutes
|
||||
TaskTimeoutSeconds: 2 * 60 * 60, // 2 hours
|
||||
RetryDelaySeconds: 15 * 60, // 15 minutes
|
||||
MaxRetries: 3,
|
||||
CleanupIntervalSeconds: 24 * 60 * 60, // 24 hours
|
||||
TaskRetentionSeconds: 7 * 24 * 60 * 60, // 7 days
|
||||
// Policy field will be populated dynamically from separate task configuration files
|
||||
Policy: nil,
|
||||
}
|
||||
}
|
||||
620
weed/admin/maintenance/maintenance_integration.go
Normal file
620
weed/admin/maintenance/maintenance_integration.go
Normal file
@@ -0,0 +1,620 @@
|
||||
package maintenance
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// MaintenanceIntegration bridges the task system with existing maintenance
|
||||
type MaintenanceIntegration struct {
|
||||
taskRegistry *types.TaskRegistry
|
||||
uiRegistry *types.UIRegistry
|
||||
|
||||
// Bridge to existing system
|
||||
maintenanceQueue *MaintenanceQueue
|
||||
maintenancePolicy *MaintenancePolicy
|
||||
|
||||
// Pending operations tracker
|
||||
pendingOperations *PendingOperations
|
||||
|
||||
// Active topology for task detection and target selection
|
||||
activeTopology *topology.ActiveTopology
|
||||
|
||||
// Master's default replication, refreshed by the scanner each cycle and
|
||||
// passed to detectors as the replica-placement fallback (matches the shell).
|
||||
defaultReplicaPlacement string
|
||||
|
||||
// Type conversion maps
|
||||
taskTypeMap map[types.TaskType]MaintenanceTaskType
|
||||
revTaskTypeMap map[MaintenanceTaskType]types.TaskType
|
||||
priorityMap map[types.TaskPriority]MaintenanceTaskPriority
|
||||
revPriorityMap map[MaintenanceTaskPriority]types.TaskPriority
|
||||
}
|
||||
|
||||
// NewMaintenanceIntegration creates the integration bridge
|
||||
func NewMaintenanceIntegration(queue *MaintenanceQueue, policy *MaintenancePolicy) *MaintenanceIntegration {
|
||||
integration := &MaintenanceIntegration{
|
||||
taskRegistry: tasks.GetGlobalTypesRegistry(), // Use global types registry with auto-registered tasks
|
||||
uiRegistry: tasks.GetGlobalUIRegistry(), // Use global UI registry with auto-registered UI providers
|
||||
maintenanceQueue: queue,
|
||||
maintenancePolicy: policy,
|
||||
pendingOperations: NewPendingOperations(),
|
||||
}
|
||||
|
||||
// Initialize active topology with 10 second recent task window
|
||||
integration.activeTopology = topology.NewActiveTopology(10)
|
||||
|
||||
// Initialize type conversion maps
|
||||
integration.initializeTypeMaps()
|
||||
|
||||
// Register all tasks
|
||||
integration.registerAllTasks()
|
||||
|
||||
return integration
|
||||
}
|
||||
|
||||
// initializeTypeMaps creates the type conversion maps for dynamic conversion
|
||||
func (s *MaintenanceIntegration) initializeTypeMaps() {
|
||||
// Initialize empty maps
|
||||
s.taskTypeMap = make(map[types.TaskType]MaintenanceTaskType)
|
||||
s.revTaskTypeMap = make(map[MaintenanceTaskType]types.TaskType)
|
||||
|
||||
// Build task type mappings dynamically from registered tasks after registration
|
||||
// This will be called from registerAllTasks() after all tasks are registered
|
||||
|
||||
// Priority mappings (these are static and don't depend on registered tasks)
|
||||
s.priorityMap = map[types.TaskPriority]MaintenanceTaskPriority{
|
||||
types.TaskPriorityLow: PriorityLow,
|
||||
types.TaskPriorityNormal: PriorityNormal,
|
||||
types.TaskPriorityHigh: PriorityHigh,
|
||||
}
|
||||
|
||||
// Reverse priority mappings
|
||||
s.revPriorityMap = map[MaintenanceTaskPriority]types.TaskPriority{
|
||||
PriorityLow: types.TaskPriorityLow,
|
||||
PriorityNormal: types.TaskPriorityNormal,
|
||||
PriorityHigh: types.TaskPriorityHigh,
|
||||
PriorityCritical: types.TaskPriorityHigh, // Map critical to high
|
||||
}
|
||||
}
|
||||
|
||||
// buildTaskTypeMappings dynamically builds task type mappings from registered tasks
|
||||
func (s *MaintenanceIntegration) buildTaskTypeMappings() {
|
||||
// Clear existing mappings
|
||||
s.taskTypeMap = make(map[types.TaskType]MaintenanceTaskType)
|
||||
s.revTaskTypeMap = make(map[MaintenanceTaskType]types.TaskType)
|
||||
|
||||
// Build mappings from registered detectors
|
||||
for workerTaskType := range s.taskRegistry.GetAllDetectors() {
|
||||
// Convert types.TaskType to MaintenanceTaskType by string conversion
|
||||
maintenanceTaskType := MaintenanceTaskType(string(workerTaskType))
|
||||
|
||||
s.taskTypeMap[workerTaskType] = maintenanceTaskType
|
||||
s.revTaskTypeMap[maintenanceTaskType] = workerTaskType
|
||||
|
||||
glog.V(3).Infof("Dynamically mapped task type: %s <-> %s", workerTaskType, maintenanceTaskType)
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Built %d dynamic task type mappings", len(s.taskTypeMap))
|
||||
}
|
||||
|
||||
// registerAllTasks registers all available tasks
|
||||
func (s *MaintenanceIntegration) registerAllTasks() {
|
||||
// Tasks are already auto-registered via import statements
|
||||
// No manual registration needed
|
||||
|
||||
// Build dynamic type mappings from registered tasks
|
||||
s.buildTaskTypeMappings()
|
||||
|
||||
// Configure tasks from policy
|
||||
s.ConfigureTasksFromPolicy()
|
||||
|
||||
registeredTaskTypes := make([]string, 0, len(s.taskTypeMap))
|
||||
for _, maintenanceTaskType := range s.taskTypeMap {
|
||||
registeredTaskTypes = append(registeredTaskTypes, string(maintenanceTaskType))
|
||||
}
|
||||
glog.V(1).Infof("Registered tasks: %v", registeredTaskTypes)
|
||||
}
|
||||
|
||||
// SetPolicy replaces the maintenance policy the integration configures tasks from and
|
||||
// applies it immediately. Without this the integration kept the policy it was built with,
|
||||
// so a policy updated at runtime reached the queue but never the detectors that decide
|
||||
// which task types are scanned for.
|
||||
//
|
||||
// Not safe to call concurrently with a running scan. ConfigureTasksFromPolicy writes
|
||||
// TaskDefinition.Config and TaskDefinition.MaxConcurrent, which ScanWithTaskDetectors reads
|
||||
// through detector.IsEnabled(); nothing synchronises the two. Today every caller runs during
|
||||
// admin server startup, before the scan loop exists. Anything that wires this to an HTTP
|
||||
// handler has to add that synchronisation first - the same applies to
|
||||
// tasks.ConfigUpdateRegistry.UpdateAllConfigs, which replaces TaskDefinition.Config outright.
|
||||
func (s *MaintenanceIntegration) SetPolicy(policy *MaintenancePolicy) {
|
||||
s.maintenancePolicy = policy
|
||||
s.ConfigureTasksFromPolicy()
|
||||
}
|
||||
|
||||
// ConfigureTasksFromPolicy dynamically configures all registered tasks based on the maintenance policy
|
||||
func (s *MaintenanceIntegration) ConfigureTasksFromPolicy() {
|
||||
if s.maintenancePolicy == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Configure all registered detectors and schedulers dynamically using policy configuration
|
||||
configuredCount := 0
|
||||
|
||||
// Get all registered task types from the registry
|
||||
for taskType, detector := range s.taskRegistry.GetAllDetectors() {
|
||||
// Configure detector using policy-based configuration
|
||||
s.configureDetectorFromPolicy(taskType, detector)
|
||||
configuredCount++
|
||||
}
|
||||
|
||||
for taskType, scheduler := range s.taskRegistry.GetAllSchedulers() {
|
||||
// Configure scheduler using policy-based configuration
|
||||
s.configureSchedulerFromPolicy(taskType, scheduler)
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Dynamically configured %d task types from maintenance policy", configuredCount)
|
||||
}
|
||||
|
||||
// configureDetectorFromPolicy configures a detector using policy-based configuration
|
||||
func (s *MaintenanceIntegration) configureDetectorFromPolicy(taskType types.TaskType, detector types.TaskDetector) {
|
||||
// Try to configure using PolicyConfigurableDetector interface if supported
|
||||
if configurableDetector, ok := detector.(types.PolicyConfigurableDetector); ok {
|
||||
configurableDetector.ConfigureFromPolicy(s.maintenancePolicy)
|
||||
glog.V(2).Infof("Configured detector %s using policy interface", taskType)
|
||||
return
|
||||
}
|
||||
|
||||
// Convert task system type to maintenance task type for policy lookup
|
||||
maintenanceTaskType, exists := s.taskTypeMap[taskType]
|
||||
if !exists {
|
||||
glog.V(3).Infof("No maintenance task type mapping for %s, skipping configuration", taskType)
|
||||
return
|
||||
}
|
||||
|
||||
// A task type the policy says nothing about is left alone. IsTaskEnabled reports
|
||||
// false for a missing entry, so applying it unconditionally would silently disable
|
||||
// every task the policy does not list - which is how ec_balance would have been
|
||||
// switched off the moment SetEnabled started working.
|
||||
if GetTaskPolicy(s.maintenancePolicy, maintenanceTaskType) == nil {
|
||||
glog.V(2).Infof("Maintenance policy has no entry for %s, leaving its detector configuration untouched", taskType)
|
||||
return
|
||||
}
|
||||
|
||||
// Apply basic configuration that all detectors should support
|
||||
if basicDetector, ok := detector.(interface{ SetEnabled(bool) }); ok {
|
||||
enabled := IsTaskEnabled(s.maintenancePolicy, maintenanceTaskType)
|
||||
basicDetector.SetEnabled(enabled)
|
||||
glog.V(3).Infof("Set enabled=%v for detector %s", enabled, taskType)
|
||||
} else {
|
||||
// For detectors that don't implement PolicyConfigurableDetector interface,
|
||||
// they should be updated to implement it for full policy-based configuration
|
||||
glog.V(2).Infof("Detector %s supports neither PolicyConfigurableDetector nor SetEnabled, its policy is ignored", taskType)
|
||||
}
|
||||
}
|
||||
|
||||
// configureSchedulerFromPolicy configures a scheduler using policy-based configuration
|
||||
func (s *MaintenanceIntegration) configureSchedulerFromPolicy(taskType types.TaskType, scheduler types.TaskScheduler) {
|
||||
// Try to configure using PolicyConfigurableScheduler interface if supported
|
||||
if configurableScheduler, ok := scheduler.(types.PolicyConfigurableScheduler); ok {
|
||||
configurableScheduler.ConfigureFromPolicy(s.maintenancePolicy)
|
||||
glog.V(2).Infof("Configured scheduler %s using policy interface", taskType)
|
||||
return
|
||||
}
|
||||
|
||||
// Apply basic configuration that all schedulers should support
|
||||
maintenanceTaskType, exists := s.taskTypeMap[taskType]
|
||||
if !exists {
|
||||
glog.V(3).Infof("No maintenance task type mapping for %s, skipping configuration", taskType)
|
||||
return
|
||||
}
|
||||
|
||||
// Same guard as on the detector side: no policy entry means no opinion, not disabled.
|
||||
if GetTaskPolicy(s.maintenancePolicy, maintenanceTaskType) == nil {
|
||||
glog.V(2).Infof("Maintenance policy has no entry for %s, leaving its scheduler configuration untouched", taskType)
|
||||
return
|
||||
}
|
||||
|
||||
// Set enabled status if scheduler supports it
|
||||
if enableableScheduler, ok := scheduler.(interface{ SetEnabled(bool) }); ok {
|
||||
enabled := IsTaskEnabled(s.maintenancePolicy, maintenanceTaskType)
|
||||
enableableScheduler.SetEnabled(enabled)
|
||||
glog.V(3).Infof("Set enabled=%v for scheduler %s", enabled, taskType)
|
||||
} else {
|
||||
// For schedulers that don't implement PolicyConfigurableScheduler interface,
|
||||
// they should be updated to implement it for full policy-based configuration
|
||||
glog.V(2).Infof("Scheduler %s supports neither PolicyConfigurableScheduler nor SetEnabled, its policy is ignored", taskType)
|
||||
}
|
||||
|
||||
// Set max concurrent if scheduler supports it
|
||||
if concurrentScheduler, ok := scheduler.(interface{ SetMaxConcurrent(int) }); ok {
|
||||
maxConcurrent := GetMaxConcurrent(s.maintenancePolicy, maintenanceTaskType)
|
||||
if maxConcurrent > 0 {
|
||||
concurrentScheduler.SetMaxConcurrent(maxConcurrent)
|
||||
glog.V(3).Infof("Set max concurrent=%d for scheduler %s", maxConcurrent, taskType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ScanWithTaskDetectors performs a scan using the task system
|
||||
func (s *MaintenanceIntegration) ScanWithTaskDetectors(volumeMetrics []*types.VolumeHealthMetrics) ([]*TaskDetectionResult, error) {
|
||||
// Note: ActiveTopology gets updated from topology info instead of volume metrics
|
||||
glog.V(2).Infof("Processed %d volume metrics for task detection", len(volumeMetrics))
|
||||
|
||||
// Filter out volumes with pending operations to avoid duplicates
|
||||
filteredMetrics := s.pendingOperations.FilterVolumeMetricsExcludingPending(volumeMetrics)
|
||||
|
||||
glog.V(1).Infof("Scanning %d volumes (filtered from %d) excluding pending operations",
|
||||
len(filteredMetrics), len(volumeMetrics))
|
||||
|
||||
var allResults []*TaskDetectionResult
|
||||
|
||||
// Create cluster info
|
||||
clusterInfo := &types.ClusterInfo{
|
||||
TotalVolumes: len(filteredMetrics),
|
||||
LastUpdated: time.Now(),
|
||||
ActiveTopology: s.activeTopology, // Provide ActiveTopology for destination planning
|
||||
DefaultReplicaPlacement: s.defaultReplicaPlacement,
|
||||
}
|
||||
|
||||
// Run detection for each registered task type
|
||||
for taskType, detector := range s.taskRegistry.GetAllDetectors() {
|
||||
if !detector.IsEnabled() {
|
||||
continue
|
||||
}
|
||||
|
||||
// Cancel stale pending tasks for this type before re-detection
|
||||
maintenanceType := s.taskTypeMap[taskType]
|
||||
if cancelled := s.maintenanceQueue.CancelPendingTasksByType(maintenanceType); cancelled > 0 {
|
||||
glog.Infof("Cancelled %d stale pending %s tasks before re-detection", cancelled, taskType)
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Running detection for task type: %s", taskType)
|
||||
|
||||
results, err := detector.ScanForTasks(filteredMetrics, clusterInfo)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to scan for %s tasks: %v", taskType, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Convert results to existing system format and check for conflicts
|
||||
for _, result := range results {
|
||||
existingResult := s.convertToExistingFormat(result)
|
||||
if existingResult != nil {
|
||||
// Double-check for conflicts with pending operations
|
||||
opType := s.mapMaintenanceTaskTypeToPendingOperationType(existingResult.TaskType)
|
||||
if !s.pendingOperations.WouldConflictWithPending(existingResult.VolumeID, opType) {
|
||||
// All task types should now have TypedParams populated during detection phase
|
||||
if existingResult.TypedParams == nil {
|
||||
glog.Warningf("Task %s for volume %d has no typed parameters - skipping (task parameter creation may have failed)",
|
||||
existingResult.TaskType, existingResult.VolumeID)
|
||||
continue
|
||||
}
|
||||
allResults = append(allResults, existingResult)
|
||||
} else {
|
||||
glog.V(2).Infof("Skipping task %s for volume %d due to conflict with pending operation",
|
||||
existingResult.TaskType, existingResult.VolumeID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Found %d %s tasks", len(results), taskType)
|
||||
}
|
||||
|
||||
return allResults, nil
|
||||
}
|
||||
|
||||
// SetDefaultReplicaPlacement records the master's default replication so detectors
|
||||
// can use it as the replica-placement fallback (matching the shell).
|
||||
func (s *MaintenanceIntegration) SetDefaultReplicaPlacement(replicaPlacement string) {
|
||||
s.defaultReplicaPlacement = replicaPlacement
|
||||
}
|
||||
|
||||
// UpdateTopologyInfo updates the volume shard tracker with topology information for empty servers
|
||||
func (s *MaintenanceIntegration) UpdateTopologyInfo(topologyInfo *master_pb.TopologyInfo) error {
|
||||
// Log topology details before update for diagnostics
|
||||
if topologyInfo != nil {
|
||||
dcCount, nodeCount, diskCount := topology.CountTopologyResources(topologyInfo)
|
||||
glog.V(2).Infof("UpdateTopologyInfo: received topology with %d datacenters, %d nodes, %d disks",
|
||||
dcCount, nodeCount, diskCount)
|
||||
} else {
|
||||
glog.Warningf("UpdateTopologyInfo: received nil topologyInfo")
|
||||
}
|
||||
|
||||
err := s.activeTopology.UpdateTopology(topologyInfo)
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("UpdateTopologyInfo: topology update failed: %v", err)
|
||||
} else {
|
||||
// Log success with current disk count
|
||||
currentDiskCount := s.activeTopology.GetDiskCount()
|
||||
glog.V(1).Infof("UpdateTopologyInfo: topology update successful, active topology now has %d disks", currentDiskCount)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// convertToExistingFormat converts task results to existing system format using dynamic mapping
|
||||
func (s *MaintenanceIntegration) convertToExistingFormat(result *types.TaskDetectionResult) *TaskDetectionResult {
|
||||
// Convert types using mapping tables
|
||||
existingType, exists := s.taskTypeMap[result.TaskType]
|
||||
if !exists {
|
||||
glog.Warningf("Unknown task type %s, skipping conversion", result.TaskType)
|
||||
// Return nil to indicate conversion failed - caller should handle this
|
||||
return nil
|
||||
}
|
||||
|
||||
existingPriority, exists := s.priorityMap[result.Priority]
|
||||
if !exists {
|
||||
glog.Warningf("Unknown priority %s, defaulting to normal", result.Priority)
|
||||
existingPriority = PriorityNormal
|
||||
}
|
||||
|
||||
return &TaskDetectionResult{
|
||||
TaskID: result.TaskID,
|
||||
TaskType: existingType,
|
||||
VolumeID: result.VolumeID,
|
||||
Server: result.Server,
|
||||
Collection: result.Collection,
|
||||
Priority: existingPriority,
|
||||
Reason: result.Reason,
|
||||
TypedParams: result.TypedParams,
|
||||
ScheduleAt: result.ScheduleAt,
|
||||
}
|
||||
}
|
||||
|
||||
// CanScheduleWithTaskSchedulers determines if a task can be scheduled using task schedulers with dynamic type conversion
|
||||
func (s *MaintenanceIntegration) CanScheduleWithTaskSchedulers(task *MaintenanceTask, runningTasks []*MaintenanceTask, availableWorkers []*MaintenanceWorker) bool {
|
||||
|
||||
// Convert existing types to task types using mapping
|
||||
taskType, exists := s.revTaskTypeMap[task.Type]
|
||||
if !exists {
|
||||
return false // Fallback to existing logic for unknown types
|
||||
}
|
||||
|
||||
// Convert task objects
|
||||
taskObject := s.convertTaskToTaskSystem(task)
|
||||
if taskObject == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
runningTaskObjects := s.convertTasksToTaskSystem(runningTasks)
|
||||
workerObjects := s.convertWorkersToTaskSystem(availableWorkers)
|
||||
|
||||
// Get the appropriate scheduler
|
||||
scheduler := s.taskRegistry.GetScheduler(taskType)
|
||||
if scheduler == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
canSchedule := scheduler.CanScheduleNow(taskObject, runningTaskObjects, workerObjects)
|
||||
|
||||
return canSchedule
|
||||
}
|
||||
|
||||
// convertTaskToTaskSystem converts existing task to task system format using dynamic mapping
|
||||
func (s *MaintenanceIntegration) convertTaskToTaskSystem(task *MaintenanceTask) *types.TaskInput {
|
||||
// Convert task type using mapping
|
||||
taskType, exists := s.revTaskTypeMap[task.Type]
|
||||
if !exists {
|
||||
glog.Errorf("Unknown task type %s in conversion, cannot convert task", task.Type)
|
||||
// Return nil to indicate conversion failed
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert priority using mapping
|
||||
priority, exists := s.revPriorityMap[task.Priority]
|
||||
if !exists {
|
||||
glog.Warningf("Unknown priority %d in conversion, defaulting to normal", task.Priority)
|
||||
priority = types.TaskPriorityNormal
|
||||
}
|
||||
|
||||
return &types.TaskInput{
|
||||
ID: task.ID,
|
||||
Type: taskType,
|
||||
Priority: priority,
|
||||
VolumeID: task.VolumeID,
|
||||
Server: task.Server,
|
||||
Collection: task.Collection,
|
||||
TypedParams: task.TypedParams,
|
||||
CreatedAt: task.CreatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
// convertTasksToTaskSystem converts multiple tasks
|
||||
func (s *MaintenanceIntegration) convertTasksToTaskSystem(tasks []*MaintenanceTask) []*types.TaskInput {
|
||||
var result []*types.TaskInput
|
||||
for _, task := range tasks {
|
||||
converted := s.convertTaskToTaskSystem(task)
|
||||
if converted != nil {
|
||||
result = append(result, converted)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// convertWorkersToTaskSystem converts workers to task system format using dynamic mapping
|
||||
func (s *MaintenanceIntegration) convertWorkersToTaskSystem(workers []*MaintenanceWorker) []*types.WorkerData {
|
||||
var result []*types.WorkerData
|
||||
for _, worker := range workers {
|
||||
capabilities := make([]types.TaskType, 0, len(worker.Capabilities))
|
||||
for _, cap := range worker.Capabilities {
|
||||
// Convert capability using mapping
|
||||
taskType, exists := s.revTaskTypeMap[cap]
|
||||
if exists {
|
||||
capabilities = append(capabilities, taskType)
|
||||
} else {
|
||||
glog.V(3).Infof("Unknown capability %s for worker %s, skipping", cap, worker.ID)
|
||||
}
|
||||
}
|
||||
|
||||
result = append(result, &types.WorkerData{
|
||||
ID: worker.ID,
|
||||
Address: worker.Address,
|
||||
Capabilities: capabilities,
|
||||
MaxConcurrent: worker.MaxConcurrent,
|
||||
CurrentLoad: worker.CurrentLoad,
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// GetTaskScheduler returns the scheduler for a task type using dynamic mapping
|
||||
func (s *MaintenanceIntegration) GetTaskScheduler(taskType MaintenanceTaskType) types.TaskScheduler {
|
||||
// Convert task type using mapping
|
||||
taskSystemType, exists := s.revTaskTypeMap[taskType]
|
||||
if !exists {
|
||||
glog.V(3).Infof("Unknown task type %s for scheduler", taskType)
|
||||
return nil
|
||||
}
|
||||
|
||||
return s.taskRegistry.GetScheduler(taskSystemType)
|
||||
}
|
||||
|
||||
// GetUIProvider returns the UI provider for a task type using dynamic mapping
|
||||
func (s *MaintenanceIntegration) GetUIProvider(taskType MaintenanceTaskType) types.TaskUIProvider {
|
||||
// Convert task type using mapping
|
||||
taskSystemType, exists := s.revTaskTypeMap[taskType]
|
||||
if !exists {
|
||||
glog.V(3).Infof("Unknown task type %s for UI provider", taskType)
|
||||
return nil
|
||||
}
|
||||
|
||||
return s.uiRegistry.GetProvider(taskSystemType)
|
||||
}
|
||||
|
||||
// GetAllTaskStats returns stats for all registered tasks
|
||||
func (s *MaintenanceIntegration) GetAllTaskStats() []*types.TaskStats {
|
||||
var stats []*types.TaskStats
|
||||
|
||||
for taskType, detector := range s.taskRegistry.GetAllDetectors() {
|
||||
uiProvider := s.uiRegistry.GetProvider(taskType)
|
||||
if uiProvider == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
stat := &types.TaskStats{
|
||||
TaskType: taskType,
|
||||
DisplayName: uiProvider.GetDisplayName(),
|
||||
Enabled: detector.IsEnabled(),
|
||||
LastScan: time.Now().Add(-detector.ScanInterval()),
|
||||
NextScan: time.Now().Add(detector.ScanInterval()),
|
||||
ScanInterval: detector.ScanInterval(),
|
||||
MaxConcurrent: s.taskRegistry.GetScheduler(taskType).GetMaxConcurrent(),
|
||||
// Would need to get these from actual queue/stats
|
||||
PendingTasks: 0,
|
||||
RunningTasks: 0,
|
||||
CompletedToday: 0,
|
||||
FailedToday: 0,
|
||||
}
|
||||
|
||||
stats = append(stats, stat)
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// mapMaintenanceTaskTypeToPendingOperationType converts a maintenance task type to a pending operation type
|
||||
func (s *MaintenanceIntegration) mapMaintenanceTaskTypeToPendingOperationType(taskType MaintenanceTaskType) PendingOperationType {
|
||||
switch taskType {
|
||||
case MaintenanceTaskType("balance"):
|
||||
return OpTypeVolumeBalance
|
||||
case MaintenanceTaskType("erasure_coding"):
|
||||
return OpTypeErasureCoding
|
||||
case MaintenanceTaskType("vacuum"):
|
||||
return OpTypeVacuum
|
||||
case MaintenanceTaskType("replication"):
|
||||
return OpTypeReplication
|
||||
default:
|
||||
// For other task types, assume they're volume operations
|
||||
return OpTypeVolumeMove
|
||||
}
|
||||
}
|
||||
|
||||
// GetPendingOperations returns the pending operations tracker
|
||||
func (s *MaintenanceIntegration) GetPendingOperations() *PendingOperations {
|
||||
return s.pendingOperations
|
||||
}
|
||||
|
||||
// GetActiveTopology returns the active topology for task detection
|
||||
func (s *MaintenanceIntegration) GetActiveTopology() *topology.ActiveTopology {
|
||||
return s.activeTopology
|
||||
}
|
||||
|
||||
// SyncTask synchronizes a maintenance task with the active topology for capacity tracking
|
||||
func (s *MaintenanceIntegration) SyncTask(task *MaintenanceTask) {
|
||||
if s.activeTopology == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Convert task type
|
||||
taskType, exists := s.revTaskTypeMap[task.Type]
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
|
||||
// Convert status
|
||||
var status topology.TaskStatus
|
||||
switch task.Status {
|
||||
case TaskStatusPending:
|
||||
status = topology.TaskStatusPending
|
||||
case TaskStatusAssigned, TaskStatusInProgress:
|
||||
status = topology.TaskStatusInProgress
|
||||
default:
|
||||
return // Don't sync completed/failed/cancelled tasks
|
||||
}
|
||||
|
||||
// Extract sources and destinations from TypedParams
|
||||
var sources []topology.TaskSource
|
||||
var destinations []topology.TaskDestination
|
||||
var estimatedSize int64
|
||||
|
||||
if task.TypedParams != nil {
|
||||
// Calculate storage impact for this task type
|
||||
// Volume size is not currently used for Balance/Vacuum impact and is not stored in MaintenanceTask
|
||||
sourceImpact, targetImpact := topology.CalculateTaskStorageImpact(topology.TaskType(string(taskType)), 0)
|
||||
|
||||
// Use unified sources and targets from TaskParams.
|
||||
// Task protos store ServerAddresses (with gRPC port, e.g., "host:port.grpcPort")
|
||||
// but the topology indexes disks by NodeId (e.g., "host:port").
|
||||
// Strip the gRPC port suffix via ToHttpAddress() to match the topology key.
|
||||
for _, src := range task.TypedParams.Sources {
|
||||
resolvedSrc := pb.ServerAddress(src.Node).ToHttpAddress()
|
||||
glog.V(2).Infof("SyncTask %s: source proto Node=%q resolved to %q, diskId=%d", task.ID, src.Node, resolvedSrc, src.DiskId)
|
||||
sources = append(sources, topology.TaskSource{
|
||||
SourceServer: resolvedSrc,
|
||||
SourceDisk: src.DiskId,
|
||||
StorageChange: sourceImpact,
|
||||
})
|
||||
// Sum estimated size from all sources
|
||||
estimatedSize += int64(src.EstimatedSize)
|
||||
}
|
||||
for _, target := range task.TypedParams.Targets {
|
||||
resolvedTarget := pb.ServerAddress(target.Node).ToHttpAddress()
|
||||
glog.V(2).Infof("SyncTask %s: target proto Node=%q resolved to %q, diskId=%d", task.ID, target.Node, resolvedTarget, target.DiskId)
|
||||
destinations = append(destinations, topology.TaskDestination{
|
||||
TargetServer: resolvedTarget,
|
||||
TargetDisk: target.DiskId,
|
||||
StorageChange: targetImpact,
|
||||
})
|
||||
}
|
||||
|
||||
// Handle type-specific params for additional task-specific sync logic
|
||||
if vacuumParams := task.TypedParams.GetVacuumParams(); vacuumParams != nil {
|
||||
// TODO: Add vacuum-specific sync logic if necessary
|
||||
} else if ecParams := task.TypedParams.GetErasureCodingParams(); ecParams != nil {
|
||||
// TODO: Add EC-specific sync logic if necessary
|
||||
} else if balanceParams := task.TypedParams.GetBalanceParams(); balanceParams != nil {
|
||||
// TODO: Add balance-specific sync logic if necessary
|
||||
}
|
||||
}
|
||||
|
||||
// Restore into topology
|
||||
s.activeTopology.RestoreMaintenanceTask(task.ID, task.VolumeID, topology.TaskType(string(taskType)), status, sources, destinations, estimatedSize)
|
||||
}
|
||||
675
weed/admin/maintenance/maintenance_manager.go
Normal file
675
weed/admin/maintenance/maintenance_manager.go
Normal file
@@ -0,0 +1,675 @@
|
||||
package maintenance
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/ec_balance"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// BuildPolicyFromTaskConfigs loads each registered task's configuration and builds the
|
||||
// MaintenancePolicy the maintenance system runs on.
|
||||
//
|
||||
// Every entry is produced by the task's own ToTaskPolicy(), so a policy entry always carries
|
||||
// exactly what that task's config holds. Hand-copying the fields here instead made this a
|
||||
// second, silently diverging definition of every task's policy: it dropped the erasure
|
||||
// coding preferred tags and replica placement and the balance IO rate limit outright.
|
||||
//
|
||||
// Every task registered through base.RegisterTask needs an entry, because a task type the
|
||||
// policy does not list has no enabled flag and no concurrency limit of its own -
|
||||
// IsTaskEnabled reports false for a missing entry.
|
||||
//
|
||||
// configPersistence is duck-typed as interface{} because weed/admin/dash already imports this
|
||||
// package, so importing *dash.ConfigPersistence back here would create an import cycle. It must be
|
||||
// a value implementing the LoadXTaskPolicy() accessors the task loaders assert on; passing nil (or
|
||||
// anything else) makes every task fall back to its compiled-in defaults.
|
||||
func BuildPolicyFromTaskConfigs(configPersistence interface{}) *worker_pb.MaintenancePolicy {
|
||||
policy := &worker_pb.MaintenancePolicy{
|
||||
GlobalMaxConcurrent: 4,
|
||||
DefaultRepeatIntervalSeconds: 6 * 3600, // 6 hours in seconds
|
||||
DefaultCheckIntervalSeconds: 12 * 3600, // 12 hours in seconds
|
||||
TaskPolicies: make(map[string]*worker_pb.TaskPolicy),
|
||||
}
|
||||
|
||||
if vacuumConfig := vacuum.LoadConfigFromPersistence(configPersistence); vacuumConfig != nil {
|
||||
policy.TaskPolicies[string(types.TaskTypeVacuum)] = vacuumConfig.ToTaskPolicy()
|
||||
}
|
||||
|
||||
if ecConfig := erasure_coding.LoadConfigFromPersistence(configPersistence); ecConfig != nil {
|
||||
policy.TaskPolicies[string(types.TaskTypeErasureCoding)] = ecConfig.ToTaskPolicy()
|
||||
}
|
||||
|
||||
if balanceConfig := balance.LoadConfigFromPersistence(configPersistence); balanceConfig != nil {
|
||||
policy.TaskPolicies[string(types.TaskTypeBalance)] = balanceConfig.ToTaskPolicy()
|
||||
}
|
||||
|
||||
if ecBalanceConfig := ec_balance.LoadConfigFromPersistence(configPersistence); ecBalanceConfig != nil {
|
||||
policy.TaskPolicies[string(types.TaskTypeECBalance)] = ecBalanceConfig.ToTaskPolicy()
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Built maintenance policy from separate task configs - %d task policies loaded", len(policy.TaskPolicies))
|
||||
return policy
|
||||
}
|
||||
|
||||
// MaintenanceManager coordinates the maintenance system
|
||||
type MaintenanceManager struct {
|
||||
config *MaintenanceConfig
|
||||
scanner *MaintenanceScanner
|
||||
queue *MaintenanceQueue
|
||||
adminClient AdminClient
|
||||
running bool
|
||||
stopChan chan struct{}
|
||||
// Error handling and backoff
|
||||
errorCount int
|
||||
lastError error
|
||||
lastErrorTime time.Time
|
||||
backoffDelay time.Duration
|
||||
mutex sync.RWMutex
|
||||
scanInProgress bool
|
||||
}
|
||||
|
||||
// NewMaintenanceManager creates a new maintenance manager.
|
||||
//
|
||||
// configPersistence is the config store to read persisted task configs from when the policy has to
|
||||
// be built here. See BuildPolicyFromTaskConfigs for why it is duck-typed; pass nil when no config
|
||||
// store is available.
|
||||
func NewMaintenanceManager(adminClient AdminClient, config *MaintenanceConfig, configPersistence interface{}) *MaintenanceManager {
|
||||
if config == nil {
|
||||
config = DefaultMaintenanceConfig()
|
||||
}
|
||||
|
||||
// Use the policy from the config (which is populated from separate task files in LoadMaintenanceConfig)
|
||||
policy := config.Policy
|
||||
if policy == nil {
|
||||
// Fallback: build policy from separate task configuration files if not already populated
|
||||
policy = BuildPolicyFromTaskConfigs(configPersistence)
|
||||
}
|
||||
|
||||
queue := NewMaintenanceQueue(policy)
|
||||
scanner := NewMaintenanceScanner(adminClient, policy, queue)
|
||||
|
||||
return &MaintenanceManager{
|
||||
config: config,
|
||||
scanner: scanner,
|
||||
queue: queue,
|
||||
adminClient: adminClient,
|
||||
stopChan: make(chan struct{}),
|
||||
backoffDelay: time.Second, // Start with 1 second backoff
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins the maintenance manager
|
||||
func (mm *MaintenanceManager) Start() error {
|
||||
if !mm.config.GetEnabled() {
|
||||
glog.V(1).Infof("Maintenance system is disabled")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate configuration durations to prevent ticker panics
|
||||
if err := mm.validateConfig(); err != nil {
|
||||
return fmt.Errorf("invalid maintenance configuration: %w", err)
|
||||
}
|
||||
|
||||
mm.mutex.Lock()
|
||||
mm.running = true
|
||||
mm.mutex.Unlock()
|
||||
|
||||
// Start background processes
|
||||
go mm.scanLoop()
|
||||
go mm.cleanupLoop()
|
||||
go mm.topologyStatusLoop() // Periodic diagnostic logging
|
||||
|
||||
glog.Infof("Maintenance manager started with scan interval %ds", mm.config.ScanIntervalSeconds)
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateConfig validates the maintenance configuration durations
|
||||
func (mm *MaintenanceManager) validateConfig() error {
|
||||
if mm.config.ScanIntervalSeconds <= 0 {
|
||||
glog.Warningf("Invalid scan interval %ds, using default 30m", mm.config.ScanIntervalSeconds)
|
||||
mm.config.ScanIntervalSeconds = 30 * 60 // 30 minutes in seconds
|
||||
}
|
||||
|
||||
if mm.config.CleanupIntervalSeconds <= 0 {
|
||||
glog.Warningf("Invalid cleanup interval %ds, using default 24h", mm.config.CleanupIntervalSeconds)
|
||||
mm.config.CleanupIntervalSeconds = 24 * 60 * 60 // 24 hours in seconds
|
||||
}
|
||||
|
||||
if mm.config.WorkerTimeoutSeconds <= 0 {
|
||||
glog.Warningf("Invalid worker timeout %ds, using default 5m", mm.config.WorkerTimeoutSeconds)
|
||||
mm.config.WorkerTimeoutSeconds = 5 * 60 // 5 minutes in seconds
|
||||
}
|
||||
|
||||
if mm.config.TaskTimeoutSeconds <= 0 {
|
||||
glog.Warningf("Invalid task timeout %ds, using default 2h", mm.config.TaskTimeoutSeconds)
|
||||
mm.config.TaskTimeoutSeconds = 2 * 60 * 60 // 2 hours in seconds
|
||||
}
|
||||
|
||||
if mm.config.RetryDelaySeconds <= 0 {
|
||||
glog.Warningf("Invalid retry delay %ds, using default 15m", mm.config.RetryDelaySeconds)
|
||||
mm.config.RetryDelaySeconds = 15 * 60 // 15 minutes in seconds
|
||||
}
|
||||
|
||||
if mm.config.TaskRetentionSeconds <= 0 {
|
||||
glog.Warningf("Invalid task retention %ds, using default 168h", mm.config.TaskRetentionSeconds)
|
||||
mm.config.TaskRetentionSeconds = 7 * 24 * 60 * 60 // 7 days in seconds
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsRunning returns whether the maintenance manager is currently running.
|
||||
// running is guarded by mm.mutex because the background loops read it on every
|
||||
// iteration while Start and Stop are called from the admin server's goroutines.
|
||||
func (mm *MaintenanceManager) IsRunning() bool {
|
||||
mm.mutex.RLock()
|
||||
defer mm.mutex.RUnlock()
|
||||
return mm.running
|
||||
}
|
||||
|
||||
// Stop terminates the maintenance manager. It is a no-op when the manager is not
|
||||
// running, so a second call cannot close the already closed stop channel.
|
||||
func (mm *MaintenanceManager) Stop() {
|
||||
mm.mutex.Lock()
|
||||
if !mm.running {
|
||||
mm.mutex.Unlock()
|
||||
return
|
||||
}
|
||||
mm.running = false
|
||||
close(mm.stopChan)
|
||||
mm.mutex.Unlock()
|
||||
|
||||
glog.Infof("Maintenance manager stopped")
|
||||
}
|
||||
|
||||
// scanLoop periodically scans for maintenance tasks with adaptive timing
|
||||
func (mm *MaintenanceManager) scanLoop() {
|
||||
scanInterval := time.Duration(mm.config.ScanIntervalSeconds) * time.Second
|
||||
|
||||
// activeInterval is the interval the ticker is actually running at right now. It has
|
||||
// to be tracked separately from the configured scanInterval, because the error backoff
|
||||
// replaces the ticker with a much shorter one. Comparing the target against the
|
||||
// configured interval instead never restores the normal cadence: once the errors stop,
|
||||
// getScanInterval returns scanInterval again, the comparison comes out false, and the
|
||||
// ticker is left at the backoff delay. A single transient scan failure therefore pinned
|
||||
// the scanner to one scan per second forever.
|
||||
activeInterval := scanInterval
|
||||
ticker := time.NewTicker(activeInterval)
|
||||
// Wrapped in a closure so the replacement ticker is stopped, not the one that happened
|
||||
// to be current when the defer was registered.
|
||||
defer func() { ticker.Stop() }()
|
||||
|
||||
for mm.IsRunning() {
|
||||
select {
|
||||
case <-mm.stopChan:
|
||||
return
|
||||
case <-ticker.C:
|
||||
glog.V(1).Infof("Performing maintenance scan every %v", activeInterval)
|
||||
|
||||
// Use the same synchronization as TriggerScan to prevent concurrent scans
|
||||
if err := mm.triggerScanInternal(false); err != nil {
|
||||
glog.V(1).Infof("Scheduled scan skipped: %v", err)
|
||||
}
|
||||
|
||||
// Adjust ticker interval based on error state (read error state safely)
|
||||
currentInterval := mm.getScanInterval(scanInterval)
|
||||
|
||||
// Reset ticker whenever the target differs from what the ticker is running at,
|
||||
// which covers both entering the backoff and returning to the normal cadence.
|
||||
if currentInterval != activeInterval {
|
||||
ticker.Stop()
|
||||
ticker = time.NewTicker(currentInterval)
|
||||
glog.V(1).Infof("Maintenance scan cadence changed from %v to %v", activeInterval, currentInterval)
|
||||
activeInterval = currentInterval
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// getScanInterval safely reads the current scan interval with error backoff
|
||||
func (mm *MaintenanceManager) getScanInterval(baseInterval time.Duration) time.Duration {
|
||||
mm.mutex.RLock()
|
||||
defer mm.mutex.RUnlock()
|
||||
|
||||
if mm.errorCount > 0 {
|
||||
// Use backoff delay when there are errors
|
||||
currentInterval := mm.backoffDelay
|
||||
if currentInterval > baseInterval {
|
||||
// Don't make it longer than the configured interval * 10
|
||||
maxInterval := baseInterval * 10
|
||||
if currentInterval > maxInterval {
|
||||
currentInterval = maxInterval
|
||||
}
|
||||
}
|
||||
return currentInterval
|
||||
}
|
||||
return baseInterval
|
||||
}
|
||||
|
||||
// cleanupLoop periodically cleans up old tasks and stale workers
|
||||
func (mm *MaintenanceManager) cleanupLoop() {
|
||||
cleanupInterval := time.Duration(mm.config.CleanupIntervalSeconds) * time.Second
|
||||
ticker := time.NewTicker(cleanupInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for mm.IsRunning() {
|
||||
select {
|
||||
case <-mm.stopChan:
|
||||
return
|
||||
case <-ticker.C:
|
||||
mm.performCleanup()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// topologyStatusLoop periodically logs topology status for diagnostics
|
||||
func (mm *MaintenanceManager) topologyStatusLoop() {
|
||||
// Log topology status every 5 minutes for diagnostic purposes
|
||||
statusInterval := 5 * time.Minute
|
||||
ticker := time.NewTicker(statusInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for mm.IsRunning() {
|
||||
select {
|
||||
case <-mm.stopChan:
|
||||
return
|
||||
case <-ticker.C:
|
||||
mm.logTopologyStatus()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// logTopologyStatus logs current topology and worker status for diagnostics
|
||||
func (mm *MaintenanceManager) logTopologyStatus() {
|
||||
if mm.scanner == nil || mm.scanner.integration == nil {
|
||||
glog.V(2).Infof("Topology status: scanner/integration not available")
|
||||
return
|
||||
}
|
||||
|
||||
activeTopology := mm.scanner.integration.GetActiveTopology()
|
||||
if activeTopology == nil {
|
||||
glog.V(1).Infof("Topology status: ActiveTopology is nil")
|
||||
return
|
||||
}
|
||||
|
||||
diskCount := activeTopology.GetDiskCount()
|
||||
nodeCount := len(activeTopology.GetAllNodes())
|
||||
|
||||
// Get queue stats
|
||||
stats := mm.queue.GetStats()
|
||||
workerCount := len(mm.queue.GetWorkers())
|
||||
|
||||
mm.mutex.RLock()
|
||||
errorCount := mm.errorCount
|
||||
mm.mutex.RUnlock()
|
||||
|
||||
glog.V(1).Infof("Topology status: %d nodes, %d disks, %d workers, %d pending tasks, %d running tasks, errors: %d",
|
||||
nodeCount, diskCount, workerCount,
|
||||
stats.TasksByStatus[TaskStatusPending],
|
||||
stats.TasksByStatus[TaskStatusInProgress]+stats.TasksByStatus[TaskStatusAssigned],
|
||||
errorCount)
|
||||
}
|
||||
|
||||
// performScan executes a maintenance scan with error handling and backoff
|
||||
func (mm *MaintenanceManager) performScan() {
|
||||
defer func() {
|
||||
// Always reset scan in progress flag when done
|
||||
mm.mutex.Lock()
|
||||
mm.scanInProgress = false
|
||||
mm.mutex.Unlock()
|
||||
}()
|
||||
|
||||
glog.Infof("Starting maintenance scan...")
|
||||
|
||||
results, err := mm.scanner.ScanForMaintenanceTasks()
|
||||
stats_collect.AdminMaintenanceLastScanTimestampSeconds.SetToCurrentTime()
|
||||
if err != nil {
|
||||
// Handle scan error
|
||||
mm.mutex.Lock()
|
||||
mm.handleScanError(err)
|
||||
mm.mutex.Unlock()
|
||||
glog.Warningf("Maintenance scan failed: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Scan succeeded - update state and process results
|
||||
mm.handleScanSuccess(results)
|
||||
}
|
||||
|
||||
// handleScanSuccess processes successful scan results with proper lock management
|
||||
func (mm *MaintenanceManager) handleScanSuccess(results []*TaskDetectionResult) {
|
||||
// Update manager state first
|
||||
mm.mutex.Lock()
|
||||
mm.resetErrorTracking()
|
||||
taskCount := len(results)
|
||||
mm.mutex.Unlock()
|
||||
|
||||
if taskCount > 0 {
|
||||
// Count tasks by type for logging (outside of lock)
|
||||
taskCounts := make(map[MaintenanceTaskType]int)
|
||||
for _, result := range results {
|
||||
taskCounts[result.TaskType]++
|
||||
}
|
||||
|
||||
// Add tasks to queue (no manager lock held)
|
||||
mm.queue.AddTasksFromResults(results)
|
||||
|
||||
// Log detailed scan results
|
||||
glog.Infof("Maintenance scan completed: found %d tasks", taskCount)
|
||||
for taskType, count := range taskCounts {
|
||||
glog.Infof(" - %s: %d tasks", taskType, count)
|
||||
}
|
||||
} else {
|
||||
glog.Infof("Maintenance scan completed: no maintenance tasks needed")
|
||||
}
|
||||
}
|
||||
|
||||
// handleScanError handles scan errors with exponential backoff and reduced logging
|
||||
func (mm *MaintenanceManager) handleScanError(err error) {
|
||||
now := time.Now()
|
||||
mm.errorCount++
|
||||
mm.lastError = err
|
||||
mm.lastErrorTime = now
|
||||
|
||||
// Use exponential backoff with jitter
|
||||
if mm.errorCount > 1 {
|
||||
mm.backoffDelay = mm.backoffDelay * 2
|
||||
if mm.backoffDelay > 5*time.Minute {
|
||||
mm.backoffDelay = 5 * time.Minute // Cap at 5 minutes
|
||||
}
|
||||
}
|
||||
|
||||
// Reduce log frequency based on error count and time
|
||||
shouldLog := false
|
||||
if mm.errorCount <= 3 {
|
||||
// Log first 3 errors immediately
|
||||
shouldLog = true
|
||||
} else if mm.errorCount <= 10 && mm.errorCount%3 == 0 {
|
||||
// Log every 3rd error for errors 4-10
|
||||
shouldLog = true
|
||||
} else if mm.errorCount%10 == 0 {
|
||||
// Log every 10th error after that
|
||||
shouldLog = true
|
||||
}
|
||||
|
||||
if shouldLog {
|
||||
// Check if it's a connection error to provide better messaging
|
||||
if isConnectionError(err) {
|
||||
if mm.errorCount == 1 {
|
||||
glog.Errorf("Maintenance scan failed: %v (will retry with backoff)", err)
|
||||
} else {
|
||||
glog.Errorf("Maintenance scan still failing after %d attempts: %v (backoff: %v)",
|
||||
mm.errorCount, err, mm.backoffDelay)
|
||||
}
|
||||
} else {
|
||||
glog.Errorf("Maintenance scan failed: %v", err)
|
||||
}
|
||||
} else {
|
||||
// Use debug level for suppressed errors
|
||||
glog.V(3).Infof("Maintenance scan failed (error #%d, suppressed): %v", mm.errorCount, err)
|
||||
}
|
||||
}
|
||||
|
||||
// resetErrorTracking resets error tracking when scan succeeds
|
||||
func (mm *MaintenanceManager) resetErrorTracking() {
|
||||
if mm.errorCount > 0 {
|
||||
glog.V(1).Infof("Maintenance scan recovered after %d failed attempts", mm.errorCount)
|
||||
mm.errorCount = 0
|
||||
mm.lastError = nil
|
||||
mm.backoffDelay = time.Second // Reset to initial delay
|
||||
}
|
||||
}
|
||||
|
||||
// isConnectionError checks if the error is a connection-related error
|
||||
func isConnectionError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
errStr := err.Error()
|
||||
return strings.Contains(errStr, "connection refused") ||
|
||||
strings.Contains(errStr, "connection error") ||
|
||||
strings.Contains(errStr, "dial tcp") ||
|
||||
strings.Contains(errStr, "connection timeout") ||
|
||||
strings.Contains(errStr, "no route to host") ||
|
||||
strings.Contains(errStr, "network unreachable")
|
||||
}
|
||||
|
||||
// performCleanup cleans up old tasks and stale workers
|
||||
func (mm *MaintenanceManager) performCleanup() {
|
||||
glog.V(2).Infof("Starting maintenance cleanup")
|
||||
|
||||
taskRetention := time.Duration(mm.config.TaskRetentionSeconds) * time.Second
|
||||
workerTimeout := time.Duration(mm.config.WorkerTimeoutSeconds) * time.Second
|
||||
|
||||
removedTasks := mm.queue.CleanupOldTasks(taskRetention)
|
||||
removedWorkers := mm.queue.RemoveStaleWorkers(workerTimeout)
|
||||
|
||||
// Clean up stale pending operations (operations running for more than 4 hours)
|
||||
staleOperationTimeout := 4 * time.Hour
|
||||
removedOperations := 0
|
||||
if mm.scanner != nil && mm.scanner.integration != nil {
|
||||
pendingOps := mm.scanner.integration.GetPendingOperations()
|
||||
if pendingOps != nil {
|
||||
removedOperations = pendingOps.CleanupStaleOperations(staleOperationTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
if removedTasks > 0 || removedWorkers > 0 || removedOperations > 0 {
|
||||
glog.V(1).Infof("Cleanup completed: removed %d old tasks, %d stale workers, and %d stale operations",
|
||||
removedTasks, removedWorkers, removedOperations)
|
||||
}
|
||||
}
|
||||
|
||||
// GetQueue returns the maintenance queue
|
||||
func (mm *MaintenanceManager) GetQueue() *MaintenanceQueue {
|
||||
return mm.queue
|
||||
}
|
||||
|
||||
// GetConfig returns the maintenance configuration
|
||||
func (mm *MaintenanceManager) GetConfig() *MaintenanceConfig {
|
||||
return mm.config
|
||||
}
|
||||
|
||||
// GetStats returns maintenance statistics
|
||||
func (mm *MaintenanceManager) GetStats() *MaintenanceStats {
|
||||
stats := mm.queue.GetStats()
|
||||
|
||||
mm.mutex.RLock()
|
||||
defer mm.mutex.RUnlock()
|
||||
|
||||
stats.LastScanTime = time.Now() // Would need to track this properly
|
||||
|
||||
// Calculate next scan time based on current error state
|
||||
scanInterval := time.Duration(mm.config.ScanIntervalSeconds) * time.Second
|
||||
nextScanInterval := scanInterval
|
||||
if mm.errorCount > 0 {
|
||||
nextScanInterval = mm.backoffDelay
|
||||
maxInterval := scanInterval * 10
|
||||
if nextScanInterval > maxInterval {
|
||||
nextScanInterval = maxInterval
|
||||
}
|
||||
}
|
||||
stats.NextScanTime = time.Now().Add(nextScanInterval)
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// ReloadTaskConfigurations reloads task configurations from the current policy
|
||||
func (mm *MaintenanceManager) ReloadTaskConfigurations() error {
|
||||
mm.mutex.Lock()
|
||||
defer mm.mutex.Unlock()
|
||||
|
||||
// Trigger configuration reload in the integration layer
|
||||
if mm.scanner != nil && mm.scanner.integration != nil {
|
||||
mm.scanner.integration.ConfigureTasksFromPolicy()
|
||||
glog.V(1).Infof("Task configurations reloaded from policy")
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("integration not available for configuration reload")
|
||||
}
|
||||
|
||||
// GetErrorState returns the current error state for monitoring
|
||||
func (mm *MaintenanceManager) GetErrorState() (errorCount int, lastError error, backoffDelay time.Duration) {
|
||||
mm.mutex.RLock()
|
||||
defer mm.mutex.RUnlock()
|
||||
return mm.errorCount, mm.lastError, mm.backoffDelay
|
||||
}
|
||||
|
||||
// GetTasks returns tasks with filtering
|
||||
func (mm *MaintenanceManager) GetTasks(status MaintenanceTaskStatus, taskType MaintenanceTaskType, limit int) []*MaintenanceTask {
|
||||
return mm.queue.GetTasks(status, taskType, limit)
|
||||
}
|
||||
|
||||
// GetWorkers returns all registered workers
|
||||
func (mm *MaintenanceManager) GetWorkers() []*MaintenanceWorker {
|
||||
return mm.queue.GetWorkers()
|
||||
}
|
||||
|
||||
// GetWorkerSlots returns used/max task slots per worker ID.
|
||||
func (mm *MaintenanceManager) GetWorkerSlots() map[string]WorkerSlots {
|
||||
return mm.queue.GetWorkerSlots()
|
||||
}
|
||||
|
||||
// TriggerScan manually triggers a maintenance scan
|
||||
func (mm *MaintenanceManager) TriggerScan() error {
|
||||
return mm.triggerScanInternal(true)
|
||||
}
|
||||
|
||||
// triggerScanInternal handles both manual and automatic scan triggers
|
||||
func (mm *MaintenanceManager) triggerScanInternal(isManual bool) error {
|
||||
// running and scanInProgress are checked under one lock so a Stop that lands
|
||||
// between the two checks cannot leave a scan running after shutdown.
|
||||
mm.mutex.Lock()
|
||||
if !mm.running {
|
||||
mm.mutex.Unlock()
|
||||
return fmt.Errorf("maintenance manager is not running")
|
||||
}
|
||||
|
||||
// Prevent multiple concurrent scans
|
||||
if mm.scanInProgress {
|
||||
mm.mutex.Unlock()
|
||||
if isManual {
|
||||
glog.V(1).Infof("Manual scan already in progress, ignoring trigger request")
|
||||
} else {
|
||||
glog.V(2).Infof("Automatic scan already in progress, ignoring scheduled scan")
|
||||
}
|
||||
return fmt.Errorf("scan already in progress")
|
||||
}
|
||||
mm.scanInProgress = true
|
||||
mm.mutex.Unlock()
|
||||
|
||||
go mm.performScan()
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateConfig updates the maintenance configuration
|
||||
func (mm *MaintenanceManager) UpdateConfig(config *MaintenanceConfig) error {
|
||||
if config == nil {
|
||||
return fmt.Errorf("config cannot be nil")
|
||||
}
|
||||
|
||||
mm.config = config
|
||||
mm.queue.policy = config.Policy
|
||||
mm.scanner.policy = config.Policy
|
||||
|
||||
// Propagate global policy changes to individual task configuration files
|
||||
if config.Policy != nil {
|
||||
mm.saveTaskConfigsFromPolicy(config.Policy)
|
||||
}
|
||||
|
||||
// The integration holds its own reference to the policy and is what pushes the
|
||||
// enabled flag and the concurrency limit into the registered detectors and
|
||||
// schedulers. Without this the queue and the scanner saw the new policy while the
|
||||
// detectors kept scanning under the old one until the next restart.
|
||||
if mm.scanner != nil && mm.scanner.integration != nil {
|
||||
mm.scanner.integration.SetPolicy(config.Policy)
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Maintenance configuration updated")
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveTaskConfigsFromPolicy propagates global policy settings to separate task configuration files
|
||||
func (mm *MaintenanceManager) saveTaskConfigsFromPolicy(policy *worker_pb.MaintenancePolicy) {
|
||||
if mm.queue.persistence == nil || policy == nil {
|
||||
return
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Propagating maintenance policy changes to separate task configs")
|
||||
for taskType, taskPolicy := range policy.TaskPolicies {
|
||||
if err := mm.queue.persistence.SaveTaskPolicy(taskType, taskPolicy); err != nil {
|
||||
glog.Errorf("Failed to save task policy for %s: %v", taskType, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CancelTask cancels a pending task
|
||||
func (mm *MaintenanceManager) CancelTask(taskID string) error {
|
||||
mm.queue.mutex.Lock()
|
||||
defer mm.queue.mutex.Unlock()
|
||||
|
||||
task, exists := mm.queue.tasks[taskID]
|
||||
if !exists {
|
||||
return fmt.Errorf("task %s not found", taskID)
|
||||
}
|
||||
|
||||
if task.Status == TaskStatusPending {
|
||||
task.Status = TaskStatusCancelled
|
||||
task.CompletedAt = &[]time.Time{time.Now()}[0]
|
||||
|
||||
// Remove from pending tasks
|
||||
for i, pendingTask := range mm.queue.pendingTasks {
|
||||
if pendingTask.ID == taskID {
|
||||
mm.queue.pendingTasks = append(mm.queue.pendingTasks[:i], mm.queue.pendingTasks[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Notify ActiveTopology to release capacity
|
||||
if mm.scanner != nil && mm.scanner.integration != nil {
|
||||
if at := mm.scanner.integration.GetActiveTopology(); at != nil {
|
||||
_ = at.CompleteTask(taskID)
|
||||
}
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Cancelled task %s", taskID)
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("task %s cannot be cancelled (status: %s)", taskID, task.Status)
|
||||
}
|
||||
|
||||
// RegisterWorker registers a new worker
|
||||
func (mm *MaintenanceManager) RegisterWorker(worker *MaintenanceWorker) {
|
||||
mm.queue.RegisterWorker(worker)
|
||||
}
|
||||
|
||||
// GetNextTask returns the next task for a worker
|
||||
func (mm *MaintenanceManager) GetNextTask(workerID string, capabilities []MaintenanceTaskType) *MaintenanceTask {
|
||||
return mm.queue.GetNextTask(workerID, capabilities)
|
||||
}
|
||||
|
||||
// CompleteTask marks a task as completed
|
||||
func (mm *MaintenanceManager) CompleteTask(taskID string, error string) {
|
||||
mm.queue.CompleteTask(taskID, error)
|
||||
}
|
||||
|
||||
// UpdateTaskProgress updates task progress
|
||||
func (mm *MaintenanceManager) UpdateTaskProgress(taskID string, progress float64) {
|
||||
mm.queue.UpdateTaskProgress(taskID, progress)
|
||||
}
|
||||
|
||||
// UpdateWorkerHeartbeat updates worker heartbeat
|
||||
func (mm *MaintenanceManager) UpdateWorkerHeartbeat(workerID string) {
|
||||
mm.queue.UpdateWorkerHeartbeat(workerID)
|
||||
}
|
||||
140
weed/admin/maintenance/maintenance_manager_test.go
Normal file
140
weed/admin/maintenance/maintenance_manager_test.go
Normal file
@@ -0,0 +1,140 @@
|
||||
package maintenance
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMaintenanceManager_ErrorHandling(t *testing.T) {
|
||||
config := DefaultMaintenanceConfig()
|
||||
config.ScanIntervalSeconds = 1 // Short interval for testing (1 second)
|
||||
|
||||
manager := NewMaintenanceManager(nil, config, nil)
|
||||
|
||||
// Test initial state
|
||||
if manager.errorCount != 0 {
|
||||
t.Errorf("Expected initial error count to be 0, got %d", manager.errorCount)
|
||||
}
|
||||
|
||||
if manager.backoffDelay != time.Second {
|
||||
t.Errorf("Expected initial backoff delay to be 1s, got %v", manager.backoffDelay)
|
||||
}
|
||||
|
||||
// Test error handling
|
||||
err := errors.New("dial tcp [::1]:19333: connect: connection refused")
|
||||
manager.handleScanError(err)
|
||||
|
||||
if manager.errorCount != 1 {
|
||||
t.Errorf("Expected error count to be 1, got %d", manager.errorCount)
|
||||
}
|
||||
|
||||
if manager.lastError != err {
|
||||
t.Errorf("Expected last error to be set")
|
||||
}
|
||||
|
||||
// Test exponential backoff
|
||||
initialDelay := manager.backoffDelay
|
||||
manager.handleScanError(err)
|
||||
|
||||
if manager.backoffDelay != initialDelay*2 {
|
||||
t.Errorf("Expected backoff delay to double, got %v", manager.backoffDelay)
|
||||
}
|
||||
|
||||
if manager.errorCount != 2 {
|
||||
t.Errorf("Expected error count to be 2, got %d", manager.errorCount)
|
||||
}
|
||||
|
||||
// Test backoff cap
|
||||
for i := 0; i < 10; i++ {
|
||||
manager.handleScanError(err)
|
||||
}
|
||||
|
||||
if manager.backoffDelay > 5*time.Minute {
|
||||
t.Errorf("Expected backoff delay to be capped at 5 minutes, got %v", manager.backoffDelay)
|
||||
}
|
||||
|
||||
// Test error reset
|
||||
manager.resetErrorTracking()
|
||||
|
||||
if manager.errorCount != 0 {
|
||||
t.Errorf("Expected error count to be reset to 0, got %d", manager.errorCount)
|
||||
}
|
||||
|
||||
if manager.backoffDelay != time.Second {
|
||||
t.Errorf("Expected backoff delay to be reset to 1s, got %v", manager.backoffDelay)
|
||||
}
|
||||
|
||||
if manager.lastError != nil {
|
||||
t.Errorf("Expected last error to be reset to nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsConnectionError(t *testing.T) {
|
||||
tests := []struct {
|
||||
err error
|
||||
expected bool
|
||||
}{
|
||||
{nil, false},
|
||||
{errors.New("connection refused"), true},
|
||||
{errors.New("dial tcp [::1]:19333: connect: connection refused"), true},
|
||||
{errors.New("connection error: desc = \"transport: Error while dialing\""), true},
|
||||
{errors.New("connection timeout"), true},
|
||||
{errors.New("no route to host"), true},
|
||||
{errors.New("network unreachable"), true},
|
||||
{errors.New("some other error"), false},
|
||||
{errors.New("invalid argument"), false},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
result := isConnectionError(test.err)
|
||||
if result != test.expected {
|
||||
t.Errorf("For error %v, expected %v, got %v", test.err, test.expected, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintenanceManager_GetErrorState(t *testing.T) {
|
||||
config := DefaultMaintenanceConfig()
|
||||
manager := NewMaintenanceManager(nil, config, nil)
|
||||
|
||||
// Test initial state
|
||||
errorCount, lastError, backoffDelay := manager.GetErrorState()
|
||||
if errorCount != 0 || lastError != nil || backoffDelay != time.Second {
|
||||
t.Errorf("Expected initial state to be clean")
|
||||
}
|
||||
|
||||
// Add some errors
|
||||
err := errors.New("test error")
|
||||
manager.handleScanError(err)
|
||||
manager.handleScanError(err)
|
||||
|
||||
errorCount, lastError, backoffDelay = manager.GetErrorState()
|
||||
if errorCount != 2 || lastError != err || backoffDelay != 2*time.Second {
|
||||
t.Errorf("Expected error state to be tracked correctly: count=%d, err=%v, delay=%v",
|
||||
errorCount, lastError, backoffDelay)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintenanceManager_LogThrottling(t *testing.T) {
|
||||
config := DefaultMaintenanceConfig()
|
||||
manager := NewMaintenanceManager(nil, config, nil)
|
||||
|
||||
// This is a basic test to ensure the error handling doesn't panic
|
||||
// In practice, you'd want to capture log output to verify throttling
|
||||
err := errors.New("test error")
|
||||
|
||||
// Generate many errors to test throttling
|
||||
for i := 0; i < 25; i++ {
|
||||
manager.handleScanError(err)
|
||||
}
|
||||
|
||||
// Should not panic and should have capped backoff
|
||||
if manager.backoffDelay > 5*time.Minute {
|
||||
t.Errorf("Expected backoff to be capped at 5 minutes")
|
||||
}
|
||||
|
||||
if manager.errorCount != 25 {
|
||||
t.Errorf("Expected error count to be 25, got %d", manager.errorCount)
|
||||
}
|
||||
}
|
||||
133
weed/admin/maintenance/maintenance_policy_test.go
Normal file
133
weed/admin/maintenance/maintenance_policy_test.go
Normal file
@@ -0,0 +1,133 @@
|
||||
package maintenance
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
)
|
||||
|
||||
// stubConfigPersistence implements the LoadXTaskPolicy accessors that the task config loaders
|
||||
// type-assert on. The real implementation is *dash.ConfigPersistence, which this package cannot
|
||||
// import: weed/admin/dash already imports weed/admin/maintenance, so the dependency only runs one
|
||||
// way and the persistence argument has to stay duck-typed.
|
||||
type stubConfigPersistence struct {
|
||||
vacuum *worker_pb.TaskPolicy
|
||||
ec *worker_pb.TaskPolicy
|
||||
balance *worker_pb.TaskPolicy
|
||||
ecBalance *worker_pb.TaskPolicy
|
||||
}
|
||||
|
||||
func (s *stubConfigPersistence) LoadVacuumTaskPolicy() (*worker_pb.TaskPolicy, error) {
|
||||
return s.vacuum, nil
|
||||
}
|
||||
|
||||
func (s *stubConfigPersistence) LoadErasureCodingTaskPolicy() (*worker_pb.TaskPolicy, error) {
|
||||
return s.ec, nil
|
||||
}
|
||||
|
||||
func (s *stubConfigPersistence) LoadBalanceTaskPolicy() (*worker_pb.TaskPolicy, error) {
|
||||
return s.balance, nil
|
||||
}
|
||||
|
||||
func (s *stubConfigPersistence) LoadEcBalanceTaskPolicy() (*worker_pb.TaskPolicy, error) {
|
||||
return s.ecBalance, nil
|
||||
}
|
||||
|
||||
func disabledStub() *stubConfigPersistence {
|
||||
return &stubConfigPersistence{
|
||||
vacuum: &worker_pb.TaskPolicy{
|
||||
Enabled: false,
|
||||
MaxConcurrent: 2,
|
||||
RepeatIntervalSeconds: 2 * 3600,
|
||||
TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
|
||||
VacuumConfig: &worker_pb.VacuumTaskConfig{GarbageThreshold: 0.3, MinVolumeAgeHours: 24},
|
||||
},
|
||||
},
|
||||
ec: &worker_pb.TaskPolicy{
|
||||
Enabled: false,
|
||||
MaxConcurrent: 1,
|
||||
RepeatIntervalSeconds: 3600,
|
||||
TaskConfig: &worker_pb.TaskPolicy_ErasureCodingConfig{
|
||||
ErasureCodingConfig: &worker_pb.ErasureCodingTaskConfig{FullnessRatio: 0.95, QuietForSeconds: 3600, MinVolumeSizeMb: 30},
|
||||
},
|
||||
},
|
||||
balance: &worker_pb.TaskPolicy{
|
||||
Enabled: false,
|
||||
MaxConcurrent: 1,
|
||||
RepeatIntervalSeconds: 30 * 60,
|
||||
TaskConfig: &worker_pb.TaskPolicy_BalanceConfig{
|
||||
BalanceConfig: &worker_pb.BalanceTaskConfig{ImbalanceThreshold: 0.2, MinServerCount: 7},
|
||||
},
|
||||
},
|
||||
ecBalance: &worker_pb.TaskPolicy{
|
||||
Enabled: false,
|
||||
MaxConcurrent: 1,
|
||||
RepeatIntervalSeconds: 60 * 60,
|
||||
TaskConfig: &worker_pb.TaskPolicy_EcBalanceConfig{
|
||||
EcBalanceConfig: &worker_pb.EcBalanceTaskConfig{ImbalanceThreshold: 0.2, MinServerCount: 5},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildPolicyFromTaskConfigsUsesPersistence covers a reported bug: the persistence argument used to be a
|
||||
// literal nil, which no type assertion can satisfy, so a task disabled on disk came back enabled.
|
||||
func TestBuildPolicyFromTaskConfigsUsesPersistence(t *testing.T) {
|
||||
policy := BuildPolicyFromTaskConfigs(disabledStub())
|
||||
|
||||
for _, taskType := range []string{"vacuum", "erasure_coding", "balance", "ec_balance"} {
|
||||
taskPolicy := policy.TaskPolicies[taskType]
|
||||
if taskPolicy == nil {
|
||||
t.Fatalf("no %s task policy built", taskType)
|
||||
}
|
||||
if taskPolicy.Enabled {
|
||||
t.Errorf("%s enabled = true, want false from the persisted config", taskType)
|
||||
}
|
||||
}
|
||||
|
||||
if got := policy.TaskPolicies["balance"].GetBalanceConfig().GetMinServerCount(); got != 7 {
|
||||
t.Errorf("balance min server count = %d, want persisted 7", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildPolicyFromTaskConfigsWithoutPersistence keeps the documented fallback: with no config
|
||||
// store there is nothing to read, so the compiled-in defaults apply.
|
||||
func TestBuildPolicyFromTaskConfigsWithoutPersistence(t *testing.T) {
|
||||
policy := BuildPolicyFromTaskConfigs(nil)
|
||||
|
||||
for _, taskType := range []string{"vacuum", "erasure_coding", "balance", "ec_balance"} {
|
||||
taskPolicy := policy.TaskPolicies[taskType]
|
||||
if taskPolicy == nil {
|
||||
t.Fatalf("no %s task policy built", taskType)
|
||||
}
|
||||
if !taskPolicy.Enabled {
|
||||
t.Errorf("%s enabled = false, want the compiled-in default true", taskType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewMaintenanceManagerUsesPersistenceForPolicyFallback covers the path the admin server takes
|
||||
// when no maintenance.pb has been written yet: the config carries no policy, so the manager builds
|
||||
// one itself and must read the persisted task configs to do it.
|
||||
func TestNewMaintenanceManagerUsesPersistenceForPolicyFallback(t *testing.T) {
|
||||
config := DefaultMaintenanceConfig()
|
||||
if config.Policy != nil {
|
||||
t.Fatal("default config unexpectedly carries a policy, test no longer covers the fallback")
|
||||
}
|
||||
|
||||
manager := NewMaintenanceManager(nil, config, disabledStub())
|
||||
|
||||
policy := manager.queue.policy
|
||||
if policy == nil {
|
||||
t.Fatal("queue policy is nil")
|
||||
}
|
||||
if IsTaskEnabled(policy, MaintenanceTaskType("balance")) {
|
||||
t.Error("balance enabled = true, want false from the persisted config")
|
||||
}
|
||||
if IsTaskEnabled(policy, MaintenanceTaskType("vacuum")) {
|
||||
t.Error("vacuum enabled = true, want false from the persisted config")
|
||||
}
|
||||
if manager.scanner.policy != policy {
|
||||
t.Error("scanner and queue disagree on the policy")
|
||||
}
|
||||
}
|
||||
209
weed/admin/maintenance/maintenance_policy_wiring_test.go
Normal file
209
weed/admin/maintenance/maintenance_policy_wiring_test.go
Normal file
@@ -0,0 +1,209 @@
|
||||
package maintenance
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// The detectors and schedulers live in a process-global registry, so these tests restore
|
||||
// whatever they change. Otherwise a test that disables a task leaks that state into every
|
||||
// later test in the package.
|
||||
func snapshotDetectorState(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
registry := tasks.GetGlobalTypesRegistry()
|
||||
enabled := make(map[types.TaskType]bool)
|
||||
maxConcurrent := make(map[types.TaskType]int)
|
||||
for taskType, detector := range registry.GetAllDetectors() {
|
||||
enabled[taskType] = detector.IsEnabled()
|
||||
}
|
||||
for taskType, scheduler := range registry.GetAllSchedulers() {
|
||||
maxConcurrent[taskType] = scheduler.GetMaxConcurrent()
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
for taskType, detector := range registry.GetAllDetectors() {
|
||||
if setter, ok := detector.(interface{ SetEnabled(bool) }); ok {
|
||||
setter.SetEnabled(enabled[taskType])
|
||||
}
|
||||
}
|
||||
for taskType, scheduler := range registry.GetAllSchedulers() {
|
||||
if setter, ok := scheduler.(interface{ SetMaxConcurrent(int) }); ok {
|
||||
setter.SetMaxConcurrent(maxConcurrent[taskType])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestPolicyReachesRegisteredDetectors is the regression test for the half of the scan-flood bug
|
||||
// that a corrected policy alone did not fix: MaintenanceIntegration pushes the policy into
|
||||
// detectors and schedulers through interface{ SetEnabled(bool) } and
|
||||
// interface{ SetMaxConcurrent(int) } type assertions, but every task is backed by
|
||||
// base.GenericDetector/base.GenericScheduler, which implemented neither. The assertions
|
||||
// failed silently for every task on every startup, so the policy never reached
|
||||
// detector.IsEnabled() - which is what ScanWithTaskDetectors gates scanning on.
|
||||
func TestPolicyReachesRegisteredDetectors(t *testing.T) {
|
||||
snapshotDetectorState(t)
|
||||
|
||||
registry := tasks.GetGlobalTypesRegistry()
|
||||
if len(registry.GetAllDetectors()) == 0 {
|
||||
t.Fatal("no detectors registered, the test cannot prove anything")
|
||||
}
|
||||
|
||||
// Enable everything first so the disabling below cannot pass by accident.
|
||||
policy := policyWithAllTasks(t, true)
|
||||
NewMaintenanceIntegration(NewMaintenanceQueue(policy), policy)
|
||||
|
||||
for taskType, detector := range registry.GetAllDetectors() {
|
||||
if !detector.IsEnabled() {
|
||||
t.Fatalf("detector %s is disabled after an all-enabled policy was applied", taskType)
|
||||
}
|
||||
}
|
||||
|
||||
// Now disable everything through the policy and check it lands on the detectors.
|
||||
policy = policyWithAllTasks(t, false)
|
||||
NewMaintenanceIntegration(NewMaintenanceQueue(policy), policy)
|
||||
|
||||
for taskType, detector := range registry.GetAllDetectors() {
|
||||
if detector.IsEnabled() {
|
||||
t.Errorf("detector %s still reports enabled after the policy disabled it; "+
|
||||
"the policy is not reaching the flag ScanWithTaskDetectors gates on", taskType)
|
||||
}
|
||||
}
|
||||
for taskType, scheduler := range registry.GetAllSchedulers() {
|
||||
if scheduler.IsEnabled() {
|
||||
t.Errorf("scheduler %s still reports enabled after the policy disabled it", taskType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPolicyMaxConcurrentReachesSchedulers covers the SetMaxConcurrent half of the same
|
||||
// wiring. GetMaxConcurrent is what MaintenanceQueue.getMaxConcurrentForTaskType asks before
|
||||
// starting another task of a type.
|
||||
func TestPolicyMaxConcurrentReachesSchedulers(t *testing.T) {
|
||||
snapshotDetectorState(t)
|
||||
|
||||
const wantMaxConcurrent = 7
|
||||
|
||||
policy := policyWithAllTasks(t, true)
|
||||
for _, taskPolicy := range policy.TaskPolicies {
|
||||
taskPolicy.MaxConcurrent = wantMaxConcurrent
|
||||
}
|
||||
NewMaintenanceIntegration(NewMaintenanceQueue(policy), policy)
|
||||
|
||||
registry := tasks.GetGlobalTypesRegistry()
|
||||
for taskType, scheduler := range registry.GetAllSchedulers() {
|
||||
if _, covered := policy.TaskPolicies[string(taskType)]; !covered {
|
||||
continue
|
||||
}
|
||||
if got := scheduler.GetMaxConcurrent(); got != wantMaxConcurrent {
|
||||
t.Errorf("scheduler %s max concurrent = %d, want %d from the policy", taskType, got, wantMaxConcurrent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestPolicyWithoutEntryLeavesTaskAlone guards the direction that would have been a silent
|
||||
// outage: IsTaskEnabled reports false for a task type the policy does not list, so applying
|
||||
// it unconditionally would disable every task the policy has no entry for.
|
||||
func TestPolicyWithoutEntryLeavesTaskAlone(t *testing.T) {
|
||||
snapshotDetectorState(t)
|
||||
|
||||
registry := tasks.GetGlobalTypesRegistry()
|
||||
|
||||
// Start from a policy that enables everything.
|
||||
enabling := policyWithAllTasks(t, true)
|
||||
NewMaintenanceIntegration(NewMaintenanceQueue(enabling), enabling)
|
||||
|
||||
// An empty policy has an entry for nothing at all.
|
||||
empty := &MaintenancePolicy{TaskPolicies: make(map[string]*worker_pb.TaskPolicy)}
|
||||
NewMaintenanceIntegration(NewMaintenanceQueue(empty), empty)
|
||||
|
||||
for taskType, detector := range registry.GetAllDetectors() {
|
||||
if !detector.IsEnabled() {
|
||||
t.Errorf("detector %s was disabled by a policy that has no entry for it; "+
|
||||
"a missing entry means no opinion, not disabled", taskType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildPolicyCoversEveryRegisteredTask keeps BuildPolicyFromTaskConfigs and the task
|
||||
// registry in step. A registered task with no policy entry has no enabled flag and no
|
||||
// concurrency limit of its own, which is the state ec_balance was in.
|
||||
func TestBuildPolicyCoversEveryRegisteredTask(t *testing.T) {
|
||||
policy := BuildPolicyFromTaskConfigs(nil)
|
||||
|
||||
for taskType := range tasks.GetGlobalTypesRegistry().GetAllDetectors() {
|
||||
if _, ok := policy.TaskPolicies[string(taskType)]; !ok {
|
||||
t.Errorf("task %s is registered as a detector but BuildPolicyFromTaskConfigs "+
|
||||
"builds no policy entry for it, so IsTaskEnabled reports false for it", taskType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildPolicyIncludesEcBalance pins the specific gap above.
|
||||
func TestBuildPolicyIncludesEcBalance(t *testing.T) {
|
||||
policy := BuildPolicyFromTaskConfigs(nil)
|
||||
|
||||
ecBalance := policy.TaskPolicies[string(types.TaskTypeECBalance)]
|
||||
if ecBalance == nil {
|
||||
t.Fatal("no ec_balance entry in the built maintenance policy")
|
||||
}
|
||||
if !IsTaskEnabled(policy, MaintenanceTaskType(types.TaskTypeECBalance)) {
|
||||
t.Error("ec_balance reports disabled with no persisted config, want the compiled-in default of enabled")
|
||||
}
|
||||
if ecBalance.GetEcBalanceConfig() == nil {
|
||||
t.Error("ec_balance policy entry carries no EcBalanceConfig")
|
||||
}
|
||||
}
|
||||
|
||||
// policyWithAllTasks builds a policy that has an entry for every registered task type,
|
||||
// all sharing the same enabled flag.
|
||||
func policyWithAllTasks(t *testing.T, enabled bool) *MaintenancePolicy {
|
||||
t.Helper()
|
||||
|
||||
policy := &MaintenancePolicy{
|
||||
GlobalMaxConcurrent: 4,
|
||||
TaskPolicies: make(map[string]*worker_pb.TaskPolicy),
|
||||
}
|
||||
for taskType := range tasks.GetGlobalTypesRegistry().GetAllDetectors() {
|
||||
policy.TaskPolicies[string(taskType)] = &worker_pb.TaskPolicy{
|
||||
Enabled: enabled,
|
||||
MaxConcurrent: 1,
|
||||
RepeatIntervalSeconds: 3600,
|
||||
}
|
||||
}
|
||||
return policy
|
||||
}
|
||||
|
||||
// TestPolicyHelpersTolerateNilPolicy pins the nil handling in the exported policy helpers.
|
||||
// MaintenanceConfig.Policy is nil until something builds one, and UpdateConfig accepts a
|
||||
// config that has none, so these are reachable with a nil policy - GetTaskPolicy used to
|
||||
// dereference it and take the admin process down.
|
||||
func TestPolicyHelpersTolerateNilPolicy(t *testing.T) {
|
||||
const taskType = MaintenanceTaskType("balance")
|
||||
|
||||
if got := GetTaskPolicy(nil, taskType); got != nil {
|
||||
t.Errorf("GetTaskPolicy(nil) = %v, want nil", got)
|
||||
}
|
||||
if IsTaskEnabled(nil, taskType) {
|
||||
t.Error("IsTaskEnabled(nil) = true, want false")
|
||||
}
|
||||
if got := GetMaxConcurrent(nil, taskType); got != 1 {
|
||||
t.Errorf("GetMaxConcurrent(nil) = %d, want the safe default 1", got)
|
||||
}
|
||||
if got := GetRepeatInterval(nil, taskType); got != 0 {
|
||||
t.Errorf("GetRepeatInterval(nil) = %d, want 0 so callers fall back to their own default", got)
|
||||
}
|
||||
|
||||
// A policy that exists but lists nothing must behave the same way.
|
||||
empty := &MaintenancePolicy{}
|
||||
if got := GetTaskPolicy(empty, taskType); got != nil {
|
||||
t.Errorf("GetTaskPolicy(empty) = %v, want nil", got)
|
||||
}
|
||||
if IsTaskEnabled(empty, taskType) {
|
||||
t.Error("IsTaskEnabled(empty) = true, want false")
|
||||
}
|
||||
}
|
||||
1268
weed/admin/maintenance/maintenance_queue.go
Normal file
1268
weed/admin/maintenance/maintenance_queue.go
Normal file
File diff suppressed because it is too large
Load Diff
1097
weed/admin/maintenance/maintenance_queue_test.go
Normal file
1097
weed/admin/maintenance/maintenance_queue_test.go
Normal file
File diff suppressed because it is too large
Load Diff
168
weed/admin/maintenance/maintenance_scan_cadence_test.go
Normal file
168
weed/admin/maintenance/maintenance_scan_cadence_test.go
Normal file
@@ -0,0 +1,168 @@
|
||||
package maintenance
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
)
|
||||
|
||||
// scanCadenceClient is a minimal AdminClient whose first call blocks until the test
|
||||
// releases it. Blocking the first scan is what makes the cadence test deterministic:
|
||||
// the scan loop evaluates the error state right after triggering a scan, and that scan
|
||||
// runs in its own goroutine, so without the block the loop could observe the error
|
||||
// counter either before or after the scan finished.
|
||||
type scanCadenceClient struct {
|
||||
mu sync.Mutex
|
||||
calls int
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (c *scanCadenceClient) WithMasterClient(fn func(client master_pb.SeaweedClient) error) error {
|
||||
c.mu.Lock()
|
||||
c.calls++
|
||||
first := c.calls == 1
|
||||
c.mu.Unlock()
|
||||
|
||||
if first {
|
||||
<-c.release
|
||||
}
|
||||
|
||||
// Returning nil without invoking fn yields an empty, successful scan.
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *scanCadenceClient) callCount() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.calls
|
||||
}
|
||||
|
||||
// TestScanLoopRestoresIntervalAfterBackoff is the regression test for the scan flood. The loop shortens its ticker to the
|
||||
// error backoff delay after a failed scan. It used to compare the target interval against
|
||||
// the *configured* scan interval rather than against the interval the ticker was actually
|
||||
// running at, so once the errors stopped the comparison came out false and the ticker was
|
||||
// never restored: one transient failure pinned the scanner to one scan per second for as
|
||||
// long as the process lived.
|
||||
//
|
||||
// The test seeds the error state, lets the loop drop into the 1s backoff, then lets a scan
|
||||
// succeed and asserts the loop goes back to the configured 3s cadence instead of keeping
|
||||
// the 1s one.
|
||||
func TestScanLoopRestoresIntervalAfterBackoff(t *testing.T) {
|
||||
const baseInterval = 3 * time.Second
|
||||
|
||||
client := &scanCadenceClient{release: make(chan struct{})}
|
||||
|
||||
config := DefaultMaintenanceConfig()
|
||||
config.ScanIntervalSeconds = int32(baseInterval / time.Second)
|
||||
manager := NewMaintenanceManager(client, config, nil)
|
||||
|
||||
// Seed a failed scan so the very first cadence decision drops to the backoff delay.
|
||||
// backoffDelay is what getScanInterval returns while errorCount > 0.
|
||||
manager.mutex.Lock()
|
||||
manager.errorCount = 1
|
||||
manager.backoffDelay = time.Second
|
||||
manager.running = true
|
||||
manager.mutex.Unlock()
|
||||
|
||||
go manager.scanLoop()
|
||||
defer manager.Stop()
|
||||
|
||||
// First tick at ~3s: the scan blocks in the client, so the loop still sees errorCount == 1
|
||||
// and switches the ticker to the 1s backoff.
|
||||
deadline := time.Now().Add(baseInterval + 2*time.Second)
|
||||
for client.callCount() == 0 {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("scan loop never triggered its first scan")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
|
||||
// Let the blocked scan complete successfully, which clears the error state.
|
||||
close(client.release)
|
||||
|
||||
// Wait for the error state to clear so the next cadence decision is unambiguous.
|
||||
deadline = time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
errorCount, _, _ := manager.GetErrorState()
|
||||
if errorCount == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("error tracking never reset, errorCount=%d", errorCount)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
|
||||
// The ticker is at 1s now. Wait for the next tick, where the loop must notice the
|
||||
// recovery and restore the 3s cadence.
|
||||
time.Sleep(1500 * time.Millisecond)
|
||||
before := client.callCount()
|
||||
|
||||
// Observe a window that a 1s cadence would fill with scans and a 3s cadence would not.
|
||||
const window = 5 * time.Second
|
||||
time.Sleep(window)
|
||||
callsInWindow := client.callCount() - before
|
||||
|
||||
// One scan makes two master-client calls. 3s cadence: at most 2 scans (4 calls) in 5s,
|
||||
// even if a stall shifts the window. 1s cadence: about 5 scans (10 calls).
|
||||
if callsInWindow > 4 {
|
||||
t.Errorf("scan loop made %d master-client calls in %v after recovering from a failed scan; "+
|
||||
"the ticker was left at the %v backoff instead of returning to the configured %v",
|
||||
callsInWindow, window, time.Second, baseInterval)
|
||||
}
|
||||
if callsInWindow == 0 {
|
||||
t.Errorf("scan loop ran no scans in %v, expected the %v cadence to fire at least once", window, baseInterval)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanLoopEntersBackoffOnError checks the other half of the cadence logic: a failing
|
||||
// scan still has to shorten the ticker, so the fix above did not simply pin the loop to
|
||||
// the configured interval.
|
||||
func TestScanLoopEntersBackoffOnError(t *testing.T) {
|
||||
config := DefaultMaintenanceConfig()
|
||||
config.ScanIntervalSeconds = 600
|
||||
manager := NewMaintenanceManager(nil, config, nil)
|
||||
|
||||
baseInterval := time.Duration(config.ScanIntervalSeconds) * time.Second
|
||||
|
||||
if got := manager.getScanInterval(baseInterval); got != baseInterval {
|
||||
t.Errorf("healthy scan interval = %v, want the configured %v", got, baseInterval)
|
||||
}
|
||||
|
||||
manager.mutex.Lock()
|
||||
manager.errorCount = 1
|
||||
manager.backoffDelay = time.Second
|
||||
manager.mutex.Unlock()
|
||||
|
||||
if got := manager.getScanInterval(baseInterval); got != time.Second {
|
||||
t.Errorf("scan interval while failing = %v, want the %v backoff", got, time.Second)
|
||||
}
|
||||
|
||||
manager.mutex.Lock()
|
||||
manager.resetErrorTracking()
|
||||
manager.mutex.Unlock()
|
||||
|
||||
if got := manager.getScanInterval(baseInterval); got != baseInterval {
|
||||
t.Errorf("scan interval after recovery = %v, want the configured %v", got, baseInterval)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStopIsIdempotent guards the stop channel against a double close, which used to panic
|
||||
// when StopMaintenanceManager ran twice (for example on a shutdown path that also runs on
|
||||
// a signal handler).
|
||||
func TestStopIsIdempotent(t *testing.T) {
|
||||
manager := NewMaintenanceManager(nil, DefaultMaintenanceConfig(), nil)
|
||||
|
||||
manager.mutex.Lock()
|
||||
manager.running = true
|
||||
manager.mutex.Unlock()
|
||||
|
||||
manager.Stop()
|
||||
manager.Stop()
|
||||
|
||||
if manager.IsRunning() {
|
||||
t.Error("manager still reports running after Stop")
|
||||
}
|
||||
}
|
||||
224
weed/admin/maintenance/maintenance_scanner.go
Normal file
224
weed/admin/maintenance/maintenance_scanner.go
Normal file
@@ -0,0 +1,224 @@
|
||||
package maintenance
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// NewMaintenanceScanner creates a new maintenance scanner
|
||||
func NewMaintenanceScanner(adminClient AdminClient, policy *MaintenancePolicy, queue *MaintenanceQueue) *MaintenanceScanner {
|
||||
scanner := &MaintenanceScanner{
|
||||
adminClient: adminClient,
|
||||
policy: policy,
|
||||
queue: queue,
|
||||
lastScan: make(map[MaintenanceTaskType]time.Time),
|
||||
}
|
||||
|
||||
// Initialize integration
|
||||
scanner.integration = NewMaintenanceIntegration(queue, policy)
|
||||
|
||||
// Set up bidirectional relationship
|
||||
queue.SetIntegration(scanner.integration)
|
||||
|
||||
glog.V(1).Infof("Initialized maintenance scanner with task system")
|
||||
|
||||
return scanner
|
||||
}
|
||||
|
||||
// ScanForMaintenanceTasks analyzes the cluster and generates maintenance tasks
|
||||
func (ms *MaintenanceScanner) ScanForMaintenanceTasks() ([]*TaskDetectionResult, error) {
|
||||
// Get volume health metrics directly in task-system format, along with topology info
|
||||
taskMetrics, topologyInfo, err := ms.getVolumeHealthMetrics()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get volume health metrics: %w", err)
|
||||
}
|
||||
|
||||
// Use task system for all task types
|
||||
if ms.integration != nil {
|
||||
// Update topology information for complete cluster view (including empty servers)
|
||||
// This must happen before task detection to ensure EC placement can consider all servers
|
||||
if topologyInfo != nil {
|
||||
if err := ms.integration.UpdateTopologyInfo(topologyInfo); err != nil {
|
||||
glog.Errorf("Failed to update topology info for empty servers: %v", err)
|
||||
// Don't fail the scan - continue with just volume-bearing servers
|
||||
} else {
|
||||
glog.V(1).Infof("Updated topology info for complete cluster view including empty servers")
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh the master's default replication so detectors can use it as the
|
||||
// replica-placement fallback (matches the shell ec.balance default).
|
||||
ms.integration.SetDefaultReplicaPlacement(ms.getDefaultReplicaPlacement())
|
||||
|
||||
// Use task detection system with complete cluster information
|
||||
results, err := ms.integration.ScanWithTaskDetectors(taskMetrics)
|
||||
if err != nil {
|
||||
glog.Errorf("Task scanning failed: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Maintenance scan completed: found %d tasks", len(results))
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// No integration available
|
||||
glog.Warningf("No integration available, no tasks will be scheduled")
|
||||
return []*TaskDetectionResult{}, nil
|
||||
}
|
||||
|
||||
// getDefaultReplicaPlacement reads the master's configured default replication,
|
||||
// used by detectors as the replica-placement fallback. Returns "" on error so
|
||||
// detectors fall back to even spread rather than failing the scan.
|
||||
func (ms *MaintenanceScanner) getDefaultReplicaPlacement() string {
|
||||
var replicaPlacement string
|
||||
err := ms.adminClient.WithMasterClient(func(client master_pb.SeaweedClient) error {
|
||||
resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
replicaPlacement = resp.DefaultReplication
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(1).Infof("could not fetch master default replication: %v", err)
|
||||
return ""
|
||||
}
|
||||
return replicaPlacement
|
||||
}
|
||||
|
||||
// getVolumeHealthMetrics collects health information for all volumes.
|
||||
// Returns metrics in task-system format directly (no intermediate copy) and
|
||||
// the topology info for updating the active topology.
|
||||
func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*types.VolumeHealthMetrics, *master_pb.TopologyInfo, error) {
|
||||
var metrics []*types.VolumeHealthMetrics
|
||||
var topologyInfo *master_pb.TopologyInfo
|
||||
|
||||
glog.V(1).Infof("Collecting volume health metrics from master")
|
||||
err := ms.adminClient.WithMasterClient(func(client master_pb.SeaweedClient) error {
|
||||
|
||||
// Streamed, so the master never builds the whole listing to send it.
|
||||
resp, err := pb.CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.TopologyInfo == nil {
|
||||
glog.Warningf("No topology info received from master")
|
||||
return nil
|
||||
}
|
||||
|
||||
volumeSizeLimitBytes := uint64(resp.VolumeSizeLimitMb) * 1024 * 1024 // Convert MB to bytes
|
||||
|
||||
// Track node counts for summary logging (avoid accumulating full ID slices)
|
||||
var totalNodes, nodesWithVolumes, nodesWithoutVolumes int
|
||||
|
||||
for _, dc := range resp.TopologyInfo.DataCenterInfos {
|
||||
glog.V(3).Infof("Processing datacenter: %s", dc.Id)
|
||||
for _, rack := range dc.RackInfos {
|
||||
glog.V(3).Infof("Processing rack: %s in datacenter: %s", rack.Id, dc.Id)
|
||||
for _, node := range rack.DataNodeInfos {
|
||||
totalNodes++
|
||||
glog.V(3).Infof("Found volume server in topology: %s (disks: %d)", node.Id, len(node.DiskInfos))
|
||||
|
||||
hasVolumes := false
|
||||
// Process each disk on this node
|
||||
for diskType, diskInfo := range node.DiskInfos {
|
||||
if len(diskInfo.VolumeInfos) > 0 {
|
||||
hasVolumes = true
|
||||
glog.V(3).Infof("Volume server %s disk %s has %d volumes", node.Id, diskType, len(diskInfo.VolumeInfos))
|
||||
}
|
||||
|
||||
// Process volumes on this specific disk
|
||||
for _, volInfo := range diskInfo.VolumeInfos {
|
||||
metric := &types.VolumeHealthMetrics{
|
||||
VolumeID: volInfo.Id,
|
||||
Server: node.Id,
|
||||
ServerAddress: node.Address,
|
||||
DiskType: diskType, // Track which disk this volume is on
|
||||
DiskId: volInfo.DiskId, // Use disk ID from volume info
|
||||
DataCenter: dc.Id, // Data center from current loop
|
||||
Rack: rack.Id, // Rack from current loop
|
||||
Collection: volInfo.Collection,
|
||||
Size: volInfo.Size,
|
||||
DeletedBytes: volInfo.DeletedByteCount,
|
||||
LastModified: time.Unix(int64(volInfo.ModifiedAtSecond), 0),
|
||||
IsReadOnly: volInfo.ReadOnly,
|
||||
HasRemoteCopy: volInfo.RemoteStorageName != "",
|
||||
IsECVolume: false, // Will be determined from volume structure
|
||||
ReplicaCount: 1, // Will be counted
|
||||
ExpectedReplicas: int(volInfo.ReplicaPlacement),
|
||||
}
|
||||
|
||||
// Calculate derived metrics
|
||||
if metric.Size > 0 {
|
||||
metric.GarbageRatio = float64(metric.DeletedBytes) / float64(metric.Size)
|
||||
// Calculate fullness ratio using actual volume size limit from master
|
||||
metric.FullnessRatio = float64(metric.Size) / float64(volumeSizeLimitBytes)
|
||||
}
|
||||
metric.Age = time.Since(metric.LastModified)
|
||||
|
||||
glog.V(4).Infof("Volume %d on %s:%s (ID %d): size=%d, limit=%d, fullness=%.2f",
|
||||
metric.VolumeID, metric.Server, metric.DiskType, metric.DiskId, metric.Size, volumeSizeLimitBytes, metric.FullnessRatio)
|
||||
|
||||
metrics = append(metrics, metric)
|
||||
}
|
||||
}
|
||||
|
||||
if hasVolumes {
|
||||
nodesWithVolumes++
|
||||
} else {
|
||||
nodesWithoutVolumes++
|
||||
glog.V(1).Infof("Volume server %s found in topology but has no volumes", node.Id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
glog.Infof("Topology discovery: %d volume servers (%d with volumes, %d without)",
|
||||
totalNodes, nodesWithVolumes, nodesWithoutVolumes)
|
||||
|
||||
// Return topology info as a local value (not retained on the scanner struct)
|
||||
topologyInfo = resp.TopologyInfo
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to get volume health metrics: %v", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
glog.V(1).Infof("Successfully collected metrics for %d actual volumes with disk ID information", len(metrics))
|
||||
|
||||
// Count actual replicas and identify EC volumes
|
||||
ms.enrichVolumeMetrics(metrics)
|
||||
|
||||
return metrics, topologyInfo, nil
|
||||
}
|
||||
|
||||
// enrichVolumeMetrics adds additional information like replica counts
|
||||
func (ms *MaintenanceScanner) enrichVolumeMetrics(metrics []*types.VolumeHealthMetrics) {
|
||||
// Group volumes by ID to count replicas
|
||||
volumeGroups := make(map[uint32][]*types.VolumeHealthMetrics)
|
||||
for _, metric := range metrics {
|
||||
volumeGroups[metric.VolumeID] = append(volumeGroups[metric.VolumeID], metric)
|
||||
}
|
||||
|
||||
// Update replica counts for actual volumes
|
||||
for volumeID, replicas := range volumeGroups {
|
||||
replicaCount := len(replicas)
|
||||
for _, replica := range replicas {
|
||||
replica.ReplicaCount = replicaCount
|
||||
}
|
||||
glog.V(4).Infof("Volume %d has %d replicas", volumeID, replicaCount)
|
||||
}
|
||||
|
||||
// TODO: Identify EC volumes by checking volume structure
|
||||
// This would require querying volume servers for EC shard information
|
||||
}
|
||||
361
weed/admin/maintenance/maintenance_types.go
Normal file
361
weed/admin/maintenance/maintenance_types.go
Normal file
@@ -0,0 +1,361 @@
|
||||
package maintenance
|
||||
|
||||
import (
|
||||
"html/template"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
)
|
||||
|
||||
// AdminClient interface defines what the maintenance system needs from the admin server
|
||||
type AdminClient interface {
|
||||
WithMasterClient(fn func(client master_pb.SeaweedClient) error) error
|
||||
}
|
||||
|
||||
// MaintenanceTaskType represents different types of maintenance operations
|
||||
type MaintenanceTaskType string
|
||||
|
||||
// MaintenanceTaskPriority represents task execution priority
|
||||
type MaintenanceTaskPriority int
|
||||
|
||||
const (
|
||||
PriorityLow MaintenanceTaskPriority = iota
|
||||
PriorityNormal
|
||||
PriorityHigh
|
||||
PriorityCritical
|
||||
)
|
||||
|
||||
// MaintenanceTaskStatus represents the current status of a task
|
||||
type MaintenanceTaskStatus string
|
||||
|
||||
const (
|
||||
TaskStatusPending MaintenanceTaskStatus = "pending"
|
||||
TaskStatusAssigned MaintenanceTaskStatus = "assigned"
|
||||
TaskStatusInProgress MaintenanceTaskStatus = "in_progress"
|
||||
TaskStatusCompleted MaintenanceTaskStatus = "completed"
|
||||
TaskStatusFailed MaintenanceTaskStatus = "failed"
|
||||
TaskStatusCancelled MaintenanceTaskStatus = "cancelled"
|
||||
)
|
||||
|
||||
// MaintenanceTask represents a single maintenance operation
|
||||
type MaintenanceTask struct {
|
||||
ID string `json:"id"`
|
||||
Type MaintenanceTaskType `json:"type"`
|
||||
Priority MaintenanceTaskPriority `json:"priority"`
|
||||
Status MaintenanceTaskStatus `json:"status"`
|
||||
VolumeID uint32 `json:"volume_id,omitempty"`
|
||||
Server string `json:"server,omitempty"`
|
||||
Collection string `json:"collection,omitempty"`
|
||||
TypedParams *worker_pb.TaskParams `json:"typed_params,omitempty"`
|
||||
Reason string `json:"reason"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ScheduledAt time.Time `json:"scheduled_at"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
WorkerID string `json:"worker_id,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Progress float64 `json:"progress"` // 0-100
|
||||
RetryCount int `json:"retry_count"`
|
||||
MaxRetries int `json:"max_retries"`
|
||||
|
||||
// Enhanced fields for detailed task tracking
|
||||
CreatedBy string `json:"created_by,omitempty"` // Who/what created this task
|
||||
CreationContext string `json:"creation_context,omitempty"` // Additional context about creation
|
||||
AssignmentHistory []*TaskAssignmentRecord `json:"assignment_history,omitempty"` // History of worker assignments
|
||||
DetailedReason string `json:"detailed_reason,omitempty"` // More detailed explanation than Reason
|
||||
Tags map[string]string `json:"tags,omitempty"` // Additional metadata tags
|
||||
}
|
||||
|
||||
// TaskAssignmentRecord tracks when a task was assigned to a worker
|
||||
type TaskAssignmentRecord struct {
|
||||
WorkerID string `json:"worker_id"`
|
||||
WorkerAddress string `json:"worker_address"`
|
||||
AssignedAt time.Time `json:"assigned_at"`
|
||||
UnassignedAt *time.Time `json:"unassigned_at,omitempty"`
|
||||
Reason string `json:"reason"` // Why was it assigned/unassigned
|
||||
}
|
||||
|
||||
// TaskExecutionLog represents a log entry from task execution
|
||||
type TaskExecutionLog struct {
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Level string `json:"level"` // "info", "warn", "error", "debug"
|
||||
Message string `json:"message"`
|
||||
Source string `json:"source"` // Which component logged this
|
||||
TaskID string `json:"task_id"`
|
||||
WorkerID string `json:"worker_id"`
|
||||
// Optional structured fields carried from worker logs
|
||||
Fields map[string]string `json:"fields,omitempty"`
|
||||
// Optional progress/status carried from worker logs
|
||||
Progress *float64 `json:"progress,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
// TaskDetailData represents comprehensive information about a task for the detail view
|
||||
type TaskDetailData struct {
|
||||
Task *MaintenanceTask `json:"task"`
|
||||
AssignmentHistory []*TaskAssignmentRecord `json:"assignment_history"`
|
||||
ExecutionLogs []*TaskExecutionLog `json:"execution_logs"`
|
||||
RelatedTasks []*MaintenanceTask `json:"related_tasks,omitempty"` // Other tasks on same volume/server
|
||||
WorkerInfo *MaintenanceWorker `json:"worker_info,omitempty"` // Current or last assigned worker
|
||||
CreationMetrics *TaskCreationMetrics `json:"creation_metrics,omitempty"` // Metrics that led to task creation
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
}
|
||||
|
||||
// TaskCreationMetrics holds metrics that led to the task being created
|
||||
type TaskCreationMetrics struct {
|
||||
TriggerMetric string `json:"trigger_metric"` // What metric triggered this task
|
||||
MetricValue float64 `json:"metric_value"` // Value of the trigger metric
|
||||
Threshold float64 `json:"threshold"` // Threshold that was exceeded
|
||||
VolumeMetrics *VolumeHealthMetrics `json:"volume_metrics,omitempty"`
|
||||
AdditionalData map[string]interface{} `json:"additional_data,omitempty"`
|
||||
}
|
||||
|
||||
// MaintenanceConfig holds configuration for the maintenance system
|
||||
// DEPRECATED: Use worker_pb.MaintenanceConfig instead
|
||||
type MaintenanceConfig = worker_pb.MaintenanceConfig
|
||||
|
||||
// MaintenancePolicy defines policies for maintenance operations
|
||||
// DEPRECATED: Use worker_pb.MaintenancePolicy instead
|
||||
type MaintenancePolicy = worker_pb.MaintenancePolicy
|
||||
|
||||
// TaskPolicy represents configuration for a specific task type
|
||||
// DEPRECATED: Use worker_pb.TaskPolicy instead
|
||||
type TaskPolicy = worker_pb.TaskPolicy
|
||||
|
||||
// TaskPersistence interface for task state persistence
|
||||
type TaskPersistence interface {
|
||||
SaveTaskState(task *MaintenanceTask) error
|
||||
LoadTaskState(taskID string) (*MaintenanceTask, error)
|
||||
LoadAllTaskStates() ([]*MaintenanceTask, error)
|
||||
DeleteTaskState(taskID string) error
|
||||
DeleteAllTaskStates() error
|
||||
CleanupCompletedTasks() error
|
||||
|
||||
// Policy persistence
|
||||
SaveTaskPolicy(taskType string, policy *TaskPolicy) error
|
||||
}
|
||||
|
||||
// Default configuration values
|
||||
func DefaultMaintenanceConfig() *MaintenanceConfig {
|
||||
return DefaultMaintenanceConfigProto()
|
||||
}
|
||||
|
||||
// Policy helper functions (since we can't add methods to type aliases)
|
||||
|
||||
// GetTaskPolicy returns the policy for a specific task type, or nil when the maintenance
|
||||
// policy has no entry for it.
|
||||
//
|
||||
// A nil maintenance policy is a legitimate state, not a programming error:
|
||||
// MaintenanceConfig.Policy is unset until something builds one, and UpdateConfig accepts a
|
||||
// config that carries none. Dereferencing it here panicked the whole admin process.
|
||||
func GetTaskPolicy(mp *MaintenancePolicy, taskType MaintenanceTaskType) *TaskPolicy {
|
||||
if mp == nil || mp.TaskPolicies == nil {
|
||||
return nil
|
||||
}
|
||||
return mp.TaskPolicies[string(taskType)]
|
||||
}
|
||||
|
||||
// IsTaskEnabled returns whether a task type is enabled
|
||||
func IsTaskEnabled(mp *MaintenancePolicy, taskType MaintenanceTaskType) bool {
|
||||
policy := GetTaskPolicy(mp, taskType)
|
||||
if policy == nil {
|
||||
return false
|
||||
}
|
||||
return policy.Enabled
|
||||
}
|
||||
|
||||
// GetMaxConcurrent returns the max concurrent limit for a task type
|
||||
func GetMaxConcurrent(mp *MaintenancePolicy, taskType MaintenanceTaskType) int {
|
||||
policy := GetTaskPolicy(mp, taskType)
|
||||
if policy == nil {
|
||||
return 1
|
||||
}
|
||||
return int(policy.MaxConcurrent)
|
||||
}
|
||||
|
||||
// GetRepeatInterval returns the repeat interval for a task type
|
||||
func GetRepeatInterval(mp *MaintenancePolicy, taskType MaintenanceTaskType) int {
|
||||
policy := GetTaskPolicy(mp, taskType)
|
||||
if policy == nil {
|
||||
if mp == nil {
|
||||
return 0
|
||||
}
|
||||
return int(mp.DefaultRepeatIntervalSeconds)
|
||||
}
|
||||
return int(policy.RepeatIntervalSeconds)
|
||||
}
|
||||
|
||||
// SetTaskConfig sets a configuration value for a task type (legacy method - use typed setters above)
|
||||
// Note: SetTaskConfig was removed - use typed setters: SetVacuumTaskConfig, SetErasureCodingTaskConfig, SetBalanceTaskConfig, or SetReplicationTaskConfig
|
||||
|
||||
// MaintenanceWorker represents a worker instance
|
||||
type MaintenanceWorker struct {
|
||||
ID string `json:"id"`
|
||||
Address string `json:"address"`
|
||||
LastHeartbeat time.Time `json:"last_heartbeat"`
|
||||
Status string `json:"status"` // active, inactive, busy
|
||||
CurrentTask *MaintenanceTask `json:"current_task,omitempty"`
|
||||
Capabilities []MaintenanceTaskType `json:"capabilities"`
|
||||
MaxConcurrent int `json:"max_concurrent"`
|
||||
CurrentLoad int `json:"current_load"`
|
||||
}
|
||||
|
||||
// WorkerSlots is a point-in-time snapshot of one worker's task slot usage,
|
||||
// taken under the queue lock so callers never read live worker fields.
|
||||
type WorkerSlots struct {
|
||||
Used int `json:"used"`
|
||||
Max int `json:"max"`
|
||||
}
|
||||
|
||||
// MaintenanceQueue manages the task queue and worker coordination
|
||||
type MaintenanceQueue struct {
|
||||
tasks map[string]*MaintenanceTask
|
||||
workers map[string]*MaintenanceWorker
|
||||
pendingTasks []*MaintenanceTask
|
||||
mutex sync.RWMutex
|
||||
policy *MaintenancePolicy
|
||||
integration *MaintenanceIntegration
|
||||
persistence TaskPersistence // Interface for task persistence
|
||||
}
|
||||
|
||||
// MaintenanceScanner analyzes the cluster and generates maintenance tasks
|
||||
type MaintenanceScanner struct {
|
||||
adminClient AdminClient
|
||||
policy *MaintenancePolicy
|
||||
queue *MaintenanceQueue
|
||||
lastScan map[MaintenanceTaskType]time.Time
|
||||
integration *MaintenanceIntegration
|
||||
}
|
||||
|
||||
// TaskDetectionResult represents the result of scanning for maintenance needs
|
||||
type TaskDetectionResult struct {
|
||||
TaskID string `json:"task_id"`
|
||||
TaskType MaintenanceTaskType `json:"task_type"`
|
||||
VolumeID uint32 `json:"volume_id,omitempty"`
|
||||
Server string `json:"server,omitempty"`
|
||||
Collection string `json:"collection,omitempty"`
|
||||
Priority MaintenanceTaskPriority `json:"priority"`
|
||||
Reason string `json:"reason"`
|
||||
TypedParams *worker_pb.TaskParams `json:"typed_params,omitempty"`
|
||||
ScheduleAt time.Time `json:"schedule_at"`
|
||||
}
|
||||
|
||||
// VolumeHealthMetrics represents the health metrics for a volume
|
||||
type VolumeHealthMetrics struct {
|
||||
VolumeID uint32 `json:"volume_id"`
|
||||
Server string `json:"server"`
|
||||
ServerAddress string `json:"server_address"`
|
||||
DiskType string `json:"disk_type"` // Disk type (e.g., "hdd", "ssd") or disk path (e.g., "/data1")
|
||||
DiskId uint32 `json:"disk_id"` // ID of the disk in Store.Locations array
|
||||
DataCenter string `json:"data_center"` // Data center of the server
|
||||
Rack string `json:"rack"` // Rack of the server
|
||||
Collection string `json:"collection"`
|
||||
Size uint64 `json:"size"`
|
||||
DeletedBytes uint64 `json:"deleted_bytes"`
|
||||
GarbageRatio float64 `json:"garbage_ratio"`
|
||||
LastModified time.Time `json:"last_modified"`
|
||||
Age time.Duration `json:"age"`
|
||||
ReplicaCount int `json:"replica_count"`
|
||||
ExpectedReplicas int `json:"expected_replicas"`
|
||||
IsReadOnly bool `json:"is_read_only"`
|
||||
HasRemoteCopy bool `json:"has_remote_copy"`
|
||||
IsECVolume bool `json:"is_ec_volume"`
|
||||
FullnessRatio float64 `json:"fullness_ratio"`
|
||||
}
|
||||
|
||||
// MaintenanceStats provides statistics about maintenance operations
|
||||
type MaintenanceStats struct {
|
||||
TotalTasks int `json:"total_tasks"`
|
||||
TasksByStatus map[MaintenanceTaskStatus]int `json:"tasks_by_status"`
|
||||
TasksByType map[MaintenanceTaskType]int `json:"tasks_by_type"`
|
||||
ActiveWorkers int `json:"active_workers"`
|
||||
CompletedToday int `json:"completed_today"`
|
||||
FailedToday int `json:"failed_today"`
|
||||
AverageTaskTime time.Duration `json:"average_task_time"`
|
||||
LastScanTime time.Time `json:"last_scan_time"`
|
||||
NextScanTime time.Time `json:"next_scan_time"`
|
||||
}
|
||||
|
||||
// MaintenanceQueueData represents data for the queue visualization UI
|
||||
type MaintenanceQueueData struct {
|
||||
Tasks []*MaintenanceTask `json:"tasks"`
|
||||
Workers []*MaintenanceWorker `json:"workers"`
|
||||
Stats *QueueStats `json:"stats"`
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
}
|
||||
|
||||
// QueueStats provides statistics for the queue UI
|
||||
type QueueStats struct {
|
||||
PendingTasks int `json:"pending_tasks"`
|
||||
RunningTasks int `json:"running_tasks"`
|
||||
CompletedToday int `json:"completed_today"`
|
||||
FailedToday int `json:"failed_today"`
|
||||
TotalTasks int `json:"total_tasks"`
|
||||
}
|
||||
|
||||
// MaintenanceConfigData represents configuration data for the UI
|
||||
type MaintenanceConfigData struct {
|
||||
Config *MaintenanceConfig `json:"config"`
|
||||
IsEnabled bool `json:"is_enabled"`
|
||||
LastScanTime time.Time `json:"last_scan_time"`
|
||||
NextScanTime time.Time `json:"next_scan_time"`
|
||||
SystemStats *MaintenanceStats `json:"system_stats"`
|
||||
MenuItems []*MaintenanceMenuItem `json:"menu_items"`
|
||||
}
|
||||
|
||||
// MaintenanceMenuItem represents a menu item for task configuration
|
||||
type MaintenanceMenuItem struct {
|
||||
TaskType MaintenanceTaskType `json:"task_type"`
|
||||
DisplayName string `json:"display_name"`
|
||||
Description string `json:"description"`
|
||||
Icon string `json:"icon"`
|
||||
IsEnabled bool `json:"is_enabled"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
// WorkerDetailsData represents detailed worker information
|
||||
type WorkerDetailsData struct {
|
||||
Worker *MaintenanceWorker `json:"worker"`
|
||||
CurrentTasks []*MaintenanceTask `json:"current_tasks"`
|
||||
RecentTasks []*MaintenanceTask `json:"recent_tasks"`
|
||||
Performance *WorkerPerformance `json:"performance"`
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
}
|
||||
|
||||
// WorkerPerformance tracks worker performance metrics
|
||||
type WorkerPerformance struct {
|
||||
TasksCompleted int `json:"tasks_completed"`
|
||||
TasksFailed int `json:"tasks_failed"`
|
||||
AverageTaskTime time.Duration `json:"average_task_time"`
|
||||
Uptime time.Duration `json:"uptime"`
|
||||
SuccessRate float64 `json:"success_rate"`
|
||||
}
|
||||
|
||||
// TaskConfigData represents data for individual task configuration page
|
||||
type TaskConfigData struct {
|
||||
TaskType MaintenanceTaskType `json:"task_type"`
|
||||
TaskName string `json:"task_name"`
|
||||
TaskIcon string `json:"task_icon"`
|
||||
Description string `json:"description"`
|
||||
ConfigFormHTML template.HTML `json:"config_form_html"`
|
||||
}
|
||||
|
||||
// ClusterReplicationTask represents a cluster replication task parameters
|
||||
type ClusterReplicationTask struct {
|
||||
SourcePath string `json:"source_path"`
|
||||
TargetCluster string `json:"target_cluster"`
|
||||
TargetPath string `json:"target_path"`
|
||||
ReplicationMode string `json:"replication_mode"` // "sync", "async", "backup"
|
||||
Priority int `json:"priority"`
|
||||
Checksum string `json:"checksum,omitempty"`
|
||||
FileSize int64 `json:"file_size"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
// Helper functions to extract configuration fields
|
||||
|
||||
// Note: Removed getVacuumConfigField, getErasureCodingConfigField, getBalanceConfigField, getReplicationConfigField
|
||||
// These were orphaned after removing GetTaskConfig - use typed getters instead
|
||||
311
weed/admin/maintenance/pending_operations.go
Normal file
311
weed/admin/maintenance/pending_operations.go
Normal file
@@ -0,0 +1,311 @@
|
||||
package maintenance
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
// PendingOperationType represents the type of pending operation
|
||||
type PendingOperationType string
|
||||
|
||||
const (
|
||||
OpTypeVolumeMove PendingOperationType = "volume_move"
|
||||
OpTypeVolumeBalance PendingOperationType = "volume_balance"
|
||||
OpTypeErasureCoding PendingOperationType = "erasure_coding"
|
||||
OpTypeVacuum PendingOperationType = "vacuum"
|
||||
OpTypeReplication PendingOperationType = "replication"
|
||||
)
|
||||
|
||||
// PendingOperation represents a pending volume/shard operation
|
||||
type PendingOperation struct {
|
||||
VolumeID uint32 `json:"volume_id"`
|
||||
OperationType PendingOperationType `json:"operation_type"`
|
||||
SourceNode string `json:"source_node"`
|
||||
DestNode string `json:"dest_node,omitempty"` // Empty for non-movement operations
|
||||
TaskID string `json:"task_id"`
|
||||
StartTime time.Time `json:"start_time"`
|
||||
EstimatedSize uint64 `json:"estimated_size"` // Bytes
|
||||
Collection string `json:"collection"`
|
||||
Status string `json:"status"` // "assigned", "in_progress", "completing"
|
||||
}
|
||||
|
||||
// PendingOperations tracks all pending volume/shard operations
|
||||
type PendingOperations struct {
|
||||
// Operations by volume ID for conflict detection
|
||||
byVolumeID map[uint32]*PendingOperation
|
||||
|
||||
// Operations by task ID for updates
|
||||
byTaskID map[string]*PendingOperation
|
||||
|
||||
// Operations by node for capacity calculations
|
||||
bySourceNode map[string][]*PendingOperation
|
||||
byDestNode map[string][]*PendingOperation
|
||||
|
||||
mutex sync.RWMutex
|
||||
}
|
||||
|
||||
// NewPendingOperations creates a new pending operations tracker
|
||||
func NewPendingOperations() *PendingOperations {
|
||||
return &PendingOperations{
|
||||
byVolumeID: make(map[uint32]*PendingOperation),
|
||||
byTaskID: make(map[string]*PendingOperation),
|
||||
bySourceNode: make(map[string][]*PendingOperation),
|
||||
byDestNode: make(map[string][]*PendingOperation),
|
||||
}
|
||||
}
|
||||
|
||||
// AddOperation adds a pending operation
|
||||
func (po *PendingOperations) AddOperation(op *PendingOperation) {
|
||||
po.mutex.Lock()
|
||||
defer po.mutex.Unlock()
|
||||
|
||||
// Check for existing operation on this volume
|
||||
if existing, exists := po.byVolumeID[op.VolumeID]; exists {
|
||||
glog.V(1).Infof("Replacing existing pending operation on volume %d: %s -> %s",
|
||||
op.VolumeID, existing.TaskID, op.TaskID)
|
||||
po.removeOperationUnlocked(existing)
|
||||
}
|
||||
|
||||
// Add new operation
|
||||
po.byVolumeID[op.VolumeID] = op
|
||||
po.byTaskID[op.TaskID] = op
|
||||
|
||||
// Add to node indexes
|
||||
po.bySourceNode[op.SourceNode] = append(po.bySourceNode[op.SourceNode], op)
|
||||
if op.DestNode != "" {
|
||||
po.byDestNode[op.DestNode] = append(po.byDestNode[op.DestNode], op)
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Added pending operation: volume %d, type %s, task %s, %s -> %s",
|
||||
op.VolumeID, op.OperationType, op.TaskID, op.SourceNode, op.DestNode)
|
||||
}
|
||||
|
||||
// RemoveOperation removes a completed operation
|
||||
func (po *PendingOperations) RemoveOperation(taskID string) {
|
||||
po.mutex.Lock()
|
||||
defer po.mutex.Unlock()
|
||||
|
||||
if op, exists := po.byTaskID[taskID]; exists {
|
||||
po.removeOperationUnlocked(op)
|
||||
glog.V(2).Infof("Removed completed operation: volume %d, task %s", op.VolumeID, taskID)
|
||||
}
|
||||
}
|
||||
|
||||
// removeOperationUnlocked removes an operation (must hold lock)
|
||||
func (po *PendingOperations) removeOperationUnlocked(op *PendingOperation) {
|
||||
delete(po.byVolumeID, op.VolumeID)
|
||||
delete(po.byTaskID, op.TaskID)
|
||||
|
||||
// Remove from source node list
|
||||
if ops, exists := po.bySourceNode[op.SourceNode]; exists {
|
||||
for i, other := range ops {
|
||||
if other.TaskID == op.TaskID {
|
||||
po.bySourceNode[op.SourceNode] = append(ops[:i], ops[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove from dest node list
|
||||
if op.DestNode != "" {
|
||||
if ops, exists := po.byDestNode[op.DestNode]; exists {
|
||||
for i, other := range ops {
|
||||
if other.TaskID == op.TaskID {
|
||||
po.byDestNode[op.DestNode] = append(ops[:i], ops[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// HasPendingOperationOnVolume checks if a volume has a pending operation
|
||||
func (po *PendingOperations) HasPendingOperationOnVolume(volumeID uint32) bool {
|
||||
po.mutex.RLock()
|
||||
defer po.mutex.RUnlock()
|
||||
|
||||
_, exists := po.byVolumeID[volumeID]
|
||||
return exists
|
||||
}
|
||||
|
||||
// GetPendingOperationOnVolume returns the pending operation on a volume
|
||||
func (po *PendingOperations) GetPendingOperationOnVolume(volumeID uint32) *PendingOperation {
|
||||
po.mutex.RLock()
|
||||
defer po.mutex.RUnlock()
|
||||
|
||||
return po.byVolumeID[volumeID]
|
||||
}
|
||||
|
||||
// WouldConflictWithPending checks if a new operation would conflict with pending ones
|
||||
func (po *PendingOperations) WouldConflictWithPending(volumeID uint32, opType PendingOperationType) bool {
|
||||
po.mutex.RLock()
|
||||
defer po.mutex.RUnlock()
|
||||
|
||||
if existing, exists := po.byVolumeID[volumeID]; exists {
|
||||
// Volume already has a pending operation
|
||||
glog.V(3).Infof("Volume %d conflict: already has %s operation (task %s)",
|
||||
volumeID, existing.OperationType, existing.TaskID)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// GetPendingCapacityImpactForNode calculates pending capacity changes for a node
|
||||
func (po *PendingOperations) GetPendingCapacityImpactForNode(nodeID string) (incoming uint64, outgoing uint64) {
|
||||
po.mutex.RLock()
|
||||
defer po.mutex.RUnlock()
|
||||
|
||||
// Calculate outgoing capacity (volumes leaving this node)
|
||||
if ops, exists := po.bySourceNode[nodeID]; exists {
|
||||
for _, op := range ops {
|
||||
// Only count movement operations
|
||||
if op.DestNode != "" {
|
||||
outgoing += op.EstimatedSize
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate incoming capacity (volumes coming to this node)
|
||||
if ops, exists := po.byDestNode[nodeID]; exists {
|
||||
for _, op := range ops {
|
||||
incoming += op.EstimatedSize
|
||||
}
|
||||
}
|
||||
|
||||
return incoming, outgoing
|
||||
}
|
||||
|
||||
// FilterVolumeMetricsExcludingPending filters out volumes with pending operations
|
||||
func (po *PendingOperations) FilterVolumeMetricsExcludingPending(metrics []*types.VolumeHealthMetrics) []*types.VolumeHealthMetrics {
|
||||
po.mutex.RLock()
|
||||
defer po.mutex.RUnlock()
|
||||
|
||||
var filtered []*types.VolumeHealthMetrics
|
||||
excludedCount := 0
|
||||
|
||||
for _, metric := range metrics {
|
||||
if _, hasPending := po.byVolumeID[metric.VolumeID]; !hasPending {
|
||||
filtered = append(filtered, metric)
|
||||
} else {
|
||||
excludedCount++
|
||||
glog.V(3).Infof("Excluding volume %d from scan due to pending operation", metric.VolumeID)
|
||||
}
|
||||
}
|
||||
|
||||
if excludedCount > 0 {
|
||||
glog.V(1).Infof("Filtered out %d volumes with pending operations from %d total volumes",
|
||||
excludedCount, len(metrics))
|
||||
}
|
||||
|
||||
return filtered
|
||||
}
|
||||
|
||||
// GetNodeCapacityProjection calculates projected capacity for a node
|
||||
func (po *PendingOperations) GetNodeCapacityProjection(nodeID string, currentUsed uint64, totalCapacity uint64) NodeCapacityProjection {
|
||||
incoming, outgoing := po.GetPendingCapacityImpactForNode(nodeID)
|
||||
|
||||
projectedUsed := currentUsed + incoming - outgoing
|
||||
projectedFree := totalCapacity - projectedUsed
|
||||
|
||||
return NodeCapacityProjection{
|
||||
NodeID: nodeID,
|
||||
CurrentUsed: currentUsed,
|
||||
TotalCapacity: totalCapacity,
|
||||
PendingIncoming: incoming,
|
||||
PendingOutgoing: outgoing,
|
||||
ProjectedUsed: projectedUsed,
|
||||
ProjectedFree: projectedFree,
|
||||
}
|
||||
}
|
||||
|
||||
// GetAllPendingOperations returns all pending operations
|
||||
func (po *PendingOperations) GetAllPendingOperations() []*PendingOperation {
|
||||
po.mutex.RLock()
|
||||
defer po.mutex.RUnlock()
|
||||
|
||||
var operations []*PendingOperation
|
||||
for _, op := range po.byVolumeID {
|
||||
operations = append(operations, op)
|
||||
}
|
||||
|
||||
return operations
|
||||
}
|
||||
|
||||
// UpdateOperationStatus updates the status of a pending operation
|
||||
func (po *PendingOperations) UpdateOperationStatus(taskID string, status string) {
|
||||
po.mutex.Lock()
|
||||
defer po.mutex.Unlock()
|
||||
|
||||
if op, exists := po.byTaskID[taskID]; exists {
|
||||
op.Status = status
|
||||
glog.V(3).Infof("Updated operation status: task %s, volume %d -> %s", taskID, op.VolumeID, status)
|
||||
}
|
||||
}
|
||||
|
||||
// CleanupStaleOperations removes operations that have been running too long
|
||||
func (po *PendingOperations) CleanupStaleOperations(maxAge time.Duration) int {
|
||||
po.mutex.Lock()
|
||||
defer po.mutex.Unlock()
|
||||
|
||||
cutoff := time.Now().Add(-maxAge)
|
||||
var staleOps []*PendingOperation
|
||||
|
||||
for _, op := range po.byVolumeID {
|
||||
if op.StartTime.Before(cutoff) {
|
||||
staleOps = append(staleOps, op)
|
||||
}
|
||||
}
|
||||
|
||||
for _, op := range staleOps {
|
||||
po.removeOperationUnlocked(op)
|
||||
glog.Warningf("Removed stale pending operation: volume %d, task %s, age %v",
|
||||
op.VolumeID, op.TaskID, time.Since(op.StartTime))
|
||||
}
|
||||
|
||||
return len(staleOps)
|
||||
}
|
||||
|
||||
// NodeCapacityProjection represents projected capacity for a node
|
||||
type NodeCapacityProjection struct {
|
||||
NodeID string `json:"node_id"`
|
||||
CurrentUsed uint64 `json:"current_used"`
|
||||
TotalCapacity uint64 `json:"total_capacity"`
|
||||
PendingIncoming uint64 `json:"pending_incoming"`
|
||||
PendingOutgoing uint64 `json:"pending_outgoing"`
|
||||
ProjectedUsed uint64 `json:"projected_used"`
|
||||
ProjectedFree uint64 `json:"projected_free"`
|
||||
}
|
||||
|
||||
// GetStats returns statistics about pending operations
|
||||
func (po *PendingOperations) GetStats() PendingOperationsStats {
|
||||
po.mutex.RLock()
|
||||
defer po.mutex.RUnlock()
|
||||
|
||||
stats := PendingOperationsStats{
|
||||
TotalOperations: len(po.byVolumeID),
|
||||
ByType: make(map[PendingOperationType]int),
|
||||
ByStatus: make(map[string]int),
|
||||
}
|
||||
|
||||
var totalSize uint64
|
||||
for _, op := range po.byVolumeID {
|
||||
stats.ByType[op.OperationType]++
|
||||
stats.ByStatus[op.Status]++
|
||||
totalSize += op.EstimatedSize
|
||||
}
|
||||
|
||||
stats.TotalEstimatedSize = totalSize
|
||||
return stats
|
||||
}
|
||||
|
||||
// PendingOperationsStats provides statistics about pending operations
|
||||
type PendingOperationsStats struct {
|
||||
TotalOperations int `json:"total_operations"`
|
||||
ByType map[PendingOperationType]int `json:"by_type"`
|
||||
ByStatus map[string]int `json:"by_status"`
|
||||
TotalEstimatedSize uint64 `json:"total_estimated_size"`
|
||||
}
|
||||
250
weed/admin/maintenance/pending_operations_test.go
Normal file
250
weed/admin/maintenance/pending_operations_test.go
Normal file
@@ -0,0 +1,250 @@
|
||||
package maintenance
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/types"
|
||||
)
|
||||
|
||||
func TestPendingOperations_ConflictDetection(t *testing.T) {
|
||||
pendingOps := NewPendingOperations()
|
||||
|
||||
// Add a pending erasure coding operation on volume 123
|
||||
op := &PendingOperation{
|
||||
VolumeID: 123,
|
||||
OperationType: OpTypeErasureCoding,
|
||||
SourceNode: "node1",
|
||||
TaskID: "task-001",
|
||||
StartTime: time.Now(),
|
||||
EstimatedSize: 1024 * 1024 * 1024, // 1GB
|
||||
Collection: "test",
|
||||
Status: "assigned",
|
||||
}
|
||||
|
||||
pendingOps.AddOperation(op)
|
||||
|
||||
// Test conflict detection
|
||||
if !pendingOps.HasPendingOperationOnVolume(123) {
|
||||
t.Errorf("Expected volume 123 to have pending operation")
|
||||
}
|
||||
|
||||
if !pendingOps.WouldConflictWithPending(123, OpTypeVacuum) {
|
||||
t.Errorf("Expected conflict when trying to add vacuum operation on volume 123")
|
||||
}
|
||||
|
||||
if pendingOps.HasPendingOperationOnVolume(124) {
|
||||
t.Errorf("Expected volume 124 to have no pending operation")
|
||||
}
|
||||
|
||||
if pendingOps.WouldConflictWithPending(124, OpTypeVacuum) {
|
||||
t.Errorf("Expected no conflict for volume 124")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPendingOperations_CapacityProjection(t *testing.T) {
|
||||
pendingOps := NewPendingOperations()
|
||||
|
||||
// Add operation moving volume from node1 to node2
|
||||
op1 := &PendingOperation{
|
||||
VolumeID: 100,
|
||||
OperationType: OpTypeVolumeMove,
|
||||
SourceNode: "node1",
|
||||
DestNode: "node2",
|
||||
TaskID: "task-001",
|
||||
StartTime: time.Now(),
|
||||
EstimatedSize: 2 * 1024 * 1024 * 1024, // 2GB
|
||||
Collection: "test",
|
||||
Status: "in_progress",
|
||||
}
|
||||
|
||||
// Add operation moving volume from node3 to node1
|
||||
op2 := &PendingOperation{
|
||||
VolumeID: 101,
|
||||
OperationType: OpTypeVolumeMove,
|
||||
SourceNode: "node3",
|
||||
DestNode: "node1",
|
||||
TaskID: "task-002",
|
||||
StartTime: time.Now(),
|
||||
EstimatedSize: 1 * 1024 * 1024 * 1024, // 1GB
|
||||
Collection: "test",
|
||||
Status: "assigned",
|
||||
}
|
||||
|
||||
pendingOps.AddOperation(op1)
|
||||
pendingOps.AddOperation(op2)
|
||||
|
||||
// Test capacity impact for node1
|
||||
incoming, outgoing := pendingOps.GetPendingCapacityImpactForNode("node1")
|
||||
expectedIncoming := uint64(1 * 1024 * 1024 * 1024) // 1GB incoming
|
||||
expectedOutgoing := uint64(2 * 1024 * 1024 * 1024) // 2GB outgoing
|
||||
|
||||
if incoming != expectedIncoming {
|
||||
t.Errorf("Expected incoming capacity %d, got %d", expectedIncoming, incoming)
|
||||
}
|
||||
|
||||
if outgoing != expectedOutgoing {
|
||||
t.Errorf("Expected outgoing capacity %d, got %d", expectedOutgoing, outgoing)
|
||||
}
|
||||
|
||||
// Test projection for node1
|
||||
currentUsed := uint64(10 * 1024 * 1024 * 1024) // 10GB current
|
||||
totalCapacity := uint64(50 * 1024 * 1024 * 1024) // 50GB total
|
||||
|
||||
projection := pendingOps.GetNodeCapacityProjection("node1", currentUsed, totalCapacity)
|
||||
|
||||
expectedProjectedUsed := currentUsed + incoming - outgoing // 10 + 1 - 2 = 9GB
|
||||
expectedProjectedFree := totalCapacity - expectedProjectedUsed // 50 - 9 = 41GB
|
||||
|
||||
if projection.ProjectedUsed != expectedProjectedUsed {
|
||||
t.Errorf("Expected projected used %d, got %d", expectedProjectedUsed, projection.ProjectedUsed)
|
||||
}
|
||||
|
||||
if projection.ProjectedFree != expectedProjectedFree {
|
||||
t.Errorf("Expected projected free %d, got %d", expectedProjectedFree, projection.ProjectedFree)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPendingOperations_VolumeFiltering(t *testing.T) {
|
||||
pendingOps := NewPendingOperations()
|
||||
|
||||
// Create volume metrics
|
||||
metrics := []*types.VolumeHealthMetrics{
|
||||
{VolumeID: 100, Server: "node1", ServerAddress: "192.168.1.1:8080"},
|
||||
{VolumeID: 101, Server: "node2", ServerAddress: "192.168.1.2:8080"},
|
||||
{VolumeID: 102, Server: "node3", ServerAddress: "192.168.1.3:8080"},
|
||||
{VolumeID: 103, Server: "node1", ServerAddress: "192.168.1.1:8080"},
|
||||
}
|
||||
|
||||
// Add pending operations on volumes 101 and 103
|
||||
op1 := &PendingOperation{
|
||||
VolumeID: 101,
|
||||
OperationType: OpTypeVacuum,
|
||||
SourceNode: "node2",
|
||||
TaskID: "task-001",
|
||||
StartTime: time.Now(),
|
||||
EstimatedSize: 1024 * 1024 * 1024,
|
||||
Status: "in_progress",
|
||||
}
|
||||
|
||||
op2 := &PendingOperation{
|
||||
VolumeID: 103,
|
||||
OperationType: OpTypeErasureCoding,
|
||||
SourceNode: "node1",
|
||||
TaskID: "task-002",
|
||||
StartTime: time.Now(),
|
||||
EstimatedSize: 2 * 1024 * 1024 * 1024,
|
||||
Status: "assigned",
|
||||
}
|
||||
|
||||
pendingOps.AddOperation(op1)
|
||||
pendingOps.AddOperation(op2)
|
||||
|
||||
// Filter metrics
|
||||
filtered := pendingOps.FilterVolumeMetricsExcludingPending(metrics)
|
||||
|
||||
// Should only have volumes 100 and 102 (101 and 103 are filtered out)
|
||||
if len(filtered) != 2 {
|
||||
t.Errorf("Expected 2 filtered metrics, got %d", len(filtered))
|
||||
}
|
||||
|
||||
// Check that correct volumes remain
|
||||
foundVolumes := make(map[uint32]bool)
|
||||
for _, metric := range filtered {
|
||||
foundVolumes[metric.VolumeID] = true
|
||||
}
|
||||
|
||||
if !foundVolumes[100] || !foundVolumes[102] {
|
||||
t.Errorf("Expected volumes 100 and 102 to remain after filtering")
|
||||
}
|
||||
|
||||
if foundVolumes[101] || foundVolumes[103] {
|
||||
t.Errorf("Expected volumes 101 and 103 to be filtered out")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPendingOperations_OperationLifecycle(t *testing.T) {
|
||||
pendingOps := NewPendingOperations()
|
||||
|
||||
// Add operation
|
||||
op := &PendingOperation{
|
||||
VolumeID: 200,
|
||||
OperationType: OpTypeVolumeBalance,
|
||||
SourceNode: "node1",
|
||||
DestNode: "node2",
|
||||
TaskID: "task-balance-001",
|
||||
StartTime: time.Now(),
|
||||
EstimatedSize: 1024 * 1024 * 1024,
|
||||
Status: "assigned",
|
||||
}
|
||||
|
||||
pendingOps.AddOperation(op)
|
||||
|
||||
// Check it exists
|
||||
if !pendingOps.HasPendingOperationOnVolume(200) {
|
||||
t.Errorf("Expected volume 200 to have pending operation")
|
||||
}
|
||||
|
||||
// Update status
|
||||
pendingOps.UpdateOperationStatus("task-balance-001", "in_progress")
|
||||
|
||||
retrievedOp := pendingOps.GetPendingOperationOnVolume(200)
|
||||
if retrievedOp == nil {
|
||||
t.Errorf("Expected to retrieve pending operation for volume 200")
|
||||
} else if retrievedOp.Status != "in_progress" {
|
||||
t.Errorf("Expected operation status to be 'in_progress', got '%s'", retrievedOp.Status)
|
||||
}
|
||||
|
||||
// Complete operation
|
||||
pendingOps.RemoveOperation("task-balance-001")
|
||||
|
||||
if pendingOps.HasPendingOperationOnVolume(200) {
|
||||
t.Errorf("Expected volume 200 to have no pending operation after removal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPendingOperations_StaleCleanup(t *testing.T) {
|
||||
pendingOps := NewPendingOperations()
|
||||
|
||||
// Add recent operation
|
||||
recentOp := &PendingOperation{
|
||||
VolumeID: 300,
|
||||
OperationType: OpTypeVacuum,
|
||||
SourceNode: "node1",
|
||||
TaskID: "task-recent",
|
||||
StartTime: time.Now(),
|
||||
EstimatedSize: 1024 * 1024 * 1024,
|
||||
Status: "in_progress",
|
||||
}
|
||||
|
||||
// Add stale operation (24 hours ago)
|
||||
staleOp := &PendingOperation{
|
||||
VolumeID: 301,
|
||||
OperationType: OpTypeErasureCoding,
|
||||
SourceNode: "node2",
|
||||
TaskID: "task-stale",
|
||||
StartTime: time.Now().Add(-24 * time.Hour),
|
||||
EstimatedSize: 2 * 1024 * 1024 * 1024,
|
||||
Status: "in_progress",
|
||||
}
|
||||
|
||||
pendingOps.AddOperation(recentOp)
|
||||
pendingOps.AddOperation(staleOp)
|
||||
|
||||
// Clean up operations older than 1 hour
|
||||
removedCount := pendingOps.CleanupStaleOperations(1 * time.Hour)
|
||||
|
||||
if removedCount != 1 {
|
||||
t.Errorf("Expected to remove 1 stale operation, removed %d", removedCount)
|
||||
}
|
||||
|
||||
// Recent operation should still exist
|
||||
if !pendingOps.HasPendingOperationOnVolume(300) {
|
||||
t.Errorf("Expected recent operation on volume 300 to still exist")
|
||||
}
|
||||
|
||||
// Stale operation should be removed
|
||||
if pendingOps.HasPendingOperationOnVolume(301) {
|
||||
t.Errorf("Expected stale operation on volume 301 to be removed")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user