snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/worker/tasks/erasure_coding/config.go

@@ -0,0 +1,255 @@
+package erasure_coding
+
+import (
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/admin/config"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
+)
+
+// Config extends BaseConfig with erasure coding specific settings
+type Config struct {
+	base.BaseConfig
+	QuietForSeconds  int      `json:"quiet_for_seconds"`
+	FullnessRatio    float64  `json:"fullness_ratio"`
+	CollectionFilter string   `json:"collection_filter"`
+	MinSizeMB        int      `json:"min_size_mb"`
+	PreferredTags    []string `json:"preferred_tags"`
+	ReplicaPlacement string   `json:"replica_placement"` // e.g. "020"; empty falls back to the master default replication
+}
+
+// NewDefaultConfig creates a new default erasure coding configuration
+func NewDefaultConfig() *Config {
+	return &Config{
+		BaseConfig: base.BaseConfig{
+			Enabled:             true,
+			ScanIntervalSeconds: 60 * 60, // 1 hour
+			MaxConcurrent:       1,
+		},
+		QuietForSeconds:  3600, // 1 hour, matching the shell ec.encode -quietFor default
+		FullnessRatio:    0.95, // 95%, matching the shell ec.encode -fullPercent default
+		CollectionFilter: "",
+		MinSizeMB:        30, // 30MB (more reasonable than 100MB)
+		PreferredTags:    nil,
+	}
+}
+
+// GetConfigSpec returns the configuration schema for erasure coding tasks
+func GetConfigSpec() base.ConfigSpec {
+	return base.ConfigSpec{
+		Fields: []*config.Field{
+			{
+				Name:         "enabled",
+				JSONName:     "enabled",
+				Type:         config.FieldTypeBool,
+				DefaultValue: true,
+				Required:     false,
+				DisplayName:  "Enable Erasure Coding Tasks",
+				Description:  "Whether erasure coding tasks should be automatically created",
+				HelpText:     "Toggle this to enable or disable automatic erasure coding task generation",
+				InputType:    "checkbox",
+				CSSClasses:   "form-check-input",
+			},
+			{
+				Name:         "scan_interval_seconds",
+				JSONName:     "scan_interval_seconds",
+				Type:         config.FieldTypeInterval,
+				DefaultValue: 60 * 60,
+				MinValue:     10 * 60,
+				MaxValue:     24 * 60 * 60,
+				Required:     true,
+				DisplayName:  "Scan Interval",
+				Description:  "How often to scan for volumes needing erasure coding",
+				HelpText:     "The system will check for volumes that need erasure coding at this interval",
+				Placeholder:  "1",
+				Unit:         config.UnitHours,
+				InputType:    "interval",
+				CSSClasses:   "form-control",
+			},
+			{
+				Name:         "max_concurrent",
+				JSONName:     "max_concurrent",
+				Type:         config.FieldTypeInt,
+				DefaultValue: 1,
+				MinValue:     1,
+				MaxValue:     5,
+				Required:     true,
+				DisplayName:  "Max Concurrent Tasks",
+				Description:  "Maximum number of erasure coding tasks that can run simultaneously",
+				HelpText:     "Limits the number of erasure coding operations running at the same time",
+				Placeholder:  "1 (default)",
+				Unit:         config.UnitCount,
+				InputType:    "number",
+				CSSClasses:   "form-control",
+			},
+			{
+				Name:         "quiet_for_seconds",
+				JSONName:     "quiet_for_seconds",
+				Type:         config.FieldTypeInterval,
+				DefaultValue: 3600,
+				MinValue:     1,
+				MaxValue:     3600,
+				Required:     true,
+				DisplayName:  "Quiet Period",
+				Description:  "Minimum time volume must be quiet before erasure coding",
+				HelpText:     "Volume must not be modified for this duration before erasure coding",
+				Placeholder:  "60",
+				Unit:         config.UnitMinutes,
+				InputType:    "interval",
+				CSSClasses:   "form-control",
+			},
+			{
+				Name:         "fullness_ratio",
+				JSONName:     "fullness_ratio",
+				Type:         config.FieldTypeFloat,
+				DefaultValue: 0.95,
+				MinValue:     0.0001,
+				MaxValue:     1.0,
+				Required:     true,
+				DisplayName:  "Fullness Ratio",
+				Description:  "Minimum fullness ratio to trigger erasure coding",
+				HelpText:     "Only volumes with this fullness ratio or higher will be erasure coded",
+				Placeholder:  "0.80 (80%)",
+				Unit:         config.UnitNone,
+				InputType:    "number",
+				CSSClasses:   "form-control",
+			},
+			{
+				Name:         "collection_filter",
+				JSONName:     "collection_filter",
+				Type:         config.FieldTypeString,
+				DefaultValue: "",
+				Required:     false,
+				DisplayName:  "Collection Filter",
+				Description:  "Only process volumes from specific collections (comma-separated names, wildcards, or regex patterns)",
+				HelpText:     "Leave empty to process all collections, or list the collections to process",
+				Placeholder:  "pictures,videos",
+				InputType:    "text",
+				CSSClasses:   "form-control",
+			},
+			{
+				Name:         "min_size_mb",
+				JSONName:     "min_size_mb",
+				Type:         config.FieldTypeInt,
+				DefaultValue: 30,
+				MinValue:     1,
+				MaxValue:     1000,
+				Required:     true,
+				DisplayName:  "Minimum Size (MB)",
+				Description:  "Minimum volume size to consider for erasure coding",
+				HelpText:     "Only volumes larger than this size will be considered for erasure coding",
+				Placeholder:  "30",
+				Unit:         config.UnitNone,
+				InputType:    "number",
+				CSSClasses:   "form-control",
+			},
+			{
+				Name:         "preferred_tags",
+				JSONName:     "preferred_tags",
+				Type:         config.FieldTypeString,
+				DefaultValue: "",
+				Required:     false,
+				DisplayName:  "Preferred Disk Tags",
+				Description:  "Comma-separated disk tags to prioritize for EC shard placement",
+				HelpText:     "EC shards will be placed on disks with these tags first, then fall back to other disks if needed",
+				Placeholder:  "fast,ssd",
+				InputType:    "text",
+				CSSClasses:   "form-control",
+			},
+			{
+				Name:         "replica_placement",
+				JSONName:     "replica_placement",
+				Type:         config.FieldTypeString,
+				DefaultValue: "",
+				Required:     false,
+				DisplayName:  "Replica Placement",
+				Description:  "EC shard replica placement constraint (e.g. 020)",
+				HelpText:     "Leave empty to use the master default replication. When set, the 2nd/3rd digits cap EC shards per rack and per node (best-effort during encode: relaxed rather than failing if the cluster can't satisfy them, then enforced by rebalancing). The 1st (data-center) digit is ignored for EC placement",
+				Placeholder:  "020",
+				InputType:    "text",
+				CSSClasses:   "form-control",
+			},
+		},
+	}
+}
+
+// ToTaskPolicy converts configuration to a TaskPolicy protobuf message
+func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
+	// Defensive copy of PreferredTags to prevent external mutation
+	preferredTagsCopy := append([]string(nil), c.PreferredTags...)
+	return &worker_pb.TaskPolicy{
+		Enabled:               c.Enabled,
+		MaxConcurrent:         int32(c.MaxConcurrent),
+		RepeatIntervalSeconds: int32(c.ScanIntervalSeconds),
+		CheckIntervalSeconds:  int32(c.ScanIntervalSeconds),
+		TaskConfig: &worker_pb.TaskPolicy_ErasureCodingConfig{
+			ErasureCodingConfig: &worker_pb.ErasureCodingTaskConfig{
+				FullnessRatio:    float64(c.FullnessRatio),
+				QuietForSeconds:  int32(c.QuietForSeconds),
+				MinVolumeSizeMb:  int32(c.MinSizeMB),
+				CollectionFilter: c.CollectionFilter,
+				PreferredTags:    preferredTagsCopy,
+				ReplicaPlacement: c.ReplicaPlacement,
+			},
+		},
+	}
+}
+
+// FromTaskPolicy loads configuration from a TaskPolicy protobuf message
+func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
+	if policy == nil {
+		return fmt.Errorf("policy is nil")
+	}
+
+	// Set general TaskPolicy fields
+	c.Enabled = policy.Enabled
+	c.MaxConcurrent = int(policy.MaxConcurrent)
+	c.ScanIntervalSeconds = int(policy.RepeatIntervalSeconds) // Direct seconds-to-seconds mapping
+
+	// Set erasure coding-specific fields from the task config
+	if ecConfig := policy.GetErasureCodingConfig(); ecConfig != nil {
+		c.FullnessRatio = float64(ecConfig.FullnessRatio)
+		c.QuietForSeconds = int(ecConfig.QuietForSeconds)
+		c.MinSizeMB = int(ecConfig.MinVolumeSizeMb)
+		c.CollectionFilter = ecConfig.CollectionFilter
+		c.PreferredTags = append([]string(nil), ecConfig.PreferredTags...)
+		c.ReplicaPlacement = ecConfig.ReplicaPlacement
+	}
+
+	return nil
+}
+
+// LoadConfigFromPersistence loads configuration from the persistence layer if available
+func LoadConfigFromPersistence(configPersistence interface{}) *Config {
+	config := NewDefaultConfig()
+
+	// Try to load from persistence if available
+	if persistence, ok := configPersistence.(interface {
+		LoadErasureCodingTaskPolicy() (*worker_pb.TaskPolicy, error)
+	}); ok {
+		policy, err := persistence.LoadErasureCodingTaskPolicy()
+		switch {
+		case err != nil:
+			glog.Warningf("Could not read the persisted erasure coding configuration, falling back to defaults: %v", err)
+		case policy == nil:
+			glog.V(1).Infof("No erasure coding configuration persisted yet, using defaults")
+		default:
+			if err := config.FromTaskPolicy(policy); err != nil {
+				glog.Warningf("Could not apply the persisted erasure coding configuration, falling back to defaults: %v", err)
+			} else {
+				glog.V(1).Infof("Loaded erasure coding configuration from persistence")
+				return config
+			}
+		}
+	} else if configPersistence != nil {
+		// A store was handed in but does not expose the accessor, so the persisted
+		// settings are silently ignored - always a wiring bug, never a normal state.
+		glog.Warningf("%T cannot provide the persisted erasure coding configuration: it has no LoadErasureCodingTaskPolicy() method, "+
+			"so the compiled-in defaults are used and any saved erasure coding settings are ignored", configPersistence)
+	}
+
+	glog.V(1).Infof("Using default erasure coding configuration")
+	return config
+}