snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/worker/tasks/s3_lifecycle/cluster_rate_limit.go

@@ -0,0 +1,54 @@
+package s3_lifecycle
+
+// Cluster-wide rate-limit configuration plumbing for the daily-replay
+// worker. The admin holds a single "cluster delete budget" knob, divides
+// it by the number of execute-capable s3_lifecycle workers at job-dispatch
+// time, and ships the per-worker share to the worker via
+// ExecuteJobRequest.ClusterContext.Metadata. The worker reads the share,
+// constructs a rate.Limiter, and passes it to dailyrun.Run.
+//
+// These constants are the contract between admin (weed/admin/plugin/plugin.go
+// computes the share and writes the keys) and worker (this package's
+// handler.go reads them). Changing a name on one side without the other
+// would silently disable rate limiting — both sides must read these
+// exact values.
+
+const (
+	// ClusterDeletesPerSecondAdminKey is the admin-config field that
+	// holds the cluster-wide budget in delete RPCs per second. 0 means
+	// unlimited (legacy behavior). Set via the AdminConfigForm in
+	// handler.go's "Scope" section.
+	ClusterDeletesPerSecondAdminKey = "cluster_deletes_per_second"
+	// ClusterDeletesBurstAdminKey holds the token-bucket burst. 0 means
+	// "2 × rps" (computed by the admin allocator).
+	ClusterDeletesBurstAdminKey = "cluster_deletes_burst"
+	// MetaLogRetentionDaysAdminKey holds the operator's declaration of
+	// how far back the filer's meta-log subscription can reliably reach.
+	// Rules whose effective TTL exceeds this window can't be serviced by
+	// replay alone and get partitioned into engine.PromotedHash's walk
+	// set; a partition flip (operator shrinks retention) then trips the
+	// recovery branch on the next run. 0 = unbounded (current behavior,
+	// falls back to maxTTL in runShard so PromotedHash stays empty).
+	MetaLogRetentionDaysAdminKey = "meta_log_retention_days"
+	// WalkerIntervalMinutesAdminKey throttles the per-shard steady-state
+	// and empty-replay walker fires. dailyrun.runShard checks the time
+	// since the persisted Cursor.LastWalkedNs and skips the walk when
+	// less than this interval has elapsed; cold-start and recovery walker
+	// fires (RecoveryView) stay unconditional. 0 means "fire on every
+	// run" (the prior behavior — appropriate when the worker is driven
+	// at the operator's intended walk cadence, e.g. once per hour).
+	// Production deployments running the worker at multi-second cadence
+	// (CI ticks, sub-minute admin schedules) should set this to roughly
+	// the per-shard walk budget — typically 60 (1h) for small clusters,
+	// 360+ (6h+) for large ones.
+	WalkerIntervalMinutesAdminKey = "walker_interval_minutes"
+
+	// MetadataKeyDeletesPerSecond is the per-worker share value the
+	// admin writes into ClusterContext.Metadata at ExecuteJob time.
+	// Stored as a string of a non-negative float64; empty/missing/zero
+	// means "no rate limit on this run" (cfg.Limiter stays nil).
+	MetadataKeyDeletesPerSecond = "s3_lifecycle.deletes_per_second"
+	// MetadataKeyDeletesBurst is the per-worker burst share. Stored as
+	// a string of a non-negative integer.
+	MetadataKeyDeletesBurst = "s3_lifecycle.deletes_burst"
+)

weed/worker/tasks/s3_lifecycle/cluster_rate_limit_test.go

@@ -0,0 +1,78 @@
+package s3_lifecycle
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBuildLimiterFromClusterContext_NilCC(t *testing.T) {
+	l, desc := buildLimiterFromClusterContext(nil)
+	assert.Nil(t, l)
+	assert.Equal(t, "unlimited", desc)
+}
+
+func TestBuildLimiterFromClusterContext_NoMetadata(t *testing.T) {
+	l, desc := buildLimiterFromClusterContext(&plugin_pb.ClusterContext{})
+	assert.Nil(t, l)
+	assert.Equal(t, "unlimited", desc)
+}
+
+func TestBuildLimiterFromClusterContext_MissingRateKey(t *testing.T) {
+	l, desc := buildLimiterFromClusterContext(&plugin_pb.ClusterContext{
+		Metadata: map[string]string{"unrelated": "x"},
+	})
+	assert.Nil(t, l)
+	assert.Equal(t, "unlimited", desc)
+}
+
+func TestBuildLimiterFromClusterContext_NonPositiveRate(t *testing.T) {
+	// 0 or negative rate means the admin didn't allocate; the worker
+	// must NOT construct a limiter that throttles every request to
+	// zero-throughput.
+	for _, raw := range []string{"0", "-1", "0.0", "not-a-number", ""} {
+		l, desc := buildLimiterFromClusterContext(&plugin_pb.ClusterContext{
+			Metadata: map[string]string{MetadataKeyDeletesPerSecond: raw},
+		})
+		assert.Nil(t, l, "rate=%q must yield nil limiter", raw)
+		assert.Equal(t, "unlimited", desc)
+	}
+}
+
+func TestBuildLimiterFromClusterContext_PositiveRateBuildsLimiter(t *testing.T) {
+	l, desc := buildLimiterFromClusterContext(&plugin_pb.ClusterContext{
+		Metadata: map[string]string{
+			MetadataKeyDeletesPerSecond: "12.5",
+			MetadataKeyDeletesBurst:     "25",
+		},
+	})
+	require.NotNil(t, l)
+	assert.Equal(t, 12.5, float64(l.Limit()))
+	assert.Equal(t, 25, l.Burst())
+	assert.Contains(t, desc, "12.5/s")
+	assert.Contains(t, desc, "burst=25")
+}
+
+func TestBuildLimiterFromClusterContext_BurstMissingDefaultsTo2xRate(t *testing.T) {
+	// burst omitted (admin allocator wrote nothing) → worker computes
+	// 2 × rps so a single tick has headroom for two refills.
+	l, _ := buildLimiterFromClusterContext(&plugin_pb.ClusterContext{
+		Metadata: map[string]string{MetadataKeyDeletesPerSecond: "10"},
+	})
+	require.NotNil(t, l)
+	assert.Equal(t, 10.0, float64(l.Limit()))
+	assert.Equal(t, 20, l.Burst(), "burst default must be 2× rps")
+}
+
+func TestBuildLimiterFromClusterContext_TinyRateClampsBurstToOne(t *testing.T) {
+	// A sub-1-rps allocation (e.g. 0.5/s across few workers) would
+	// compute 2 × 0.5 = 1, but if the int truncation produced 0 the
+	// limiter would never refill. Clamp to at least 1.
+	l, _ := buildLimiterFromClusterContext(&plugin_pb.ClusterContext{
+		Metadata: map[string]string{MetadataKeyDeletesPerSecond: "0.1"},
+	})
+	require.NotNil(t, l)
+	assert.GreaterOrEqual(t, l.Burst(), 1, "burst must never round down to 0")
+}

weed/worker/tasks/s3_lifecycle/config.go

@@ -0,0 +1,62 @@
+package s3_lifecycle
+
+import (
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+)
+
+const (
+	jobType = "s3_lifecycle"
+
+	// In-process fan-out across the 16 shards.
+	shardPipelineGoroutines = 1
+)
+
+type Config struct {
+	Workers          int
+	MetaLogRetention time.Duration
+	// WalkerInterval is the minimum time between steady-state walker
+	// fires per shard. 0 means "fire on every run", preserving prior
+	// behavior; positive values gate the walker via Cursor.LastWalkedNs
+	// inside dailyrun.runShard.
+	WalkerInterval time.Duration
+}
+
+func ParseConfig(adminValues map[string]*plugin_pb.ConfigValue, workerValues map[string]*plugin_pb.ConfigValue) Config {
+	cfg := Config{
+		Workers: shardPipelineGoroutines,
+	}
+	_ = workerValues // worker-side config form is currently empty; reserved for future per-worker tuning.
+	// Operator-declared meta-log retention. Negative or zero values stay
+	// zero so runShard falls back to maxTTL (PromotedHash dormant).
+	// Convert days->hours in int64 space before lifting to time.Duration
+	// so the unit is unambiguous.
+	if days := readInt64(adminValues, MetaLogRetentionDaysAdminKey, 0); days > 0 {
+		cfg.MetaLogRetention = time.Duration(days*24) * time.Hour
+	}
+	// Walker throttle. Negative / zero stay zero so dailyrun.runShard
+	// keeps the prior "fire every pass" semantics — important for in-
+	// repo integration tests and s3tests's sub-minute driver. Positive
+	// values throttle the steady-state walker per shard.
+	if mins := readInt64(adminValues, WalkerIntervalMinutesAdminKey, 0); mins > 0 {
+		cfg.WalkerInterval = time.Duration(mins) * time.Minute
+	}
+	return cfg
+}
+
+func readInt64(values map[string]*plugin_pb.ConfigValue, field string, fallback int64) int64 {
+	v, ok := values[field]
+	if !ok || v == nil {
+		return fallback
+	}
+	switch k := v.Kind.(type) {
+	case *plugin_pb.ConfigValue_Int64Value:
+		return k.Int64Value
+	case *plugin_pb.ConfigValue_DoubleValue:
+		return int64(k.DoubleValue)
+	case *plugin_pb.ConfigValue_StringValue:
+		return fallback
+	}
+	return fallback
+}

weed/worker/tasks/s3_lifecycle/config_test.go

@@ -0,0 +1,95 @@
+package s3_lifecycle
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+)
+
+func TestParseConfigDefaults(t *testing.T) {
+	cfg := ParseConfig(nil, nil)
+	if cfg.Workers != shardPipelineGoroutines {
+		t.Errorf("Workers default=%d, want %d", cfg.Workers, shardPipelineGoroutines)
+	}
+}
+
+func TestParseConfigIgnoresWorkerValues(t *testing.T) {
+	// Worker-side config form is currently empty. The old
+	// max_runtime_minutes knob duplicated the admin scheduler's
+	// Execution Timeout and was removed; anything in workerValues is
+	// now silently ignored. The single source of truth for the
+	// per-Execute wall-clock cap is AdminRuntimeDefaults.ExecutionTimeoutSeconds.
+	worker := map[string]*plugin_pb.ConfigValue{
+		"max_runtime_minutes": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 120}},
+	}
+	cfg := ParseConfig(nil, worker)
+	if cfg.Workers != shardPipelineGoroutines {
+		t.Errorf("Workers default=%d, want %d", cfg.Workers, shardPipelineGoroutines)
+	}
+}
+
+func TestParseConfigMetaLogRetentionDefaultsToZero(t *testing.T) {
+	// Unset key keeps MetaLogRetention at 0, which runShard treats as
+	// "no retention info supplied" and falls back to maxTTL.
+	cfg := ParseConfig(nil, nil)
+	if cfg.MetaLogRetention != 0 {
+		t.Errorf("MetaLogRetention default=%v, want 0", cfg.MetaLogRetention)
+	}
+}
+
+func TestParseConfigMetaLogRetentionDaysConvertsToDuration(t *testing.T) {
+	admin := map[string]*plugin_pb.ConfigValue{
+		MetaLogRetentionDaysAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7}},
+	}
+	cfg := ParseConfig(admin, nil)
+	if want := 7 * 24 * time.Hour; cfg.MetaLogRetention != want {
+		t.Errorf("MetaLogRetention=%v, want %v (7 days)", cfg.MetaLogRetention, want)
+	}
+}
+
+func TestParseConfigMetaLogRetentionNegativeStaysZero(t *testing.T) {
+	// A negative declaration is nonsense; stay at 0 so runShard's
+	// fallback applies rather than producing a negative window.
+	admin := map[string]*plugin_pb.ConfigValue{
+		MetaLogRetentionDaysAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: -3}},
+	}
+	cfg := ParseConfig(admin, nil)
+	if cfg.MetaLogRetention != 0 {
+		t.Errorf("negative MetaLogRetention should stay 0, got %v", cfg.MetaLogRetention)
+	}
+}
+
+func TestParseConfigWalkerIntervalDefaultsToZero(t *testing.T) {
+	// Unset key keeps WalkerInterval at 0 so dailyrun.runShard fires the
+	// walker every pass (the pre-throttle behavior the s3tests fast
+	// driver and the in-repo integration tests rely on).
+	cfg := ParseConfig(nil, nil)
+	if cfg.WalkerInterval != 0 {
+		t.Errorf("WalkerInterval default=%v, want 0", cfg.WalkerInterval)
+	}
+}
+
+func TestParseConfigWalkerIntervalMinutesConvertsToDuration(t *testing.T) {
+	admin := map[string]*plugin_pb.ConfigValue{
+		WalkerIntervalMinutesAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 90}},
+	}
+	cfg := ParseConfig(admin, nil)
+	if want := 90 * time.Minute; cfg.WalkerInterval != want {
+		t.Errorf("WalkerInterval=%v, want %v", cfg.WalkerInterval, want)
+	}
+}
+
+func TestParseConfigWalkerIntervalNegativeStaysZero(t *testing.T) {
+	// Negative declarations stay at 0 so the worker keeps "fire every
+	// pass" rather than treating the negative as past-due (which would
+	// fire every pass anyway — but via a less obvious code path that
+	// future readers would have to trace).
+	admin := map[string]*plugin_pb.ConfigValue{
+		WalkerIntervalMinutesAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: -10}},
+	}
+	cfg := ParseConfig(admin, nil)
+	if cfg.WalkerInterval != 0 {
+		t.Errorf("negative WalkerInterval should stay 0, got %v", cfg.WalkerInterval)
+	}
+}

weed/worker/tasks/s3_lifecycle/handler.go

@@ -0,0 +1,453 @@
+package s3_lifecycle
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"strconv"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/credential"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
+	pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/dailyrun"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/dispatcher"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/scheduler"
+	"golang.org/x/time/rate"
+	"google.golang.org/grpc"
+)
+
+func init() {
+	pluginworker.RegisterHandler(pluginworker.HandlerFactory{
+		JobType:  jobType,
+		Category: pluginworker.CategoryDefault,
+		Aliases:  []string{"s3-lifecycle", "s3.lifecycle", "lifecycle"},
+		Build: func(opts pluginworker.HandlerBuildOptions) (pluginworker.JobHandler, error) {
+			return NewHandler(opts.GrpcDialOption), nil
+		},
+	})
+}
+
+// Handler is the worker-side runner for S3 object lifecycle expiration.
+// One Execute call drives one bounded dailyrun.Run pass against the S3
+// endpoints discovered from the master; admin caps concurrency at one
+// job per worker so a fresh proposal only spawns a new run after the
+// prior one exits.
+type Handler struct {
+	grpcDialOption grpc.DialOption
+}
+
+func NewHandler(grpcDialOption grpc.DialOption) *Handler {
+	return &Handler{grpcDialOption: grpcDialOption}
+}
+
+func (h *Handler) Capability() *plugin_pb.JobTypeCapability {
+	return &plugin_pb.JobTypeCapability{
+		JobType:                 jobType,
+		CanDetect:               true,
+		CanExecute:              true,
+		MaxDetectionConcurrency: 1,
+		MaxExecutionConcurrency: 1,
+		DisplayName:             "S3 Lifecycle",
+		Description:             "Daily batch: scan the filer meta-log and delete objects whose lifecycle rule has fired.",
+		Weight:                  20,
+	}
+}
+
+func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor {
+	return &plugin_pb.JobTypeDescriptor{
+		JobType:           jobType,
+		DisplayName:       "S3 Lifecycle",
+		Description:       "Daily S3 object expiration scan with per-shard cursors and bounded retry.",
+		Icon:              "fas fa-recycle",
+		DescriptorVersion: 1,
+		AdminConfigForm: &plugin_pb.ConfigForm{
+			FormId:      "s3-lifecycle-admin",
+			Title:       "S3 Lifecycle",
+			Description: "Cluster-wide delete-throughput cap.",
+			Sections: []*plugin_pb.ConfigSection{
+				{
+					SectionId:   "scope",
+					Title:       "Scope",
+					Description: "Cluster-wide delete-throughput cap.",
+					Fields: []*plugin_pb.ConfigField{
+						{
+							Name:        ClusterDeletesPerSecondAdminKey,
+							Label:       "Cluster Delete Rate (per second)",
+							Description: "Cluster-wide ceiling on lifecycle delete RPCs per second, divided evenly across active s3_lifecycle workers at job-dispatch time. 0 = unlimited.",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
+						},
+						{
+							Name:        ClusterDeletesBurstAdminKey,
+							Label:       "Cluster Delete Burst",
+							Description: "Token-bucket burst capacity across the cluster (max simultaneous deletes). 0 = 2 × rate. Same allocation rule as the rate.",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
+						},
+						{
+							Name:        MetaLogRetentionDaysAdminKey,
+							Label:       "Meta-Log Retention (days)",
+							Description: "How far back the filer's meta-log subscription can reach. Rules whose TTL exceeds this run via the walker; shrinking this value will trigger a one-time recovery walk on the next run for any rule that's now too old to replay. 0 = unbounded (no partition; every rule serviced by replay).",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
+						},
+						{
+							Name:        WalkerIntervalMinutesAdminKey,
+							Label:       "Walker Interval (minutes)",
+							Description: "Minimum time between steady-state walker fires per shard. Cold-start and rule-change recovery walks ignore this — they run unconditionally. 0 = fire on every run (use when the worker is scheduled at the desired walk cadence, e.g. hourly). Set to a positive value when the worker runs at a tighter cadence than the desired walk frequency, to avoid hammering filer with a full subtree scan per run.",
+							FieldType:   plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
+							Widget:      plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
+							MinValue:    &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
+						},
+					},
+				},
+			},
+			DefaultValues: map[string]*plugin_pb.ConfigValue{
+				ClusterDeletesPerSecondAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
+				ClusterDeletesBurstAdminKey:     {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
+				MetaLogRetentionDaysAdminKey:    {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
+				WalkerIntervalMinutesAdminKey:   {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
+			},
+		},
+		// WorkerConfigForm intentionally absent — the prior
+		// "Per-Run Time Limit (minutes)" knob duplicated the admin
+		// scheduler's Execution Timeout (both wall-clock caps on the
+		// same Execute call), and operators had to keep the two values
+		// in agreement. Removed in favor of a single source of truth:
+		// AdminRuntimeDefaults.ExecutionTimeoutSeconds below.
+		AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
+			// On by default: S3 lifecycle is a standard bucket feature
+			// (PutBucketLifecycleConfiguration is part of the S3 API),
+			// and a bucket with rules set but no worker running silently
+			// retains data past its declared expiration. Operators who
+			// want the worker off can still disable it in the admin UI;
+			// the default error is "data lingers" not "worker burns CPU
+			// on empty rule sets" (the worker fast-exits with no
+			// configured rules).
+			Enabled:                  true,
+			DetectionIntervalMinutes: 24 * 60, // daily
+			DetectionTimeoutSeconds:  60,
+			MaxJobsPerDetection:      1,
+			// Effectively no per-pass wall-clock cap. Lifecycle is a
+			// scheduled batch — its natural duration is "as long as it
+			// takes to process today's events." The scheduler's global
+			// 90s default would kill every real run, and a numeric
+			// cap operators have to estimate (1h? 8h?) is a recurring
+			// footgun: too low truncates a legitimate large-bucket
+			// pass; too high makes the value meaningless.
+			//
+			// Use math.MaxInt32 seconds (~68 years) for both knobs to
+			// say "no timeout in practice" in code-review-readable form.
+			// Operators who genuinely want a cap can set one in the
+			// admin UI; the underlying context.WithTimeout machinery
+			// is unchanged.
+			ExecutionTimeoutSeconds:  math.MaxInt32,
+			JobTypeMaxRuntimeSeconds: math.MaxInt32,
+		},
+	}
+}
+
+func (h *Handler) Detect(ctx context.Context, request *plugin_pb.RunDetectionRequest, sender pluginworker.DetectionSender) error {
+	if request == nil || sender == nil {
+		return fmt.Errorf("detect: nil request or sender")
+	}
+	if request.JobType != "" && request.JobType != jobType {
+		return fmt.Errorf("job type %q is not handled by s3 lifecycle handler", request.JobType)
+	}
+	s3Endpoints := clusterS3Endpoints(request.ClusterContext)
+	if len(s3Endpoints) == 0 {
+		_ = sender.SendActivity(pluginworker.BuildDetectorActivity("skipped", "no s3 servers registered with master", nil))
+		return sender.SendComplete(&plugin_pb.DetectionComplete{JobType: jobType, Success: true})
+	}
+	filerAddresses := []string{}
+	if request.ClusterContext != nil {
+		filerAddresses = append(filerAddresses, request.ClusterContext.FilerAddresses...)
+	}
+	if len(filerAddresses) == 0 {
+		_ = sender.SendActivity(pluginworker.BuildDetectorActivity("skipped", "no filer addresses in cluster context", nil))
+		return sender.SendComplete(&plugin_pb.DetectionComplete{JobType: jobType, Success: true})
+	}
+
+	// FilerAddresses are pb.ServerAddress strings (host:httpPort.grpcPort);
+	// Execute dials filer_grpc_address verbatim, so resolve it to a gRPC address.
+	filerGrpcAddress := pb.ServerAddress(filerAddresses[0]).ToGrpcAddress()
+	proposal := &plugin_pb.JobProposal{
+		JobType:    jobType,
+		ProposalId: fmt.Sprintf("s3-lifecycle-%d", time.Now().UnixNano()),
+		Priority:   plugin_pb.JobPriority_JOB_PRIORITY_NORMAL,
+		Parameters: map[string]*plugin_pb.ConfigValue{
+			"filer_grpc_address": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: filerGrpcAddress}},
+		},
+	}
+	if err := sender.SendProposals(&plugin_pb.DetectionProposals{
+		JobType:   jobType,
+		Proposals: []*plugin_pb.JobProposal{proposal},
+	}); err != nil {
+		return err
+	}
+	return sender.SendComplete(&plugin_pb.DetectionComplete{
+		JobType:        jobType,
+		Success:        true,
+		TotalProposals: 1,
+	})
+}
+
+func (h *Handler) Execute(ctx context.Context, request *plugin_pb.ExecuteJobRequest, sender pluginworker.ExecutionSender) error {
+	if request == nil || request.Job == nil || sender == nil {
+		return fmt.Errorf("execute: nil request/job/sender")
+	}
+	if request.Job.JobType != "" && request.Job.JobType != jobType {
+		return fmt.Errorf("job type %q is not handled by s3 lifecycle handler", request.Job.JobType)
+	}
+	cfg := ParseConfig(request.GetAdminConfigValues(), request.GetWorkerConfigValues())
+	s3Endpoints := clusterS3Endpoints(request.ClusterContext)
+	if len(s3Endpoints) == 0 {
+		return fmt.Errorf("execute: no s3 servers registered with master")
+	}
+	filerAddress := readString(request.Job.Parameters, "filer_grpc_address", "")
+	if filerAddress == "" {
+		return fmt.Errorf("execute: missing filer_grpc_address in job parameters")
+	}
+
+	// Run lifetime is bounded by the scheduler's Execution Timeout
+	// (admin UI). The handler used to wrap ctx in another
+	// context.WithTimeout(cfg.MaxRuntime), but that doubled the
+	// concept — both knobs were wall-clock caps on the same Execute
+	// call, and the smaller one always won (typically the 90s
+	// scheduler default would clobber a 60-min worker setting).
+	// Single source of truth is now AdminRuntimeDefaults.ExecutionTimeoutSeconds.
+	runCtx := ctx
+
+	_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
+		JobId: request.Job.JobId, JobType: jobType,
+		State: plugin_pb.JobState_JOB_STATE_RUNNING, Stage: "starting",
+		Message: fmt.Sprintf("scheduler workers=%d s3=%v", cfg.Workers, s3Endpoints),
+	})
+
+	dialCtx, dialCancel := context.WithTimeout(runCtx, 30*time.Second)
+	filerConn, err := pb.GrpcDial(dialCtx, filerAddress, false, h.grpcDialOption)
+	dialCancel()
+	if err != nil {
+		return fmt.Errorf("dial filer %s: %w", filerAddress, err)
+	}
+	defer filerConn.Close()
+	filerClient := filer_pb.NewSeaweedFilerClient(filerConn)
+
+	bucketsPath, err := lookupBucketsPath(runCtx, filerClient)
+	if err != nil {
+		return fmt.Errorf("buckets path: %w", err)
+	}
+
+	dialCtx, dialCancel = context.WithTimeout(runCtx, 30*time.Second)
+	s3Conn, err := pb.GrpcDial(dialCtx, s3Endpoints[0], false, h.grpcDialOption)
+	dialCancel()
+	if err != nil {
+		return fmt.Errorf("dial s3 %s: %w", s3Endpoints[0], err)
+	}
+	defer s3Conn.Close()
+	rpc := s3_lifecycle_pb.NewSeaweedS3LifecycleInternalClient(s3Conn)
+
+	if err := h.executeDailyReplay(runCtx, request, bucketsPath, filerClient, rpc, cfg, sender); err != nil {
+		return err
+	}
+
+	return sendSuccessCompletion(request, sender)
+}
+
+const dailyReplaySuccessSummary = "s3 lifecycle daily replay completed"
+
+// JobCompleted must set JobType to the handler's jobType constant;
+// routed requests may leave request.Job.JobType empty, and the admin
+// ignores completions with empty JobType.
+func sendSuccessCompletion(request *plugin_pb.ExecuteJobRequest, sender pluginworker.ExecutionSender) error {
+	return sender.SendCompleted(&plugin_pb.JobCompleted{
+		JobId:   request.Job.JobId,
+		JobType: jobType,
+		Success: true,
+		Result: &plugin_pb.JobResult{
+			Summary: dailyReplaySuccessSummary,
+		},
+	})
+}
+
+// executeDailyReplay runs one bounded daily-replay pass via
+// dailyrun.Run. The walker fires inside runShard on rule-content edits
+// and against the steady-state walk view; all rule kinds are serviced.
+func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.ExecuteJobRequest, bucketsPath string, filerClient filer_pb.SeaweedFilerClient, rpc s3_lifecycle_pb.SeaweedS3LifecycleInternalClient, cfg Config, sender pluginworker.ExecutionSender) error {
+	eng := engine.New()
+	inputs, parseErrors, err := scheduler.LoadCompileInputs(ctx, filerClient, bucketsPath)
+	if err != nil {
+		return fmt.Errorf("daily_replay: load lifecycle inputs: %w", err)
+	}
+	for _, pe := range parseErrors {
+		glog.V(1).Infof("daily_replay: %s: %v", pe.Bucket, pe.Err)
+	}
+	eng.Compile(inputs, engine.CompileOptions{PriorStates: scheduler.AllActivePriorStates(inputs)})
+
+	shards := make([]int, 0, s3lifecycle.ShardCount)
+	for i := 0; i < s3lifecycle.ShardCount; i++ {
+		shards = append(shards, i)
+	}
+
+	limiter, limiterDesc := buildLimiterFromClusterContext(request.GetClusterContext())
+
+	// Reuse one LifecycleClient across the replay drain and the
+	// walker's per-entry dispatch.
+	client := lifecycleRPCAdapter{c: rpc}
+
+	// Bucket list for the walker — derived from inputs so it matches
+	// the snapshot the engine compiled.
+	buckets := make([]string, 0, len(inputs))
+	for _, in := range inputs {
+		if in.Bucket != "" {
+			buckets = append(buckets, in.Bucket)
+		}
+	}
+	walkerListFn := dailyrun.FilerListFunc(filerClient, bucketsPath)
+	// Share the limiter with processMatches so walker + replay can't
+	// combine to burst past the cluster cap.
+	walkerDispatch := &dailyrun.WalkerDispatcher{Client: client, Limiter: limiter}
+	walker := dailyrun.WalkerFunc(func(walkCtx context.Context, view *engine.Snapshot, shardID int) error {
+		return dailyrun.WalkBuckets(walkCtx, view, shardID, buckets, walkerListFn, walkerDispatch)
+	})
+
+	_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
+		JobId: request.Job.JobId, JobType: jobType,
+		State: plugin_pb.JobState_JOB_STATE_RUNNING, Stage: "starting",
+		Message: fmt.Sprintf("daily_replay shards=%d workers=%d rate=%s buckets=%d walker=on",
+			len(shards), cfg.Workers, limiterDesc, len(buckets)),
+	})
+
+	runErr := dailyrun.Run(ctx, dailyrun.Config{
+		Shards:          shards,
+		BucketsPath:     bucketsPath,
+		Engine:          eng,
+		FilerClient:     filerClient,
+		Client:          client,
+		Persister:       &dailyrun.FilerCursorPersister{Store: dispatcher.NewFilerStoreClient(filerClient)},
+		Lister:          dispatcher.NewFilerSiblingLister(filerClient, bucketsPath),
+		Workers:         cfg.Workers,
+		Limiter:         limiter,
+		RetentionWindow: cfg.MetaLogRetention,
+		Walker:          walker,
+		WalkerInterval:  cfg.WalkerInterval,
+		ClientName:      "worker-s3-lifecycle-daily",
+	})
+	if runErr != nil {
+		glog.Warningf("daily_replay: %v", runErr)
+		return runErr
+	}
+	return nil
+}
+
+// buildLimiterFromClusterContext parses the per-worker share the admin
+// wrote into ClusterContext.Metadata (see weed/admin/plugin/plugin.go's
+// s3_lifecycle injection) and returns a rate.Limiter, or nil when no
+// rate cap applies. The description string is for the JobProgressUpdate
+// so operators can see "rate=unlimited" / "rate=12.5/s burst=25" in
+// the activity log.
+//
+// Tolerant of missing keys, empty strings, malformed numbers, and
+// non-positive values — all treated as "no limit" rather than failing
+// the run. The admin allocator is the single point that decides whether
+// to populate these keys; the worker doesn't second-guess.
+func buildLimiterFromClusterContext(cc *plugin_pb.ClusterContext) (*rate.Limiter, string) {
+	if cc == nil || cc.Metadata == nil {
+		return nil, "unlimited"
+	}
+	rps, ok := parsePositiveFloat(cc.Metadata[MetadataKeyDeletesPerSecond])
+	if !ok {
+		return nil, "unlimited"
+	}
+	burst, _ := parsePositiveInt(cc.Metadata[MetadataKeyDeletesBurst])
+	if burst <= 0 {
+		// Sensible default: enough headroom for one tick's worth of
+		// throughput. Caller may also supply 0 to opt into this default.
+		burst = int(rps * 2)
+		if burst < 1 {
+			burst = 1
+		}
+	}
+	return rate.NewLimiter(rate.Limit(rps), burst), fmt.Sprintf("%.3g/s burst=%d", rps, burst)
+}
+
+func parsePositiveFloat(s string) (float64, bool) {
+	if s == "" {
+		return 0, false
+	}
+	v, err := strconv.ParseFloat(s, 64)
+	if err != nil || v <= 0 {
+		return 0, false
+	}
+	return v, true
+}
+
+func parsePositiveInt(s string) (int, bool) {
+	if s == "" {
+		return 0, false
+	}
+	v, err := strconv.Atoi(s)
+	if err != nil || v <= 0 {
+		return 0, false
+	}
+	return v, true
+}
+
+// clusterS3Endpoints returns the master-discovered S3 gRPC addresses for the
+// cluster. The handler dials the first reachable one; the master refreshes
+// the list on KeepConnected so a stale entry self-heals on the next run.
+func clusterS3Endpoints(cc *plugin_pb.ClusterContext) []string {
+	if cc == nil {
+		return nil
+	}
+	out := make([]string, 0, len(cc.S3GrpcAddresses))
+	for _, addr := range cc.S3GrpcAddresses {
+		if addr != "" {
+			out = append(out, addr)
+		}
+	}
+	return out
+}
+
+type lifecycleRPCAdapter struct {
+	c s3_lifecycle_pb.SeaweedS3LifecycleInternalClient
+}
+
+func (a lifecycleRPCAdapter) LifecycleDelete(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
+	ctx, _ = credential.WithS3InternalAdminAuth(ctx)
+	return a.c.LifecycleDelete(ctx, req)
+}
+
+func lookupBucketsPath(ctx context.Context, client filer_pb.SeaweedFilerClient) (string, error) {
+	resp, err := client.GetFilerConfiguration(ctx, &filer_pb.GetFilerConfigurationRequest{})
+	if err != nil {
+		return "", err
+	}
+	if path := resp.GetDirBuckets(); path != "" {
+		return path, nil
+	}
+	return "/buckets", nil
+}
+
+func readString(values map[string]*plugin_pb.ConfigValue, field, fallback string) string {
+	v, ok := values[field]
+	if !ok || v == nil {
+		return fallback
+	}
+	if k, ok := v.Kind.(*plugin_pb.ConfigValue_StringValue); ok {
+		return k.StringValue
+	}
+	return fallback
+}
+
+var _ pluginworker.JobHandler = (*Handler)(nil)

weed/worker/tasks/s3_lifecycle/handler_test.go

@@ -0,0 +1,550 @@
+package s3_lifecycle
+
+import (
+	"context"
+	"errors"
+	"math"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+)
+
+// Tests cover the worker-handler surface that runs without a live filer
+// or S3 server: pure helpers (clusterS3Endpoints, readString),
+// Capability/Descriptor sanity, and the Detect input-validation /
+// skip-activity / proposal paths driven by a recorder sender.
+
+// recordingSender captures everything the handler sends so each Detect
+// case can assert on activities, proposals, and completion.
+type recordingSender struct {
+	proposals  []*plugin_pb.DetectionProposals
+	completes  []*plugin_pb.DetectionComplete
+	activities []*plugin_pb.ActivityEvent
+	// errOn forces the named send to fail; lets tests cover the
+	// SendComplete error-propagation path without a full transport stub.
+	errOn map[string]error
+}
+
+func (r *recordingSender) SendProposals(p *plugin_pb.DetectionProposals) error {
+	if err := r.errOn["proposals"]; err != nil {
+		return err
+	}
+	r.proposals = append(r.proposals, p)
+	return nil
+}
+func (r *recordingSender) SendComplete(c *plugin_pb.DetectionComplete) error {
+	if err := r.errOn["complete"]; err != nil {
+		return err
+	}
+	r.completes = append(r.completes, c)
+	return nil
+}
+func (r *recordingSender) SendActivity(a *plugin_pb.ActivityEvent) error {
+	if err := r.errOn["activity"]; err != nil {
+		return err
+	}
+	r.activities = append(r.activities, a)
+	return nil
+}
+
+// ---------- clusterS3Endpoints ----------
+
+func TestClusterS3Endpoints_NilContext(t *testing.T) {
+	// A nil ClusterContext occurs in early-bootstrap detect calls; the
+	// handler must return an empty slice rather than panic.
+	assert.Nil(t, clusterS3Endpoints(nil))
+}
+
+func TestClusterS3Endpoints_EmptyList(t *testing.T) {
+	assert.Empty(t, clusterS3Endpoints(&plugin_pb.ClusterContext{}))
+}
+
+func TestClusterS3Endpoints_FiltersEmptyEntries(t *testing.T) {
+	// An empty-string address represents a stale registry entry the
+	// master is about to evict; never dial that. Order of the surviving
+	// entries must be preserved so the handler dials a deterministic
+	// host across detect runs.
+	cc := &plugin_pb.ClusterContext{S3GrpcAddresses: []string{"s3a:8333", "", "s3b:8333", ""}}
+	assert.Equal(t, []string{"s3a:8333", "s3b:8333"}, clusterS3Endpoints(cc))
+}
+
+func TestClusterS3Endpoints_AllValid(t *testing.T) {
+	cc := &plugin_pb.ClusterContext{S3GrpcAddresses: []string{"s3a:8333", "s3b:8333"}}
+	assert.Equal(t, []string{"s3a:8333", "s3b:8333"}, clusterS3Endpoints(cc))
+}
+
+// ---------- readString ----------
+
+func TestReadString_MissingKeyReturnsFallback(t *testing.T) {
+	got := readString(map[string]*plugin_pb.ConfigValue{}, "missing", "fallback")
+	assert.Equal(t, "fallback", got)
+}
+
+func TestReadString_NilValueReturnsFallback(t *testing.T) {
+	got := readString(map[string]*plugin_pb.ConfigValue{"k": nil}, "k", "fallback")
+	assert.Equal(t, "fallback", got)
+}
+
+func TestReadString_WrongKindReturnsFallback(t *testing.T) {
+	// Configs are typed; an Int64 in a string slot is a writer-side bug
+	// the handler must tolerate rather than panic on a type assertion.
+	values := map[string]*plugin_pb.ConfigValue{
+		"k": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 42}},
+	}
+	assert.Equal(t, "fallback", readString(values, "k", "fallback"))
+}
+
+func TestReadString_StringValueReturned(t *testing.T) {
+	values := map[string]*plugin_pb.ConfigValue{
+		"k": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "expected"}},
+	}
+	assert.Equal(t, "expected", readString(values, "k", "fallback"))
+}
+
+// ---------- Capability ----------
+
+func TestCapability_AdvertisesJobType(t *testing.T) {
+	h := NewHandler(nil)
+	cap := h.Capability()
+	require.NotNil(t, cap)
+	assert.Equal(t, jobType, cap.JobType)
+	assert.True(t, cap.CanDetect)
+	assert.True(t, cap.CanExecute)
+	// Admin caps concurrency at one job per worker; a rebuild of the
+	// scheduler must wait for the prior one to exit.
+	assert.Equal(t, int32(1), cap.MaxDetectionConcurrency)
+	assert.Equal(t, int32(1), cap.MaxExecutionConcurrency)
+}
+
+// ---------- Detect ----------
+
+func TestDetect_NilRequestErrors(t *testing.T) {
+	h := NewHandler(nil)
+	r := &recordingSender{}
+	err := h.Detect(context.Background(), nil, r)
+	require.Error(t, err)
+	assert.Empty(t, r.proposals)
+	assert.Empty(t, r.completes)
+}
+
+func TestDetect_NilSenderErrors(t *testing.T) {
+	h := NewHandler(nil)
+	err := h.Detect(context.Background(), &plugin_pb.RunDetectionRequest{}, nil)
+	require.Error(t, err)
+}
+
+func TestDetect_WrongJobTypeErrors(t *testing.T) {
+	// A request routed to this handler with a foreign JobType is the
+	// admin's bug; surface as an error so it's visible rather than
+	// silently emitting a bogus proposal.
+	h := NewHandler(nil)
+	r := &recordingSender{}
+	err := h.Detect(context.Background(), &plugin_pb.RunDetectionRequest{JobType: "different_job"}, r)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "different_job")
+}
+
+func TestDetect_NoS3EndpointsCompletesWithSkipActivity(t *testing.T) {
+	// A cluster with no S3 servers registered yet must not spawn the
+	// scheduler; emit a "skipped" activity for operator visibility and
+	// complete with success so the admin doesn't classify as a failure.
+	h := NewHandler(nil)
+	r := &recordingSender{}
+	err := h.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
+		JobType:        jobType,
+		ClusterContext: &plugin_pb.ClusterContext{},
+	}, r)
+	require.NoError(t, err)
+	assert.Empty(t, r.proposals, "no S3 endpoints must not yield a proposal")
+	require.Len(t, r.activities, 1)
+	assert.Equal(t, "skipped", r.activities[0].Stage)
+	assert.Contains(t, r.activities[0].Message, "no s3 servers")
+	require.Len(t, r.completes, 1)
+	assert.True(t, r.completes[0].Success, "skip is success, not failure")
+	assert.Equal(t, jobType, r.completes[0].JobType)
+}
+
+func TestDetect_NoFilerAddressesCompletesWithSkipActivity(t *testing.T) {
+	h := NewHandler(nil)
+	r := &recordingSender{}
+	err := h.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
+		JobType: jobType,
+		ClusterContext: &plugin_pb.ClusterContext{
+			S3GrpcAddresses: []string{"s3a:8333"},
+			// no FilerAddresses
+		},
+	}, r)
+	require.NoError(t, err)
+	assert.Empty(t, r.proposals)
+	require.Len(t, r.activities, 1)
+	assert.Equal(t, "skipped", r.activities[0].Stage)
+	assert.Contains(t, r.activities[0].Message, "no filer addresses")
+	require.Len(t, r.completes, 1)
+	assert.True(t, r.completes[0].Success)
+}
+
+func TestDetect_HappyPathProposesOneJobWithFirstFilerAddress(t *testing.T) {
+	// Detect must propose exactly one job that targets the first filer
+	// address in the cluster context; the master refreshes the list so
+	// a stale entry self-heals on the next run.
+	h := NewHandler(nil)
+	r := &recordingSender{}
+	err := h.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
+		JobType: jobType,
+		ClusterContext: &plugin_pb.ClusterContext{
+			S3GrpcAddresses: []string{"s3a:8333"},
+			// pb.ServerAddress dual-port form; the gRPC port is pinned off the
+			// +10000 convention so a raw-forwarding regression resurfaces here.
+			FilerAddresses: []string{"filer-a:8888.18890", "filer-b:8888.18890"},
+		},
+	}, r)
+	require.NoError(t, err)
+	assert.Empty(t, r.activities, "happy path emits no skip activity")
+
+	require.Len(t, r.proposals, 1)
+	require.Len(t, r.proposals[0].Proposals, 1)
+	prop := r.proposals[0].Proposals[0]
+	assert.Equal(t, jobType, prop.JobType)
+	assert.NotEmpty(t, prop.ProposalId, "proposal id must be unique-per-run")
+	require.Contains(t, prop.Parameters, "filer_grpc_address")
+	val := prop.Parameters["filer_grpc_address"].GetStringValue()
+	assert.Equal(t, "filer-a:18890", val, "first filer resolved to its gRPC port, not host:28890")
+
+	require.Len(t, r.completes, 1)
+	assert.True(t, r.completes[0].Success)
+	assert.Equal(t, int32(1), r.completes[0].TotalProposals)
+}
+
+func TestDetect_EmptyJobTypeAccepted(t *testing.T) {
+	// Detect is sometimes invoked with an unset JobType (broadcast
+	// detect); the handler must accept and behave as if it matched.
+	h := NewHandler(nil)
+	r := &recordingSender{}
+	err := h.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
+		ClusterContext: &plugin_pb.ClusterContext{
+			S3GrpcAddresses: []string{"s3a:8333"},
+			FilerAddresses:  []string{"f:8888.18890"},
+		},
+	}, r)
+	require.NoError(t, err)
+	require.Len(t, r.proposals, 1)
+}
+
+func TestDetect_PropagatesProposalsSendError(t *testing.T) {
+	// SendProposals failing must propagate; otherwise the worker would
+	// silently report success despite never delivering the proposal.
+	h := NewHandler(nil)
+	want := errors.New("transport down")
+	r := &recordingSender{errOn: map[string]error{"proposals": want}}
+	err := h.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
+		JobType: jobType,
+		ClusterContext: &plugin_pb.ClusterContext{
+			S3GrpcAddresses: []string{"s3a:8333"},
+			FilerAddresses:  []string{"f:8888.18890"},
+		},
+	}, r)
+	assert.ErrorIs(t, err, want)
+	assert.Empty(t, r.proposals)
+	assert.Empty(t, r.completes, "complete must not fire when proposals fail")
+}
+
+func TestDetect_PropagatesCompleteSendError(t *testing.T) {
+	// SendComplete failing must propagate; otherwise the worker would
+	// report success to the admin despite the completion signal never
+	// landing. Proposals went out before the failure, so they remain in
+	// the recorder.
+	h := NewHandler(nil)
+	want := errors.New("transport down")
+	r := &recordingSender{errOn: map[string]error{"complete": want}}
+	err := h.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
+		JobType: jobType,
+		ClusterContext: &plugin_pb.ClusterContext{
+			S3GrpcAddresses: []string{"s3a:8333"},
+			FilerAddresses:  []string{"f:8888.18890"},
+		},
+	}, r)
+	assert.ErrorIs(t, err, want)
+	assert.Len(t, r.proposals, 1, "proposals send before complete and remain recorded")
+	assert.Empty(t, r.completes)
+}
+
+// ---------- Descriptor ----------
+
+func TestDescriptor_BasicShape(t *testing.T) {
+	// Sanity-check the Descriptor's public-facing identifiers so a
+	// rename in handler.go doesn't silently break the admin UI without
+	// an admin-side change too.
+	h := NewHandler(nil)
+	d := h.Descriptor()
+	require.NotNil(t, d)
+	assert.Equal(t, jobType, d.JobType)
+	assert.NotEmpty(t, d.DisplayName)
+	assert.NotEmpty(t, d.Description)
+	assert.Greater(t, d.DescriptorVersion, uint32(0), "descriptor version must be positive (admins use it for compat)")
+}
+
+func TestDescriptor_AdminConfigFormHasNoWorkersField(t *testing.T) {
+	// "workers" used to be an admin form field controlling in-process
+	// pipeline goroutines. It's per-worker tuning, not a cluster-wide
+	// scope concern, so it was removed from the form. ParseConfig hard-
+	// codes cfg.Workers from shardPipelineGoroutines instead.
+	h := NewHandler(nil)
+	d := h.Descriptor()
+	require.NotNil(t, d.AdminConfigForm)
+	for _, sec := range d.AdminConfigForm.Sections {
+		for _, f := range sec.Fields {
+			assert.NotEqual(t, "workers", f.Name, "admin form must NOT expose the in-memory pipeline-goroutine knob")
+		}
+	}
+	_, hasDefault := d.AdminConfigForm.DefaultValues["workers"]
+	assert.False(t, hasDefault, "DefaultValues must NOT include 'workers' (form field removed)")
+}
+
+func TestDescriptor_WorkerConfigFormIsAbsent(t *testing.T) {
+	// "Per-Run Time Limit (minutes)" was the only worker-side knob and
+	// duplicated the admin scheduler's Execution Timeout (both are
+	// wall-clock caps on the same Execute call). Removed in favor of
+	// AdminRuntimeDefaults.ExecutionTimeoutSeconds — single source of
+	// truth. A WorkerConfigForm with no fields would render as an
+	// empty section in the admin UI; drop the form entirely.
+	h := NewHandler(nil)
+	d := h.Descriptor()
+	assert.Nil(t, d.WorkerConfigForm,
+		"WorkerConfigForm should be nil now that max_runtime_minutes is gone; if you re-add a worker-side knob, restore the form and pin it here")
+}
+
+func TestDescriptor_AdminRuntimeDefaultsHaveNoTimeoutInPractice(t *testing.T) {
+	// Lifecycle is a scheduled batch whose natural duration is "as
+	// long as today's events take." The scheduler's global 90s default
+	// would kill every real run, and a numeric cap operators have to
+	// estimate is a footgun (too low truncates a legitimate
+	// large-bucket pass; too high makes the value meaningless). Declare
+	// math.MaxInt32 seconds for both knobs to say "no timeout in
+	// practice" in a code-review-readable way. A future change that
+	// tightens the cap should fail this test so the choice is
+	// re-examined consciously.
+	h := NewHandler(nil)
+	d := h.Descriptor()
+	require.NotNil(t, d.AdminRuntimeDefaults)
+	assert.Equal(t, int32(math.MaxInt32), d.AdminRuntimeDefaults.ExecutionTimeoutSeconds,
+		"ExecutionTimeoutSeconds should be effectively unlimited; the scheduler's 90s default would otherwise clobber the worker mid-run")
+	assert.Equal(t, int32(math.MaxInt32), d.AdminRuntimeDefaults.JobTypeMaxRuntimeSeconds,
+		"JobTypeMaxRuntimeSeconds is the per-pass budget — keep aligned with ExecutionTimeoutSeconds")
+}
+
+func TestDescriptor_AdminRuntimeDefaultsDailyCadence(t *testing.T) {
+	// Lifecycle is a daily batch; the admin must default to a 24-hour
+	// detection interval so cron pressure doesn't escalate. Bound the
+	// detection timeout so a stuck detect can't pin a worker slot
+	// indefinitely. Max 1 job per detection = scheduler runs alone.
+	h := NewHandler(nil)
+	d := h.Descriptor()
+	require.NotNil(t, d.AdminRuntimeDefaults)
+	assert.Equal(t, int32(24*60), d.AdminRuntimeDefaults.DetectionIntervalMinutes)
+	assert.Equal(t, int32(60), d.AdminRuntimeDefaults.DetectionTimeoutSeconds, "60s timeout caps a stuck detect at one minute")
+	assert.Equal(t, int32(1), d.AdminRuntimeDefaults.MaxJobsPerDetection)
+}
+
+func TestDescriptor_AdminRuntimeDefaultsEnabledByDefault(t *testing.T) {
+	// S3 lifecycle is a standard bucket feature — operators set
+	// PutBucketLifecycleConfiguration expecting it to actually fire.
+	// Default-off would silently retain data past declared expiration
+	// until an operator notices and flips it on. Document the choice
+	// here so a future change to disabled-by-default fails the test
+	// and surfaces a conscious revisit.
+	h := NewHandler(nil)
+	d := h.Descriptor()
+	require.NotNil(t, d.AdminRuntimeDefaults)
+	assert.True(t, d.AdminRuntimeDefaults.Enabled,
+		"s3_lifecycle defaults must enable the scheduler so configured rules fire without operator opt-in")
+}
+
+// ---------- Execute ----------
+
+// recordingExecSender captures Execute-side messages. The Execute path
+// dials gRPC after passing validation, so these tests only exercise
+// the validation surface that errors out before any dial — proving
+// the handler refuses malformed jobs early instead of waiting on a
+// 30s dial timeout.
+type recordingExecSender struct {
+	progress     []*plugin_pb.JobProgressUpdate
+	completed    []*plugin_pb.JobCompleted
+	completedErr error
+}
+
+func (r *recordingExecSender) SendProgress(p *plugin_pb.JobProgressUpdate) error {
+	r.progress = append(r.progress, p)
+	return nil
+}
+func (r *recordingExecSender) SendCompleted(c *plugin_pb.JobCompleted) error {
+	if r.completedErr != nil {
+		return r.completedErr
+	}
+	r.completed = append(r.completed, c)
+	return nil
+}
+
+func TestExecute_NilRequestErrors(t *testing.T) {
+	h := NewHandler(nil)
+	err := h.Execute(context.Background(), nil, &recordingExecSender{})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "nil")
+}
+
+func TestExecute_NilJobErrors(t *testing.T) {
+	// A non-nil request with nil Job is a writer-side bug; refuse it
+	// rather than panic dereferencing request.Job.JobType.
+	h := NewHandler(nil)
+	err := h.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{}, &recordingExecSender{})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "nil")
+}
+
+func TestExecute_NilSenderErrors(t *testing.T) {
+	h := NewHandler(nil)
+	err := h.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
+		Job: &plugin_pb.JobSpec{JobType: jobType},
+	}, nil)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "nil")
+}
+
+func TestExecute_WrongJobTypeErrors(t *testing.T) {
+	// A foreign job type routed to this handler is the admin's bug;
+	// surface as an error rather than running a bogus scheduler.
+	h := NewHandler(nil)
+	err := h.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
+		Job: &plugin_pb.JobSpec{JobType: "different_job"},
+	}, &recordingExecSender{})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "different_job")
+}
+
+func TestExecute_NoS3EndpointsErrors(t *testing.T) {
+	// Detect emits a "skipped" activity for this case; Execute is
+	// stricter — the admin shouldn't have routed an Execute request
+	// without S3 endpoints, so error out instead of silently no-oping.
+	h := NewHandler(nil)
+	err := h.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
+		Job:            &plugin_pb.JobSpec{JobType: jobType},
+		ClusterContext: &plugin_pb.ClusterContext{}, // no S3GrpcAddresses
+	}, &recordingExecSender{})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "no s3 servers")
+}
+
+func TestExecute_MissingFilerAddressErrors(t *testing.T) {
+	// filer_grpc_address is set by Detect when it builds the proposal;
+	// missing it means the proposal was tampered with or the admin
+	// dropped the parameter. Refuse rather than dial nothing.
+	h := NewHandler(nil)
+	err := h.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
+		Job: &plugin_pb.JobSpec{
+			JobType:    jobType,
+			Parameters: map[string]*plugin_pb.ConfigValue{}, // no filer_grpc_address
+		},
+		ClusterContext: &plugin_pb.ClusterContext{
+			S3GrpcAddresses: []string{"s3a:8333"},
+		},
+	}, &recordingExecSender{})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "filer_grpc_address")
+}
+
+func TestExecute_EmptyJobTypeAccepted(t *testing.T) {
+	// Same convention as Detect: an empty JobType is broadcast routing
+	// and must be accepted. The handler then errors at the next
+	// validation step (no S3 endpoints) rather than at the type check.
+	h := NewHandler(nil)
+	err := h.Execute(context.Background(), &plugin_pb.ExecuteJobRequest{
+		Job:            &plugin_pb.JobSpec{}, // empty JobType
+		ClusterContext: &plugin_pb.ClusterContext{},
+	}, &recordingExecSender{})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "no s3 servers", "validation flowed past the type check")
+}
+
+// ---------- sendSuccessCompletion ----------
+
+func TestSendSuccessCompletion_EmitsCanonicalCompletion(t *testing.T) {
+	sender := &recordingExecSender{}
+	request := &plugin_pb.ExecuteJobRequest{
+		Job: &plugin_pb.JobSpec{JobId: "job-123"}, // empty JobType (broadcast routing)
+	}
+
+	require.NoError(t, sendSuccessCompletion(request, sender))
+
+	require.Len(t, sender.completed, 1, "exactly one completion must be sent")
+	c := sender.completed[0]
+	assert.Equal(t, jobType, c.JobType, "completion must use the canonical jobType, not request.Job.JobType")
+	assert.NotEmpty(t, c.JobType, "an empty JobType is dropped by the admin")
+	assert.Equal(t, "job-123", c.JobId, "completion must echo the original JobId")
+	assert.True(t, c.Success, "a clean replay is a success completion")
+	require.NotNil(t, c.Result)
+	assert.NotEmpty(t, c.Result.Summary, "admin UI surfaces a non-empty Result.Summary")
+}
+
+func TestSendSuccessCompletion_PropagatesSendError(t *testing.T) {
+	sentinel := errors.New("stream closed")
+	sender := &recordingExecSender{completedErr: sentinel}
+	request := &plugin_pb.ExecuteJobRequest{Job: &plugin_pb.JobSpec{JobId: "job-456"}}
+
+	err := sendSuccessCompletion(request, sender)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, sentinel)
+}
+
+// ---------- lookupBucketsPath ----------
+
+// stubFilerConfigClient implements filer_pb.SeaweedFilerClient for the
+// GetFilerConfiguration call lookupBucketsPath relies on; methods that
+// aren't named here would panic if called, which the tests rely on to
+// keep the surface narrow.
+type stubFilerConfigClient struct {
+	filer_pb.SeaweedFilerClient
+	resp *filer_pb.GetFilerConfigurationResponse
+	err  error
+}
+
+func (c *stubFilerConfigClient) GetFilerConfiguration(_ context.Context, _ *filer_pb.GetFilerConfigurationRequest, _ ...grpc.CallOption) (*filer_pb.GetFilerConfigurationResponse, error) {
+	if c.err != nil {
+		return nil, c.err
+	}
+	return c.resp, nil
+}
+
+func TestLookupBucketsPath_PropagatesGRPCError(t *testing.T) {
+	// A failed config lookup must surface; without it, Execute would
+	// proceed to dial S3 with an empty buckets path and quietly never
+	// dispatch anything.
+	want := errors.New("filer down")
+	got, err := lookupBucketsPath(context.Background(), &stubFilerConfigClient{err: want})
+	assert.ErrorIs(t, err, want)
+	assert.Empty(t, got)
+}
+
+func TestLookupBucketsPath_UsesConfiguredDirBuckets(t *testing.T) {
+	// When the filer reports a non-empty DirBuckets, the worker honors
+	// it. Operators with a non-default layout (e.g. "/data/buckets")
+	// can't be routed to "/buckets".
+	got, err := lookupBucketsPath(context.Background(), &stubFilerConfigClient{
+		resp: &filer_pb.GetFilerConfigurationResponse{DirBuckets: "/data/buckets"},
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "/data/buckets", got)
+}
+
+func TestLookupBucketsPath_EmptyDirBucketsFallsBackToDefault(t *testing.T) {
+	// Filer doesn't always populate DirBuckets (older configs); the
+	// helper falls back to the documented default "/buckets" rather
+	// than returning an empty path that would force-route to root.
+	got, err := lookupBucketsPath(context.Background(), &stubFilerConfigClient{
+		resp: &filer_pb.GetFilerConfigurationResponse{DirBuckets: ""},
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "/buckets", got)
+}