snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/s3api/s3lifecycle/dailyrun/cursor.go

@@ -0,0 +1,163 @@
+// Package dailyrun implements the daily-replay s3 lifecycle worker
+// described in weed/s3api/s3lifecycle/DESIGN.md. One pass per day per
+// shard reads the meta-log forward from the persisted cursor, drains
+// every event whose due_time is past now, and exits — replacing the
+// streaming + heap pipeline with a bounded, idempotent scan.
+//
+// Phase 2 (this file's first version): replay-only. Buckets whose
+// compiled rules include walker-bound action kinds (ExpirationDate,
+// ExpiredDeleteMarker, NewerNoncurrent) or any scan_only promotion are
+// refused with a typed error so flipping the algorithm flag is a loud
+// failure rather than silent data loss. Phase 4 wires the walker and
+// the recovery branch.
+package dailyrun
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/dispatcher"
+)
+
+// CursorDir is the filer directory holding per-shard daily-replay
+// cursor files. Kept distinct from dispatcher.CursorDir so a deployment
+// running both algorithms during cutover doesn't have one trample the
+// other's persisted state.
+const CursorDir = "/etc/s3/lifecycle/daily-cursors"
+
+// cursorFileVersion bumps when the on-disk shape changes. Phase 2
+// writes version 1; Phase 4 will continue writing version 1 since the
+// schema (TsNs + RuleSetHash + PromotedHash) is already final.
+const cursorFileVersion = 1
+
+// Cursor captures everything daily_run needs to decide whether to
+// recover (Phase 4) or continue steady-state. TsNs is the latest
+// meta-log event whose Matches all dispatched successfully (or as
+// NOOP_RESOLVED); RuleSetHash is the content hash of the replay-eligible
+// rules from the run that wrote this cursor; PromotedHash is the hash
+// of replay-eligible rules currently in scan_only. LastWalkedNs is the
+// runNow of the most recent successful steady-state / empty-replay
+// walker fire on this shard — runShard uses it together with
+// cfg.WalkerInterval to throttle the walker independently of Run()
+// invocation frequency (so a worker driven by a 2s test ticker doesn't
+// crush filer with a full bucket walk per tick).
+//
+// In Phase 2 PromotedHash is always the empty hash because the walker
+// path isn't wired yet — any rule that would land in walk causes the
+// run to refuse. Phase 4 starts writing a real value.
+type Cursor struct {
+	TsNs         int64
+	RuleSetHash  [32]byte
+	PromotedHash [32]byte
+	LastWalkedNs int64
+}
+
+// cursorFile is the on-disk JSON shape. Bytes are base64-encoded by
+// encoding/json automatically.
+//
+// last_walked_ns is omitempty so cursor files written before the field
+// existed still decode cleanly: an older file has no last_walked_ns,
+// which marshals back into LastWalkedNs=0, which runShard treats as
+// "never walked steady-state" and so the next pass fires the walker
+// (the same path a cold-start cursor takes). No version bump needed.
+type cursorFile struct {
+	Version      int    `json:"version"`
+	ShardID      int    `json:"shard_id"`
+	TsNs         int64  `json:"ts_ns"`
+	RuleSetHash  []byte `json:"rule_set_hash"`
+	PromotedHash []byte `json:"promoted_hash"`
+	LastWalkedNs int64  `json:"last_walked_ns,omitempty"`
+}
+
+// CursorPersister loads and saves daily-replay cursors. The Phase 2
+// production implementation is FilerCursorPersister; tests inject a
+// fake.
+type CursorPersister interface {
+	Load(ctx context.Context, shardID int) (Cursor, bool, error) // (cursor, found, err)
+	Save(ctx context.Context, shardID int, c Cursor) error
+}
+
+// FilerCursorPersister writes cursors to CursorDir as one JSON file per
+// shard. Reuses dispatcher.FilerStore for the actual filer I/O so both
+// algorithms share the same minimal storage abstraction.
+type FilerCursorPersister struct {
+	Store dispatcher.FilerStore
+}
+
+func cursorFileName(shardID int) string {
+	return fmt.Sprintf("shard-%02d.json", shardID)
+}
+
+// Load returns (zero-Cursor, false, nil) only when the cursor file
+// does not exist yet (cold start). Every other failure mode — empty
+// file, malformed JSON, wrong version, wrong shard, hash slices not
+// exactly 32 bytes — returns an error so the daily run halts and is
+// fixed by an operator rather than silently re-scanning from time zero.
+//
+// Strict shape validation is load-bearing because the cursor is
+// load → mutate → save in a single run: a partial-truncate that
+// silently zero-padded the rule_set_hash would be persisted back as a
+// real-looking hash, masking the corruption forever.
+func (p *FilerCursorPersister) Load(ctx context.Context, shardID int) (Cursor, bool, error) {
+	if p.Store == nil {
+		return Cursor{}, false, errors.New("FilerCursorPersister: nil Store")
+	}
+	content, err := p.Store.Read(ctx, CursorDir, cursorFileName(shardID))
+	if err != nil {
+		if errors.Is(err, filer_pb.ErrNotFound) {
+			return Cursor{}, false, nil
+		}
+		return Cursor{}, false, fmt.Errorf("cursor read shard=%d: %w", shardID, err)
+	}
+	if len(content) == 0 {
+		return Cursor{}, false, fmt.Errorf("cursor shard=%d: file exists but is empty (partial write or external truncation)", shardID)
+	}
+	var cf cursorFile
+	if err := json.Unmarshal(content, &cf); err != nil {
+		return Cursor{}, false, fmt.Errorf("cursor decode shard=%d: %w", shardID, err)
+	}
+	if cf.Version != cursorFileVersion {
+		return Cursor{}, false, fmt.Errorf("cursor version shard=%d: got %d, want %d", shardID, cf.Version, cursorFileVersion)
+	}
+	if cf.ShardID != shardID {
+		return Cursor{}, false, fmt.Errorf("cursor shard mismatch: file declares shard=%d, requested shard=%d", cf.ShardID, shardID)
+	}
+	if len(cf.RuleSetHash) != 32 {
+		return Cursor{}, false, fmt.Errorf("cursor rule_set_hash shard=%d: got %d bytes, want 32", shardID, len(cf.RuleSetHash))
+	}
+	if len(cf.PromotedHash) != 32 {
+		return Cursor{}, false, fmt.Errorf("cursor promoted_hash shard=%d: got %d bytes, want 32", shardID, len(cf.PromotedHash))
+	}
+	c := Cursor{TsNs: cf.TsNs, LastWalkedNs: cf.LastWalkedNs}
+	copy(c.RuleSetHash[:], cf.RuleSetHash)
+	copy(c.PromotedHash[:], cf.PromotedHash)
+	return c, true, nil
+}
+
+func (p *FilerCursorPersister) Save(ctx context.Context, shardID int, c Cursor) error {
+	if p.Store == nil {
+		return errors.New("FilerCursorPersister: nil Store")
+	}
+	cf := cursorFile{
+		Version:      cursorFileVersion,
+		ShardID:      shardID,
+		TsNs:         c.TsNs,
+		RuleSetHash:  c.RuleSetHash[:],
+		PromotedHash: c.PromotedHash[:],
+		LastWalkedNs: c.LastWalkedNs,
+	}
+	var buf bytes.Buffer
+	enc := json.NewEncoder(&buf)
+	enc.SetIndent("", "  ")
+	if err := enc.Encode(cf); err != nil {
+		return fmt.Errorf("cursor encode shard=%d: %w", shardID, err)
+	}
+	if err := p.Store.Save(ctx, CursorDir, cursorFileName(shardID), buf.Bytes()); err != nil {
+		return fmt.Errorf("cursor save shard=%d: %w", shardID, err)
+	}
+	return nil
+}

weed/s3api/s3lifecycle/dailyrun/cursor_summary_test.go

@@ -0,0 +1,74 @@
+package dailyrun
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestSummarizeShardCursorLag_AllColdStart pins the marker tokens for
+// the worker's very first heartbeat — no cursor file exists yet, no
+// walker has fired, the line must say "cold" rather than "0s" so
+// operators can tell the difference between "I'm caught up" and
+// "I haven't started." Both states would be 0s otherwise.
+func TestSummarizeShardCursorLag_AllColdStart(t *testing.T) {
+	p := newMemPersister()
+	cfg := Config{Persister: p, Shards: []int{0, 1, 2}}
+	runNow := time.Unix(1_700_000_000, 0).UTC()
+
+	got := summarizeShardCursorLag(context.Background(), cfg, runNow)
+	assert.Equal(t, "cursor_lag_max=cold walked_max_age=cold", got)
+}
+
+// TestSummarizeShardCursorLag_PicksMaxAcrossShards confirms the worst-
+// case lag wins, not the average. Operators alert on the worst shard;
+// the cluster is only as healthy as its slowest one.
+func TestSummarizeShardCursorLag_PicksMaxAcrossShards(t *testing.T) {
+	p := newMemPersister()
+	runNow := time.Unix(1_700_000_000, 0).UTC()
+	require.NoError(t, p.Save(context.Background(), 0, Cursor{
+		TsNs:         runNow.Add(-5 * time.Minute).UnixNano(),
+		LastWalkedNs: runNow.Add(-30 * time.Minute).UnixNano(),
+	}))
+	require.NoError(t, p.Save(context.Background(), 1, Cursor{
+		TsNs:         runNow.Add(-3 * time.Hour).UnixNano(), // worst cursor
+		LastWalkedNs: runNow.Add(-10 * time.Minute).UnixNano(),
+	}))
+	require.NoError(t, p.Save(context.Background(), 2, Cursor{
+		TsNs:         runNow.Add(-30 * time.Minute).UnixNano(),
+		LastWalkedNs: runNow.Add(-2 * time.Hour).UnixNano(), // worst walker
+	}))
+	cfg := Config{Persister: p, Shards: []int{0, 1, 2}}
+
+	got := summarizeShardCursorLag(context.Background(), cfg, runNow)
+	assert.Contains(t, got, "cursor_lag_max=3h0m0s")
+	assert.Contains(t, got, "walked_max_age=2h0m0s")
+}
+
+// TestSummarizeShardCursorLag_PartialFill — some shards have cursors,
+// some don't. Don't let the unwalked-yet shards contaminate the worst-
+// case calculation; just compute the max over what we have.
+func TestSummarizeShardCursorLag_PartialFill(t *testing.T) {
+	p := newMemPersister()
+	runNow := time.Unix(1_700_000_000, 0).UTC()
+	require.NoError(t, p.Save(context.Background(), 0, Cursor{
+		TsNs: runNow.Add(-15 * time.Minute).UnixNano(),
+		// LastWalkedNs not set — shard 0 hasn't walked yet
+	}))
+	require.NoError(t, p.Save(context.Background(), 1, Cursor{
+		// TsNs not set — shard 1 hasn't drained yet
+		LastWalkedNs: runNow.Add(-45 * time.Minute).UnixNano(),
+	}))
+	cfg := Config{Persister: p, Shards: []int{0, 1, 2}}
+
+	got := summarizeShardCursorLag(context.Background(), cfg, runNow)
+	// Only shard 0 contributes to cursor_lag_max; only shard 1 to walked_max_age.
+	assert.Contains(t, got, "cursor_lag_max=15m0s")
+	assert.Contains(t, got, "walked_max_age=45m0s")
+	// Shard 2 wasn't saved at all — it doesn't show up as either.
+	assert.False(t, strings.Contains(got, "cold"), "any-fill case should not emit cold markers")
+}

weed/s3api/s3lifecycle/dailyrun/cursor_test.go

@@ -0,0 +1,155 @@
+package dailyrun
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeStore is a minimal in-memory dispatcher.FilerStore for cursor tests.
+type fakeStore struct {
+	mu    sync.Mutex
+	files map[string][]byte
+}
+
+func newFakeStore() *fakeStore {
+	return &fakeStore{files: map[string][]byte{}}
+}
+
+func (s *fakeStore) key(dir, name string) string { return dir + "/" + name }
+
+func (s *fakeStore) Read(_ context.Context, dir, name string) ([]byte, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	b, ok := s.files[s.key(dir, name)]
+	if !ok {
+		return nil, filer_pb.ErrNotFound
+	}
+	return append([]byte(nil), b...), nil
+}
+
+func (s *fakeStore) Save(_ context.Context, dir, name string, content []byte) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.files[s.key(dir, name)] = append([]byte(nil), content...)
+	return nil
+}
+
+func TestFilerCursorPersister_EmptyLoadReturnsNotFound(t *testing.T) {
+	p := &FilerCursorPersister{Store: newFakeStore()}
+	c, found, err := p.Load(context.Background(), 0)
+	require.NoError(t, err)
+	assert.False(t, found)
+	assert.Equal(t, Cursor{}, c)
+}
+
+func TestFilerCursorPersister_SaveLoadRoundTrip(t *testing.T) {
+	p := &FilerCursorPersister{Store: newFakeStore()}
+	in := Cursor{
+		TsNs: 1700000000_000000123,
+	}
+	for i := range in.RuleSetHash {
+		in.RuleSetHash[i] = byte(i + 1)
+	}
+	for i := range in.PromotedHash {
+		in.PromotedHash[i] = byte(0xff - i)
+	}
+	require.NoError(t, p.Save(context.Background(), 3, in))
+
+	out, found, err := p.Load(context.Background(), 3)
+	require.NoError(t, err)
+	require.True(t, found)
+	assert.Equal(t, in, out)
+}
+
+func TestFilerCursorPersister_IsolatesShards(t *testing.T) {
+	p := &FilerCursorPersister{Store: newFakeStore()}
+	a := Cursor{TsNs: 100}
+	b := Cursor{TsNs: 200}
+	require.NoError(t, p.Save(context.Background(), 0, a))
+	require.NoError(t, p.Save(context.Background(), 1, b))
+
+	got0, _, err := p.Load(context.Background(), 0)
+	require.NoError(t, err)
+	got1, _, err := p.Load(context.Background(), 1)
+	require.NoError(t, err)
+	assert.Equal(t, int64(100), got0.TsNs)
+	assert.Equal(t, int64(200), got1.TsNs)
+}
+
+func TestFilerCursorPersister_CorruptDataReturnsError(t *testing.T) {
+	store := newFakeStore()
+	store.files[store.key(CursorDir, "shard-00.json")] = []byte("not json")
+	p := &FilerCursorPersister{Store: store}
+	_, _, err := p.Load(context.Background(), 0)
+	require.Error(t, err)
+}
+
+func TestFilerCursorPersister_EmptyFileReturnsError(t *testing.T) {
+	// A cursor file that exists but is empty signals a partial write
+	// or external truncation. Treating it as "cold start" would silently
+	// re-scan from time zero and burn through a meta-log window for no
+	// reason. Pin that we error instead.
+	store := newFakeStore()
+	store.files[store.key(CursorDir, "shard-00.json")] = []byte{}
+	p := &FilerCursorPersister{Store: store}
+	_, _, err := p.Load(context.Background(), 0)
+	require.Error(t, err)
+}
+
+func TestFilerCursorPersister_WrongVersionReturnsError(t *testing.T) {
+	// Future schema bumps must not silently overwrite a cursor written
+	// by a newer worker. Pin that contract here.
+	store := newFakeStore()
+	store.files[store.key(CursorDir, "shard-00.json")] = []byte(`{"version":42,"shard_id":0,"ts_ns":0,"rule_set_hash":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=","promoted_hash":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="}`)
+	p := &FilerCursorPersister{Store: store}
+	_, _, err := p.Load(context.Background(), 0)
+	require.Error(t, err)
+}
+
+func TestFilerCursorPersister_ShardIDMismatchReturnsError(t *testing.T) {
+	// A file declaring shard=1 read by a shard=0 caller is corruption
+	// (or a misroute); applying that cursor would advance the wrong
+	// shard's state. Reject loudly.
+	store := newFakeStore()
+	// Build a valid v1 payload but with shard_id=1 in the body.
+	in := Cursor{TsNs: 42}
+	for i := range in.RuleSetHash {
+		in.RuleSetHash[i] = byte(i + 1)
+	}
+	for i := range in.PromotedHash {
+		in.PromotedHash[i] = byte(i)
+	}
+	p := &FilerCursorPersister{Store: store}
+	require.NoError(t, p.Save(context.Background(), 1, in))
+	// Reparent that file as if it lived at shard 0 — the test's fake
+	// store only keys by name, so we copy the bytes manually.
+	store.mu.Lock()
+	store.files[store.key(CursorDir, "shard-00.json")] = store.files[store.key(CursorDir, "shard-01.json")]
+	store.mu.Unlock()
+	_, _, err := p.Load(context.Background(), 0)
+	require.Error(t, err)
+}
+
+func TestFilerCursorPersister_HashLengthMismatchReturnsError(t *testing.T) {
+	// Hand-rolled JSON with hashes shorter than 32 bytes. copy() would
+	// silently zero-pad which makes the hash comparison match a value
+	// that doesn't actually exist on disk. Reject loudly so an operator
+	// fixes the persisted state.
+	store := newFakeStore()
+	store.files[store.key(CursorDir, "shard-00.json")] = []byte(`{"version":1,"shard_id":0,"ts_ns":0,"rule_set_hash":"AA==","promoted_hash":"AA=="}`)
+	p := &FilerCursorPersister{Store: store}
+	_, _, err := p.Load(context.Background(), 0)
+	require.Error(t, err)
+}
+
+func TestFilerCursorPersister_NilStoreErrors(t *testing.T) {
+	p := &FilerCursorPersister{}
+	_, _, err := p.Load(context.Background(), 0)
+	require.Error(t, err)
+	require.Error(t, p.Save(context.Background(), 0, Cursor{}))
+}

weed/s3api/s3lifecycle/dailyrun/dispatch.go

@@ -0,0 +1,125 @@
+package dailyrun
+
+import (
+	"context"
+	"errors"
+	"math/rand"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
+)
+
+// transportRetryAttempts is the in-run retry budget for transport
+// errors (RPC dial failures, stream resets). Server-side outcomes
+// (RETRY_LATER, BLOCKED) are NOT retried in-run — the server already
+// classified them as "wait until next run." The retry exists only to
+// paper over a single flake from the network.
+const transportRetryAttempts = 3
+
+// transportRetryInitial is the first sleep after a failed attempt.
+// Doubled per attempt up to transportRetryMax. Phase 2 keeps these
+// small because the daily run's own MaxRuntime is the larger cap.
+const (
+	transportRetryInitial = 200 * time.Millisecond
+	transportRetryMax     = 5 * time.Second
+)
+
+// dispatchWithRetry sends one LifecycleDelete request, retrying on
+// transport errors up to transportRetryAttempts times with exponential
+// backoff. Returns the server's outcome on success, or an error if all
+// retries exhausted. Server outcomes (RETRY_LATER / BLOCKED) bypass
+// the retry — the daily run's halt-on-failure semantics handles them.
+func dispatchWithRetry(ctx context.Context, client LifecycleClient, m router.Match) (s3_lifecycle_pb.LifecycleDeleteOutcome, error) {
+	req := buildDeleteRequest(m)
+	backoff := transportRetryInitial
+	var lastErr error
+	for attempt := 1; attempt <= transportRetryAttempts; attempt++ {
+		resp, err := client.LifecycleDelete(ctx, req)
+		if err == nil {
+			return resp.Outcome, nil
+		}
+		// Context cancellation is shutdown, not a transport flake.
+		// Surface it immediately so the caller halts and tomorrow
+		// resumes from the same cursor.
+		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+			return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, err
+		}
+		lastErr = err
+		if attempt == transportRetryAttempts {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, ctx.Err()
+		case <-time.After(jitter(backoff)):
+		}
+		backoff *= 2
+		if backoff > transportRetryMax {
+			backoff = transportRetryMax
+		}
+	}
+	return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, lastErr
+}
+
+// jitter returns a duration in the range [d/2, d) using equal jitter.
+// Prevents thundering herds when many daily-run workers retry simultaneously.
+func jitter(d time.Duration) time.Duration {
+	if d <= 0 {
+		return 0
+	}
+	half := d / 2
+	if half <= 0 {
+		return d
+	}
+	return half + time.Duration(rand.Int63n(int64(half)))
+}
+
+// buildDeleteRequest constructs the LifecycleDelete RPC payload for a
+// router Match. Mirrors dispatcher.dispatchOne's request shape — both
+// targets the same server-side handler and the proto encoding must
+// match exactly. Duplicated rather than shared because Phase 5
+// deletes the dispatcher's call site and the cross-package dependency
+// would just create a temporary import.
+func buildDeleteRequest(m router.Match) *s3_lifecycle_pb.LifecycleDeleteRequest {
+	rh := m.Key.RuleHash
+	return &s3_lifecycle_pb.LifecycleDeleteRequest{
+		Bucket:           m.Bucket,
+		ObjectPath:       m.ObjectKey,
+		VersionId:        m.VersionID,
+		RuleHash:         rh[:],
+		ActionKind:       toProtoActionKind(m.Key.ActionKind),
+		ExpectedIdentity: toProtoIdentity(m.Identity),
+	}
+}
+
+func toProtoActionKind(k s3lifecycle.ActionKind) s3_lifecycle_pb.ActionKind {
+	switch k {
+	case s3lifecycle.ActionKindExpirationDays:
+		return s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS
+	case s3lifecycle.ActionKindExpirationDate:
+		return s3_lifecycle_pb.ActionKind_EXPIRATION_DATE
+	case s3lifecycle.ActionKindNoncurrentDays:
+		return s3_lifecycle_pb.ActionKind_NONCURRENT_DAYS
+	case s3lifecycle.ActionKindNewerNoncurrent:
+		return s3_lifecycle_pb.ActionKind_NEWER_NONCURRENT
+	case s3lifecycle.ActionKindAbortMPU:
+		return s3_lifecycle_pb.ActionKind_ABORT_MPU
+	case s3lifecycle.ActionKindExpiredDeleteMarker:
+		return s3_lifecycle_pb.ActionKind_EXPIRED_DELETE_MARKER
+	}
+	return s3_lifecycle_pb.ActionKind_ACTION_KIND_UNSPECIFIED
+}
+
+func toProtoIdentity(id *router.EntryIdentity) *s3_lifecycle_pb.EntryIdentity {
+	if id == nil {
+		return nil
+	}
+	return &s3_lifecycle_pb.EntryIdentity{
+		MtimeNs:      id.MtimeNs,
+		Size:         id.Size,
+		HeadFid:      id.HeadFid,
+		ExtendedHash: id.ExtendedHash,
+	}
+}

weed/s3api/s3lifecycle/dailyrun/dispatch_test.go

@@ -0,0 +1,164 @@
+package dailyrun
+
+import (
+	"context"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeLifecycleClient is the test double for LifecycleClient. It
+// returns a scripted sequence of (outcome, error) pairs and counts
+// invocations so retry behavior can be pinned exactly.
+type fakeLifecycleClient struct {
+	scripted []scriptedResp
+	calls    int32
+}
+
+type scriptedResp struct {
+	outcome s3_lifecycle_pb.LifecycleDeleteOutcome
+	err     error
+}
+
+func (c *fakeLifecycleClient) LifecycleDelete(_ context.Context, _ *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
+	idx := int(atomic.AddInt32(&c.calls, 1)) - 1
+	if idx >= len(c.scripted) {
+		// Default to the last scripted response if the test under-specifies.
+		idx = len(c.scripted) - 1
+	}
+	r := c.scripted[idx]
+	if r.err != nil {
+		return nil, r.err
+	}
+	return &s3_lifecycle_pb.LifecycleDeleteResponse{Outcome: r.outcome}, nil
+}
+
+func sampleMatch() router.Match {
+	return router.Match{
+		Key:       s3lifecycle.ActionKey{Bucket: "b", ActionKind: s3lifecycle.ActionKindExpirationDays},
+		Bucket:    "b",
+		ObjectKey: "obj",
+	}
+}
+
+// Speed up dispatch retries inside tests so the suite stays fast.
+// The defaults (200ms initial, 5s max) make exponential backoff cases
+// take seconds; tests shrink to microseconds via a separate helper.
+func dispatchWithRetryFast(t *testing.T, ctx context.Context, c LifecycleClient, m router.Match) (s3_lifecycle_pb.LifecycleDeleteOutcome, error) {
+	t.Helper()
+	// We can't override the package-private constants, but every test
+	// path here uses small attempt counts so even the production
+	// backoff is fine.
+	return dispatchWithRetry(ctx, c, m)
+}
+
+func TestDispatch_FirstAttemptSucceeds(t *testing.T) {
+	c := &fakeLifecycleClient{scripted: []scriptedResp{
+		{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
+	}}
+	out, err := dispatchWithRetryFast(t, context.Background(), c, sampleMatch())
+	require.NoError(t, err)
+	assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, out)
+	assert.Equal(t, int32(1), c.calls)
+}
+
+func TestDispatch_TransportRetryThenSucceed(t *testing.T) {
+	// Two transport flakes then a success. The retry loop tries 3 times
+	// by default; we expect 3 calls and the final outcome.
+	c := &fakeLifecycleClient{scripted: []scriptedResp{
+		{err: errors.New("transport boom")},
+		{err: errors.New("transport boom")},
+		{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED},
+	}}
+	out, err := dispatchWithRetryFast(t, context.Background(), c, sampleMatch())
+	require.NoError(t, err)
+	assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED, out)
+	assert.Equal(t, int32(3), c.calls)
+}
+
+func TestDispatch_ExhaustsRetriesReturnsError(t *testing.T) {
+	// All N attempts fail with transport errors. Caller must see an
+	// error and the call count must equal transportRetryAttempts.
+	failing := errors.New("transport persistent")
+	c := &fakeLifecycleClient{scripted: []scriptedResp{
+		{err: failing}, {err: failing}, {err: failing},
+	}}
+	_, err := dispatchWithRetryFast(t, context.Background(), c, sampleMatch())
+	require.Error(t, err)
+	assert.Equal(t, int32(transportRetryAttempts), c.calls)
+}
+
+func TestDispatch_ServerOutcomeRetryLaterIsNotRetried(t *testing.T) {
+	// RETRY_LATER is a SERVER outcome; the daily run's halt-on-failure
+	// caller handles it. dispatchWithRetry must surface it on the first
+	// successful RPC and NOT retry in-run.
+	c := &fakeLifecycleClient{scripted: []scriptedResp{
+		{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_RETRY_LATER},
+	}}
+	out, err := dispatchWithRetryFast(t, context.Background(), c, sampleMatch())
+	require.NoError(t, err)
+	assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_RETRY_LATER, out)
+	assert.Equal(t, int32(1), c.calls, "server outcome must not trigger transport retry")
+}
+
+func TestDispatch_ServerOutcomeBlockedNotRetried(t *testing.T) {
+	c := &fakeLifecycleClient{scripted: []scriptedResp{
+		{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_BLOCKED},
+	}}
+	out, err := dispatchWithRetryFast(t, context.Background(), c, sampleMatch())
+	require.NoError(t, err)
+	assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_BLOCKED, out)
+	assert.Equal(t, int32(1), c.calls)
+}
+
+func TestDispatch_ContextCancelledShortCircuits(t *testing.T) {
+	// Context cancellation must surface immediately, not consume the
+	// retry budget. Cursor-staying semantics in the caller depends on
+	// this distinction.
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	c := &fakeLifecycleClient{scripted: []scriptedResp{
+		{err: context.Canceled},
+	}}
+	_, err := dispatchWithRetryFast(t, ctx, c, sampleMatch())
+	require.ErrorIs(t, err, context.Canceled)
+	assert.Equal(t, int32(1), c.calls, "context cancellation must not consume the retry budget")
+}
+
+func TestBuildDeleteRequest_RuleHashAndIdentity(t *testing.T) {
+	// Pin the request shape so a future Match-struct refactor doesn't
+	// silently drop the identity-CAS witness or the rule hash slice.
+	m := router.Match{
+		Bucket:    "bucket",
+		ObjectKey: "obj.txt",
+		VersionID: "v_1",
+		Key: s3lifecycle.ActionKey{
+			Bucket:     "bucket",
+			RuleHash:   [8]byte{1, 2, 3, 4, 5, 6, 7, 8},
+			ActionKind: s3lifecycle.ActionKindNoncurrentDays,
+		},
+		Identity: &router.EntryIdentity{
+			MtimeNs:      time.Unix(1700000000, 12345).UnixNano(),
+			Size:         42,
+			HeadFid:      "1,abc",
+			ExtendedHash: []byte{0xaa, 0xbb},
+		},
+	}
+	req := buildDeleteRequest(m)
+	assert.Equal(t, "bucket", req.Bucket)
+	assert.Equal(t, "obj.txt", req.ObjectPath)
+	assert.Equal(t, "v_1", req.VersionId)
+	assert.Equal(t, []byte{1, 2, 3, 4, 5, 6, 7, 8}, req.RuleHash)
+	assert.Equal(t, s3_lifecycle_pb.ActionKind_NONCURRENT_DAYS, req.ActionKind)
+	require.NotNil(t, req.ExpectedIdentity)
+	assert.Equal(t, int64(42), req.ExpectedIdentity.Size)
+	assert.Equal(t, "1,abc", req.ExpectedIdentity.HeadFid)
+	assert.Equal(t, []byte{0xaa, 0xbb}, req.ExpectedIdentity.ExtendedHash)
+}

weed/s3api/s3lifecycle/dailyrun/filer_list_func.go

@@ -0,0 +1,345 @@
+package dailyrun
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// listPageSize is the page size for paginated directory listings. The
+// filer caps SeaweedList(..., limit=0) at DirListingLimit (1000 by
+// default) per call, so a single-page list would silently truncate
+// large directories. Atomic so tests can shrink it without racing.
+var listPageSize atomic.Uint32
+
+func init() { listPageSize.Store(1024) }
+
+// FilerListFunc returns a bootstrap.ListFunc that streams entries
+// under <bucketsPath>/<bucket>. Versioned siblings are expanded with
+// IsLatest / NumVersions / NoncurrentIndex / SuccessorModTime so the
+// walker's NoncurrentDays evaluation has the same per-version state
+// the streaming bootstrap injects via reader.Event.BootstrapVersion.
+// MPU init records at .uploads/<id> with ExtMultipartObjectKey set
+// are emitted whole with IsMPUInit=true and DestKey carrying the
+// user's intended path.
+func FilerListFunc(client filer_pb.SeaweedFilerClient, bucketsPath string) bootstrap.ListFunc {
+	return func(ctx context.Context, bucket, start string, cb func(*bootstrap.Entry) error) error {
+		if client == nil {
+			return fmt.Errorf("FilerListFunc: nil client")
+		}
+		root := strings.TrimSuffix(bucketsPath, "/") + "/" + bucket
+		return walkBucketTree(ctx, client, root, root, start, cb)
+	}
+}
+
+// walkBucketTree recurses through dir in two passes per level. Pass 1
+// expands `.versions/` dirs (populating skipBare with the bare null-
+// version keys that pass 2 must suppress). Pass 2 emits regular files
+// and recurses into non-special subdirectories.
+//
+// The two-pass shape mirrors scheduler/bootstrap.go's walkBucketDir
+// (see that file's comment for why `.versions/` has to be processed
+// before its bare sibling).
+func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir, bucketRoot, start string, cb func(*bootstrap.Entry) error) error {
+	skipBare := map[string]bool{}
+
+	// Pass 1: .versions/ dirs only.
+	if err := listAll(ctx, client, dir, func(e *filer_pb.Entry) error {
+		if e == nil || e.Attributes == nil {
+			return nil
+		}
+		if !e.IsDirectory || !isVersionsDir(e) {
+			return nil
+		}
+		full := dir + "/" + e.Name
+		key := strings.TrimPrefix(full, bucketRoot+"/")
+		return expandVersionsDir(ctx, client, bucketRoot, key, e, start, skipBare, cb)
+	}); err != nil {
+		return err
+	}
+
+	// Pass 2: everything else.
+	return listAll(ctx, client, dir, func(e *filer_pb.Entry) error {
+		if e == nil || e.Attributes == nil {
+			return nil
+		}
+		if e.IsDirectory && isVersionsDir(e) {
+			return nil
+		}
+		full := dir + "/" + e.Name
+		key := strings.TrimPrefix(full, bucketRoot+"/")
+		if e.IsDirectory {
+			if isMPUInitDir(key, e) {
+				if start != "" && key <= start {
+					return nil
+				}
+				destKey := string(e.Extended[s3_constants.ExtMultipartObjectKey])
+				return cb(&bootstrap.Entry{
+					Path:      key,
+					DestKey:   destKey,
+					IsMPUInit: true,
+					ModTime:   time.Unix(e.Attributes.Mtime, int64(e.Attributes.MtimeNs)),
+					Size:      int64(e.Attributes.FileSize),
+				})
+			}
+			return walkBucketTree(ctx, client, full, bucketRoot, start, cb)
+		}
+		if skipBare[key] {
+			return nil
+		}
+		if start != "" && key <= start {
+			return nil
+		}
+		entry := &bootstrap.Entry{
+			Path:     key,
+			ModTime:  time.Unix(e.Attributes.Mtime, int64(e.Attributes.MtimeNs)),
+			Size:     int64(e.Attributes.FileSize),
+			IsLatest: true, // Non-versioned default.
+			Tags:     extractTags(e.Extended),
+		}
+		return cb(entry)
+	})
+}
+
+// versionItem captures one sibling of a `.versions/` expansion. bareKey
+// is the bucket-relative path of the bare null-version entry when the
+// item represents it; for real version files it stays empty.
+type versionItem struct {
+	entry          *filer_pb.Entry
+	versionID      string
+	bareKey        string
+	isExplicitNull bool
+}
+
+// expandVersionsDir handles the `.versions/<key>` directory. Lists
+// version files, optionally appends the bare null-version sibling,
+// sorts newest-first, resolves the latest, and emits one Entry per
+// version with the sibling state walkEntry needs to evaluate
+// NoncurrentDays / NewerNoncurrent / ExpirationDays correctly.
+//
+// Ported from scheduler/bootstrap.go's same-named helper; both must
+// agree on sort, latest resolution, and successor derivation so the
+// streaming and walker paths reach the same verdict for the same
+// objects. Phase 5 deletes the scheduler copy.
+//
+// Resume note: every emitted sibling shares Path = logical key, so a
+// resume after a mid-expansion failure rewalks the whole sibling
+// group. Acceptable today because Phase 4b doesn't persist a
+// Checkpoint between runs (start is always "" via runShard).
+func expandVersionsDir(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketRoot, versionsKey string, versionsEntry *filer_pb.Entry, start string, skipBare map[string]bool, cb func(*bootstrap.Entry) error) error {
+	logical := strings.TrimSuffix(versionsKey, s3_constants.VersionsFolder)
+	if logical == "" {
+		return nil
+	}
+	versionsDir := bucketRoot + "/" + versionsKey
+	var children []*filer_pb.Entry
+	if err := listAll(ctx, client, versionsDir, func(e *filer_pb.Entry) error {
+		if e != nil && e.Attributes != nil && !e.IsDirectory {
+			children = append(children, e)
+		}
+		return nil
+	}); err != nil {
+		return fmt.Errorf("list %s: %w", versionsDir, err)
+	}
+	items := make([]versionItem, 0, len(children)+1)
+	for _, e := range children {
+		if id, ok := e.Extended[s3_constants.ExtVersionIdKey]; ok && len(id) > 0 {
+			items = append(items, versionItem{entry: e, versionID: string(id)})
+		}
+	}
+	if len(items) == 0 {
+		// Coincidentally-named user folder, or an empty `.versions`
+		// container. Treat it as a regular subdirectory so user-named
+		// files inside still surface.
+		return walkBucketTree(ctx, client, versionsDir, bucketRoot, start, cb)
+	}
+	if nullEntry, nullKey, explicit, ok := lookupNullVersion(ctx, client, bucketRoot, logical); ok {
+		items = append(items, versionItem{
+			entry:          nullEntry,
+			versionID:      "null",
+			bareKey:        nullKey,
+			isExplicitNull: explicit,
+		})
+	}
+
+	// Sort newest-first by mtime, ties broken by version_id (newer wins).
+	sort.SliceStable(items, func(i, j int) bool {
+		mi := items[i].entry.Attributes.Mtime*int64(1e9) + int64(items[i].entry.Attributes.MtimeNs)
+		mj := items[j].entry.Attributes.Mtime*int64(1e9) + int64(items[j].entry.Attributes.MtimeNs)
+		if mi != mj {
+			return mi > mj
+		}
+		return s3lifecycle.CompareVersionIds(items[i].versionID, items[j].versionID) < 0
+	})
+
+	// Resolve latest:
+	//   1. Pointer names a real id -> that wins.
+	//   2. Pointer absent (or stale: set but no sibling carries it)
+	//      + any EXPLICIT null bare exists -> null is latest.
+	//   3. Otherwise -> newest sibling (latestPos = 0 by default).
+	//
+	// A stale pointer falls through to the no-pointer fallback rather
+	// than silently leaving latestPos at 0 with no documented intent;
+	// the value happens to be the same today (newest sibling wins
+	// either way) but the explicit branching protects against future
+	// fallback refinements diverging by accident.
+	latestID := string(versionsEntry.Extended[s3_constants.ExtLatestVersionIdKey])
+	latestPos := 0
+	pointerResolved := false
+	if latestID != "" {
+		for i, it := range items {
+			if it.versionID == latestID {
+				latestPos = i
+				pointerResolved = true
+				break
+			}
+		}
+	}
+	if !pointerResolved {
+		for i, it := range items {
+			if it.versionID == "null" && it.isExplicitNull {
+				latestPos = i
+				break
+			}
+		}
+	}
+
+	if start != "" && logical <= start {
+		// All siblings share Path=logical, so the whole group is
+		// either above or below the resume marker.
+		return nil
+	}
+	for i, it := range items {
+		successor := s3lifecycle.SuccessorFromEntryStamp(it.entry)
+		if successor.IsZero() && i > 0 {
+			prev := items[i-1].entry.Attributes
+			successor = time.Unix(prev.Mtime, int64(prev.MtimeNs))
+		}
+		isLatest := i == latestPos
+		entry := &bootstrap.Entry{
+			Path:             logical,
+			VersionID:        it.versionID,
+			ModTime:          time.Unix(it.entry.Attributes.Mtime, int64(it.entry.Attributes.MtimeNs)),
+			Size:             int64(it.entry.Attributes.FileSize),
+			IsLatest:         isLatest,
+			IsDeleteMarker:   string(it.entry.Extended[s3_constants.ExtDeleteMarkerKey]) == "true",
+			NumVersions:      len(items),
+			SuccessorModTime: successor,
+			Tags:             extractTags(it.entry.Extended),
+		}
+		if !isLatest {
+			rank := i
+			if i > latestPos {
+				rank = i - 1
+			}
+			entry.NoncurrentIndex = &rank
+		}
+		if err := cb(entry); err != nil {
+			return err
+		}
+		if it.versionID == "null" && skipBare != nil {
+			skipBare[it.bareKey] = true
+		}
+	}
+	return nil
+}
+
+// lookupNullVersion returns the bare-key entry that represents the null
+// version of logical, if any. Both regular files and S3 directory-key
+// markers qualify. explicit is true when the entry carries
+// ExtVersionIdKey == "null" — the marker the suspended-versioning
+// write path applies; only an explicit-null bare can outrank a missing
+// `.versions/` pointer per the latest-resolution rules above.
+func lookupNullVersion(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketRoot, logical string) (*filer_pb.Entry, string, bool, bool) {
+	parent, name := util.NewFullPath(bucketRoot, logical).DirAndName()
+	resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
+		Directory: parent,
+		Name:      name,
+	})
+	if err != nil || resp == nil || resp.Entry == nil {
+		return nil, "", false, false
+	}
+	e := resp.Entry
+	if e.IsDirectory && !e.IsDirectoryKeyObject() {
+		return nil, "", false, false
+	}
+	explicit := false
+	if id, hasID := e.Extended[s3_constants.ExtVersionIdKey]; hasID && string(id) == "null" {
+		explicit = true
+	}
+	return e, strings.TrimPrefix(parent+"/"+name, bucketRoot+"/"), explicit, true
+}
+
+// listAll issues paginated SeaweedList calls until exhausted. Ported
+// from scheduler/bootstrap.go's same-named helper; Phase 5 deletes
+// the scheduler copy when the streaming path is removed.
+func listAll(ctx context.Context, client filer_pb.SeaweedFilerClient, dir string, fn func(*filer_pb.Entry) error) error {
+	pageSize := listPageSize.Load()
+	startFrom := ""
+	for {
+		var pageCount uint32
+		var lastName string
+		if err := filer_pb.SeaweedList(ctx, client, dir, "", func(e *filer_pb.Entry, _ bool) error {
+			pageCount++
+			if e != nil {
+				lastName = e.Name
+			}
+			return fn(e)
+		}, startFrom, false, pageSize); err != nil {
+			return err
+		}
+		if pageCount < pageSize {
+			return nil
+		}
+		startFrom = lastName
+	}
+}
+
+func isVersionsDir(entry *filer_pb.Entry) bool {
+	return entry.IsDirectory && strings.HasSuffix(entry.Name, s3_constants.VersionsFolder)
+}
+
+// isMPUInitDir mirrors router.mpuInitInfo: a directory at
+// .uploads/<id> carrying the destination key in Extended is the MPU
+// init record. Verified shape + presence of ExtMultipartObjectKey;
+// directories at .uploads/<id> without the key are mid-write before
+// metadata landed and stay out of the dispatch path.
+func isMPUInitDir(key string, entry *filer_pb.Entry) bool {
+	uploadsPrefix := s3_constants.MultipartUploadsFolder + "/"
+	if !strings.HasPrefix(key, uploadsPrefix) {
+		return false
+	}
+	rest := key[len(uploadsPrefix):]
+	if rest == "" || strings.ContainsRune(rest, '/') {
+		return false
+	}
+	v, ok := entry.Extended[s3_constants.ExtMultipartObjectKey]
+	return ok && len(v) > 0
+}
+
+func extractTags(ext map[string][]byte) map[string]string {
+	if len(ext) == 0 {
+		return nil
+	}
+	prefix := s3_constants.AmzObjectTagging + "-"
+	var tags map[string]string
+	for k, v := range ext {
+		if !strings.HasPrefix(k, prefix) {
+			continue
+		}
+		if tags == nil {
+			tags = make(map[string]string)
+		}
+		tags[k[len(prefix):]] = string(v)
+	}
+	return tags
+}

weed/s3api/s3lifecycle/dailyrun/filer_list_func_test.go

@@ -0,0 +1,495 @@
+package dailyrun
+
+import (
+	"context"
+	"io"
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+// fakeFilerStream implements the ListEntries server-streaming client.
+type fakeFilerStream struct {
+	responses []*filer_pb.ListEntriesResponse
+	idx       int
+	ctx       context.Context
+}
+
+func (s *fakeFilerStream) Recv() (*filer_pb.ListEntriesResponse, error) {
+	if s.ctx != nil {
+		if err := s.ctx.Err(); err != nil {
+			return nil, err
+		}
+	}
+	if s.idx >= len(s.responses) {
+		return nil, io.EOF
+	}
+	r := s.responses[s.idx]
+	s.idx++
+	return r, nil
+}
+func (s *fakeFilerStream) Header() (metadata.MD, error) { return metadata.MD{}, nil }
+func (s *fakeFilerStream) Trailer() metadata.MD         { return metadata.MD{} }
+func (s *fakeFilerStream) CloseSend() error             { return nil }
+func (s *fakeFilerStream) Context() context.Context {
+	if s.ctx != nil {
+		return s.ctx
+	}
+	return context.Background()
+}
+func (s *fakeFilerStream) SendMsg(any) error { return nil }
+func (s *fakeFilerStream) RecvMsg(any) error { return nil }
+
+// fakeFiler maps directory paths to their immediate children. Only
+// ListEntries is implemented; other methods of SeaweedFilerClient are
+// inherited from the embedded interface and panic if called.
+type fakeFiler struct {
+	filer_pb.SeaweedFilerClient
+
+	mu   sync.Mutex
+	tree map[string][]*filer_pb.Entry
+}
+
+func (c *fakeFiler) LookupDirectoryEntry(_ context.Context, in *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	for _, e := range c.tree[in.Directory] {
+		if e != nil && e.Name == in.Name {
+			return &filer_pb.LookupDirectoryEntryResponse{Entry: e}, nil
+		}
+	}
+	return nil, filer_pb.ErrNotFound
+}
+
+func (c *fakeFiler) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, _ ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	src := c.tree[in.Directory]
+	// Mirror the filer: sort by name, honor StartFromFileName exclusive,
+	// cap at Limit. listAll's pagination loop depends on these.
+	filtered := make([]*filer_pb.Entry, 0, len(src))
+	for _, e := range src {
+		if e == nil {
+			continue
+		}
+		if in.StartFromFileName != "" && !in.InclusiveStartFrom && e.Name <= in.StartFromFileName {
+			continue
+		}
+		filtered = append(filtered, e)
+	}
+	sort.SliceStable(filtered, func(i, j int) bool { return filtered[i].Name < filtered[j].Name })
+	if in.Limit > 0 && uint32(len(filtered)) > in.Limit {
+		filtered = filtered[:in.Limit]
+	}
+	resps := make([]*filer_pb.ListEntriesResponse, 0, len(filtered))
+	for _, e := range filtered {
+		resps = append(resps, &filer_pb.ListEntriesResponse{Entry: e})
+	}
+	return &fakeFilerStream{responses: resps, ctx: ctx}, nil
+}
+
+func file(name string, mtime time.Time, size int64) *filer_pb.Entry {
+	return &filer_pb.Entry{
+		Name: name,
+		Attributes: &filer_pb.FuseAttributes{
+			Mtime:    mtime.Unix(),
+			MtimeNs:  int32(mtime.Nanosecond()),
+			FileSize: uint64(size),
+		},
+	}
+}
+
+func dir(name string) *filer_pb.Entry {
+	return &filer_pb.Entry{Name: name, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}}
+}
+
+func TestFilerListFunc_EmitsFlatFiles(t *testing.T) {
+	mtime := time.Now().Add(-7 * 24 * time.Hour)
+	client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
+		"/buckets/bkt": {file("a.txt", mtime, 10), file("b.txt", mtime, 20)},
+	}}
+	listFn := FilerListFunc(client, "/buckets")
+	var got []string
+	require.NoError(t, listFn(context.Background(), "bkt", "", func(e *bootstrap.Entry) error {
+		got = append(got, e.Path)
+		return nil
+	}))
+	assert.Equal(t, []string{"a.txt", "b.txt"}, got)
+}
+
+func TestFilerListFunc_PropagatesTagsOnFlatFiles(t *testing.T) {
+	mtime := time.Now().Add(-7 * 24 * time.Hour)
+	tagged := fileWithExt("tagged.txt", mtime, 10, map[string][]byte{
+		s3_constants.AmzObjectTagging + "-env":  []byte("temp"),
+		s3_constants.AmzObjectTagging + "-tier": []byte("cold"),
+	})
+	client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
+		"/buckets/bkt": {tagged},
+	}}
+	listFn := FilerListFunc(client, "/buckets")
+	var got *bootstrap.Entry
+	require.NoError(t, listFn(context.Background(), "bkt", "", func(e *bootstrap.Entry) error {
+		got = e
+		return nil
+	}))
+	require.NotNil(t, got)
+	assert.Equal(t, map[string]string{"env": "temp", "tier": "cold"}, got.Tags)
+}
+
+func TestFilerListFunc_RecursesIntoSubdirs(t *testing.T) {
+	mtime := time.Now()
+	client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
+		"/buckets/bkt":           {dir("logs"), file("root.txt", mtime, 1)},
+		"/buckets/bkt/logs":      {dir("2026"), file("a.log", mtime, 5)},
+		"/buckets/bkt/logs/2026": {file("b.log", mtime, 7)},
+	}}
+	listFn := FilerListFunc(client, "/buckets")
+	var paths []string
+	require.NoError(t, listFn(context.Background(), "bkt", "", func(e *bootstrap.Entry) error {
+		paths = append(paths, e.Path)
+		return nil
+	}))
+	sort.Strings(paths)
+	assert.Equal(t, []string{"logs/2026/b.log", "logs/a.log", "root.txt"}, paths)
+}
+
+func TestFilerListFunc_MPUInitEmitsWithDestKey(t *testing.T) {
+	// .uploads/<id>/ with ExtMultipartObjectKey is the MPU init record.
+	// One Entry per init, IsMPUInit=true, DestKey = the user's path.
+	mtime := time.Now()
+	mpuDir := dir("upload-id-1")
+	mpuDir.Extended = map[string][]byte{
+		s3_constants.ExtMultipartObjectKey: []byte("user/path/object"),
+	}
+	client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
+		"/buckets/bkt": {
+			file("regular.txt", mtime, 1),
+			dir(s3_constants.MultipartUploadsFolder),
+		},
+		"/buckets/bkt/" + s3_constants.MultipartUploadsFolder: {
+			mpuDir,
+		},
+	}}
+	listFn := FilerListFunc(client, "/buckets")
+	var got []*bootstrap.Entry
+	require.NoError(t, listFn(context.Background(), "bkt", "", func(e *bootstrap.Entry) error {
+		got = append(got, e)
+		return nil
+	}))
+	require.Len(t, got, 2)
+	byPath := map[string]*bootstrap.Entry{}
+	for _, e := range got {
+		byPath[e.Path] = e
+	}
+	require.NotNil(t, byPath["regular.txt"])
+	require.NotNil(t, byPath[s3_constants.MultipartUploadsFolder+"/upload-id-1"])
+	mpu := byPath[s3_constants.MultipartUploadsFolder+"/upload-id-1"]
+	assert.True(t, mpu.IsMPUInit)
+	assert.Equal(t, "user/path/object", mpu.DestKey)
+}
+
+func TestFilerListFunc_MPUInitWithoutDestKeyIsSkipped(t *testing.T) {
+	// A `.uploads/<id>` directory missing ExtMultipartObjectKey is
+	// mid-write before metadata landed; the dispatcher would error
+	// on empty DestKey. Skip it so the walk doesn't halt.
+	mtime := time.Now()
+	mpuDir := dir("upload-id-2") // no Extended
+	client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
+		"/buckets/bkt": {
+			file("regular.txt", mtime, 1),
+			dir(s3_constants.MultipartUploadsFolder),
+		},
+		"/buckets/bkt/" + s3_constants.MultipartUploadsFolder: {
+			mpuDir,
+		},
+	}}
+	listFn := FilerListFunc(client, "/buckets")
+	var paths []string
+	require.NoError(t, listFn(context.Background(), "bkt", "", func(e *bootstrap.Entry) error {
+		paths = append(paths, e.Path)
+		return nil
+	}))
+	assert.Equal(t, []string{"regular.txt"}, paths)
+}
+
+// fileWithExt is a versioned-file entry helper.
+func fileWithExt(name string, mtime time.Time, size int64, ext map[string][]byte) *filer_pb.Entry {
+	e := file(name, mtime, size)
+	e.Extended = ext
+	return e
+}
+
+func versionsDir(name string, latestID string) *filer_pb.Entry {
+	d := dir(name)
+	d.Extended = map[string][]byte{}
+	if latestID != "" {
+		d.Extended[s3_constants.ExtLatestVersionIdKey] = []byte(latestID)
+	}
+	return d
+}
+
+func TestFilerListFunc_VersionedExpansionMarksLatestByPointer(t *testing.T) {
+	// .versions/<key>/ with three real versions; parent's
+	// ExtLatestVersionIdKey points to v2 → IsLatest set on v2; the
+	// other two get NoncurrentIndex computed against the latest's
+	// position in the sorted (newest-first) list.
+	t1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	t2 := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC)
+	t3 := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC)
+	versions := []*filer_pb.Entry{
+		fileWithExt("v1", t1, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte("v1")}),
+		fileWithExt("v2", t2, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte("v2")}),
+		fileWithExt("v3", t3, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte("v3")}),
+	}
+	client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
+		"/buckets/bkt": {versionsDir("foo"+s3_constants.VersionsFolder, "v2")},
+		"/buckets/bkt/foo" + s3_constants.VersionsFolder: versions,
+	}}
+	listFn := FilerListFunc(client, "/buckets")
+	var got []*bootstrap.Entry
+	require.NoError(t, listFn(context.Background(), "bkt", "", func(e *bootstrap.Entry) error {
+		got = append(got, e)
+		return nil
+	}))
+	require.Len(t, got, 3)
+
+	byID := map[string]*bootstrap.Entry{}
+	for _, e := range got {
+		byID[e.VersionID] = e
+		assert.Equal(t, "foo", e.Path, "every sibling's Path is the logical key")
+		assert.Equal(t, 3, e.NumVersions)
+	}
+	assert.True(t, byID["v2"].IsLatest, "pointer wins regardless of mtime order")
+	assert.False(t, byID["v1"].IsLatest)
+	assert.False(t, byID["v3"].IsLatest)
+	require.NotNil(t, byID["v3"].NoncurrentIndex, "noncurrent siblings get a rank")
+	require.NotNil(t, byID["v1"].NoncurrentIndex, "noncurrent siblings get a rank")
+}
+
+func TestFilerListFunc_VersionedExpansionStalePointerFallsBackToNewestSibling(t *testing.T) {
+	// ExtLatestVersionIdKey names a version that no sibling carries
+	// (stale pointer left behind by a write whose update was lost or
+	// raced). The fallback must NOT silently treat items[0] as latest
+	// via the default; structurally it must drop into the no-pointer
+	// path so the explicit-null bare check still runs.
+	tOld := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	tNew := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC)
+	versions := []*filer_pb.Entry{
+		fileWithExt("v_old", tOld, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte("vold")}),
+		fileWithExt("v_new", tNew, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte("vnew")}),
+	}
+	client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
+		"/buckets/bkt": {versionsDir("foo"+s3_constants.VersionsFolder, "v_GHOST")},
+		"/buckets/bkt/foo" + s3_constants.VersionsFolder: versions,
+	}}
+	listFn := FilerListFunc(client, "/buckets")
+	var got []*bootstrap.Entry
+	require.NoError(t, listFn(context.Background(), "bkt", "", func(e *bootstrap.Entry) error {
+		got = append(got, e)
+		return nil
+	}))
+	require.Len(t, got, 2)
+	byID := map[string]*bootstrap.Entry{}
+	for _, e := range got {
+		byID[e.VersionID] = e
+	}
+	assert.True(t, byID["vnew"].IsLatest, "stale pointer must fall back to newest sibling")
+	assert.False(t, byID["vold"].IsLatest)
+}
+
+func TestFilerListFunc_VersionedExpansionNoPointerNewestSiblingWins(t *testing.T) {
+	// Parent has no ExtLatestVersionIdKey. With no explicit-null bare
+	// version, the newest-by-mtime sibling becomes latest.
+	tNew := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC)
+	tOld := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	versions := []*filer_pb.Entry{
+		fileWithExt("v_old", tOld, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte("vold")}),
+		fileWithExt("v_new", tNew, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte("vnew")}),
+	}
+	client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
+		"/buckets/bkt": {versionsDir("foo"+s3_constants.VersionsFolder, "")},
+		"/buckets/bkt/foo" + s3_constants.VersionsFolder: versions,
+	}}
+	listFn := FilerListFunc(client, "/buckets")
+	var got []*bootstrap.Entry
+	require.NoError(t, listFn(context.Background(), "bkt", "", func(e *bootstrap.Entry) error {
+		got = append(got, e)
+		return nil
+	}))
+	require.Len(t, got, 2)
+	byID := map[string]*bootstrap.Entry{}
+	for _, e := range got {
+		byID[e.VersionID] = e
+	}
+	assert.True(t, byID["vnew"].IsLatest, "newest sibling wins when pointer is absent")
+	assert.False(t, byID["vold"].IsLatest)
+}
+
+func TestFilerListFunc_VersionedExpansionExplicitNullIsLatestWhenPointerMissing(t *testing.T) {
+	// Suspended-versioning shape: bare object marked with
+	// ExtVersionIdKey="null", parent has no pointer. null is latest.
+	t1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	tNull := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) // newest
+	bareNull := fileWithExt("foo", tNull, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte("null")})
+	client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
+		"/buckets/bkt": {
+			bareNull,
+			versionsDir("foo"+s3_constants.VersionsFolder, ""),
+		},
+		"/buckets/bkt/foo" + s3_constants.VersionsFolder: {
+			fileWithExt("v1", t1, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte("v1")}),
+		},
+	}}
+	listFn := FilerListFunc(client, "/buckets")
+	var got []*bootstrap.Entry
+	require.NoError(t, listFn(context.Background(), "bkt", "", func(e *bootstrap.Entry) error {
+		got = append(got, e)
+		return nil
+	}))
+	// 2 sibling entries from expansion (null + v1). The bare "foo"
+	// MUST be suppressed in pass 2 via skipBare.
+	require.Len(t, got, 2)
+	byID := map[string]*bootstrap.Entry{}
+	for _, e := range got {
+		byID[e.VersionID] = e
+	}
+	require.NotNil(t, byID["null"])
+	require.NotNil(t, byID["v1"])
+	assert.True(t, byID["null"].IsLatest)
+	assert.False(t, byID["v1"].IsLatest)
+
+	// Walk again, verify no duplicate emission of the bare "foo".
+	count := 0
+	require.NoError(t, listFn(context.Background(), "bkt", "", func(e *bootstrap.Entry) error {
+		if e.Path == "foo" && e.VersionID == "" {
+			t.Errorf("bare foo should be suppressed by skipBare, got %+v", e)
+		}
+		count++
+		return nil
+	}))
+	assert.Equal(t, 2, count)
+}
+
+func TestFilerListFunc_VersionedExpansionExplicitNullBeatsNewerSiblingWhenPointerMissing(t *testing.T) {
+	// Suspended-versioning current-ness is carried by the explicit bare
+	// null, not by mtime ordering. Backdating the null to make it due
+	// must not demote it behind a newer noncurrent sibling when the
+	// .versions pointer is absent.
+	tNull := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) // older
+	tV1 := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC)   // newer
+	bareNull := fileWithExt("foo", tNull, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte("null")})
+	client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
+		"/buckets/bkt": {
+			bareNull,
+			versionsDir("foo"+s3_constants.VersionsFolder, ""),
+		},
+		"/buckets/bkt/foo" + s3_constants.VersionsFolder: {
+			fileWithExt("v1", tV1, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte("v1")}),
+		},
+	}}
+	listFn := FilerListFunc(client, "/buckets")
+	var got []*bootstrap.Entry
+	require.NoError(t, listFn(context.Background(), "bkt", "", func(e *bootstrap.Entry) error {
+		got = append(got, e)
+		return nil
+	}))
+	require.Len(t, got, 2)
+	byID := map[string]*bootstrap.Entry{}
+	for _, e := range got {
+		byID[e.VersionID] = e
+	}
+	require.NotNil(t, byID["null"])
+	require.NotNil(t, byID["v1"])
+	assert.True(t, byID["null"].IsLatest, "explicit null must stay latest even when older than noncurrent siblings")
+	assert.False(t, byID["v1"].IsLatest)
+}
+
+func TestFilerListFunc_VersionsDirWithoutMarkersRecursesAsRegular(t *testing.T) {
+	// A `.versions`-named folder whose children have no
+	// ExtVersionIdKey is a coincidence (user folder). Recurse into
+	// it; the file inside should surface as a regular entry.
+	mtime := time.Now()
+	client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
+		"/buckets/bkt": {versionsDir("looksLikeUserFolder"+s3_constants.VersionsFolder, "")},
+		"/buckets/bkt/looksLikeUserFolder" + s3_constants.VersionsFolder: {
+			file("inner.txt", mtime, 1),
+		},
+	}}
+	listFn := FilerListFunc(client, "/buckets")
+	var paths []string
+	require.NoError(t, listFn(context.Background(), "bkt", "", func(e *bootstrap.Entry) error {
+		paths = append(paths, e.Path)
+		return nil
+	}))
+	assert.Equal(t, []string{"looksLikeUserFolder" + s3_constants.VersionsFolder + "/inner.txt"}, paths)
+}
+
+func TestFilerListFunc_VersionedDeleteMarkerPropagates(t *testing.T) {
+	mtime := time.Now()
+	versions := []*filer_pb.Entry{
+		fileWithExt("v1", mtime, 0, map[string][]byte{
+			s3_constants.ExtVersionIdKey:    []byte("v1"),
+			s3_constants.ExtDeleteMarkerKey: []byte("true"),
+		}),
+	}
+	client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
+		"/buckets/bkt": {versionsDir("foo"+s3_constants.VersionsFolder, "v1")},
+		"/buckets/bkt/foo" + s3_constants.VersionsFolder: versions,
+	}}
+	listFn := FilerListFunc(client, "/buckets")
+	var got *bootstrap.Entry
+	require.NoError(t, listFn(context.Background(), "bkt", "", func(e *bootstrap.Entry) error {
+		got = e
+		return nil
+	}))
+	require.NotNil(t, got)
+	assert.True(t, got.IsDeleteMarker, "ExtDeleteMarkerKey='true' must surface as IsDeleteMarker")
+}
+
+func TestFilerListFunc_HonorsStart(t *testing.T) {
+	// The walker's kill-resume contract: skip entries whose Path <= start.
+	mtime := time.Now()
+	client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
+		"/buckets/bkt": {file("a", mtime, 1), file("b", mtime, 1), file("c", mtime, 1)},
+	}}
+	listFn := FilerListFunc(client, "/buckets")
+	var got []string
+	require.NoError(t, listFn(context.Background(), "bkt", "a", func(e *bootstrap.Entry) error {
+		got = append(got, e.Path)
+		return nil
+	}))
+	assert.Equal(t, []string{"b", "c"}, got)
+}
+
+func TestFilerListFunc_NilClient(t *testing.T) {
+	listFn := FilerListFunc(nil, "/buckets")
+	require.Error(t, listFn(context.Background(), "bkt", "", func(*bootstrap.Entry) error { return nil }))
+}
+
+func TestFilerListFunc_AttributesPropagate(t *testing.T) {
+	mtime := time.Date(2026, 5, 11, 12, 0, 0, 1234, time.UTC)
+	client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
+		"/buckets/bkt": {file("obj", mtime, 4096)},
+	}}
+	listFn := FilerListFunc(client, "/buckets")
+	var got *bootstrap.Entry
+	require.NoError(t, listFn(context.Background(), "bkt", "", func(e *bootstrap.Entry) error {
+		got = e
+		return nil
+	}))
+	require.NotNil(t, got)
+	assert.Equal(t, "obj", got.Path)
+	assert.Equal(t, mtime.Unix(), got.ModTime.Unix())
+	assert.Equal(t, int64(1234), got.ModTime.UnixNano()-mtime.Unix()*int64(time.Second))
+	assert.Equal(t, int64(4096), got.Size)
+	assert.True(t, got.IsLatest)
+}

weed/s3api/s3lifecycle/dailyrun/jitter_test.go

@@ -0,0 +1,44 @@
+package dailyrun
+
+import (
+	"testing"
+	"time"
+)
+
+func TestJitterBounds(t *testing.T) {
+	cases := []time.Duration{
+		200 * time.Millisecond,
+		1 * time.Second,
+		5 * time.Second,
+	}
+
+	for _, d := range cases {
+		for i := 0; i < 100; i++ {
+			j := jitter(d)
+			if j < d/2 {
+				t.Errorf("jitter(%v) = %v, below lower bound %v", d, j, d/2)
+			}
+			if j >= d {
+				t.Errorf("jitter(%v) = %v, at or above upper bound %v", d, j, d)
+			}
+		}
+	}
+}
+
+func TestJitterZeroAndNegative(t *testing.T) {
+	if j := jitter(0); j != 0 {
+		t.Errorf("jitter(0) = %v, want 0", j)
+	}
+	if j := jitter(-1 * time.Second); j != 0 {
+		t.Errorf("jitter(-1s) = %v, want 0", j)
+	}
+}
+
+func TestJitterTinyDuration(t *testing.T) {
+	// When d < 2, half == 0 and rand.Int63n(0) panics.
+	// We should return d unmodified in that case.
+	j := jitter(1 * time.Nanosecond)
+	if j != 1*time.Nanosecond {
+		t.Errorf("jitter(1ns) = %v, want 1ns", j)
+	}
+}

weed/s3api/s3lifecycle/dailyrun/process_matches_test.go

@@ -0,0 +1,197 @@
+package dailyrun
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	dto "github.com/prometheus/client_model/go"
+	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// recordingClient captures every LifecycleDelete request the
+// daily-run path emits. Default outcome is DONE; tests that need
+// other outcomes set responses by index, or outcomeByObject when
+// shards dispatch concurrently and call order isn't fixed.
+type recordingClient struct {
+	mu              sync.Mutex
+	requests        []*s3_lifecycle_pb.LifecycleDeleteRequest
+	responses       []s3_lifecycle_pb.LifecycleDeleteOutcome
+	outcomeByObject map[string]s3_lifecycle_pb.LifecycleDeleteOutcome
+	calls           atomic.Int32
+}
+
+func (c *recordingClient) LifecycleDelete(_ context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
+	c.mu.Lock()
+	c.requests = append(c.requests, req)
+	idx := int(c.calls.Add(1)) - 1
+	byObject, pinned := c.outcomeByObject[req.ObjectPath]
+	c.mu.Unlock()
+	if pinned {
+		return &s3_lifecycle_pb.LifecycleDeleteResponse{Outcome: byObject}, nil
+	}
+	out := s3_lifecycle_pb.LifecycleDeleteOutcome_DONE
+	if idx < len(c.responses) {
+		out = c.responses[idx]
+	}
+	return &s3_lifecycle_pb.LifecycleDeleteResponse{Outcome: out}, nil
+}
+
+func (c *recordingClient) seenObjects() []string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	out := make([]string, 0, len(c.requests))
+	for _, r := range c.requests {
+		out = append(out, r.ObjectPath)
+	}
+	return out
+}
+
+// Phase 2's processMatches must NOT use a per-rule done flag.
+// routePointerTransitionExpand can return multiple matches for the
+// same ActionKey on different objects, each with its own
+// SuccessorModTime from a distinct demoting event. A not-yet-due
+// sibling must never gate a sibling that's already past its DueTime.
+// Pin that here.
+func TestProcessMatches_NotYetDueDoesNotSuppressDueOnSameRule(t *testing.T) {
+	runNow := time.Now()
+	rh := [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
+	rule := s3lifecycle.ActionKey{
+		Bucket:     "b",
+		RuleHash:   rh,
+		ActionKind: s3lifecycle.ActionKindNoncurrentDays,
+	}
+	// Three matches for the same ActionKey, different objects, mixed
+	// due-times. Order intentionally puts the not-yet-due match first
+	// so a buggy per-rule done flag would suppress the others.
+	matches := []router.Match{
+		{Key: rule, Bucket: "b", ObjectKey: "future-sibling", DueTime: runNow.Add(48 * time.Hour)},
+		{Key: rule, Bucket: "b", ObjectKey: "due-sibling-a", DueTime: runNow.Add(-48 * time.Hour)},
+		{Key: rule, Bucket: "b", ObjectKey: "due-sibling-b", DueTime: runNow.Add(-24 * time.Hour)},
+	}
+	client := &recordingClient{}
+	cfg := Config{Client: client}
+	skipped, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{TsNs: runNow.UnixNano()}, matches)
+	require.NoError(t, err)
+	require.False(t, halted)
+	require.True(t, skipped, "at least one future-DueTime match must flag skippedAny")
+
+	// Both due siblings must have been dispatched.
+	seen := client.seenObjects()
+	assert.ElementsMatch(t, []string{"due-sibling-a", "due-sibling-b"}, seen,
+		"due matches must dispatch regardless of where the future-sibling sits in the slice")
+}
+
+func TestProcessMatches_OrderingDoesNotMatter(t *testing.T) {
+	runNow := time.Now()
+	rh := [8]byte{0xaa}
+	rule := s3lifecycle.ActionKey{Bucket: "b", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}
+	matches := []router.Match{
+		{Key: rule, Bucket: "b", ObjectKey: "a", DueTime: runNow.Add(-2 * time.Hour)},
+		{Key: rule, Bucket: "b", ObjectKey: "b", DueTime: runNow.Add(-1 * time.Hour)},
+		{Key: rule, Bucket: "b", ObjectKey: "future", DueTime: runNow.Add(time.Hour)},
+	}
+	client := &recordingClient{}
+	cfg := Config{Client: client}
+	skipped, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{}, matches)
+	require.NoError(t, err)
+	require.False(t, halted)
+	require.True(t, skipped)
+	assert.ElementsMatch(t, []string{"a", "b"}, client.seenObjects())
+}
+
+func TestProcessMatches_HaltOnServerOutcomeStopsRemaining(t *testing.T) {
+	runNow := time.Now()
+	rh := [8]byte{0xbb}
+	rule := s3lifecycle.ActionKey{Bucket: "b", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}
+	matches := []router.Match{
+		{Key: rule, Bucket: "b", ObjectKey: "ok", DueTime: runNow.Add(-time.Hour)},
+		{Key: rule, Bucket: "b", ObjectKey: "blocked", DueTime: runNow.Add(-time.Hour)},
+		{Key: rule, Bucket: "b", ObjectKey: "would-be-next", DueTime: runNow.Add(-time.Hour)},
+	}
+	client := &recordingClient{responses: []s3_lifecycle_pb.LifecycleDeleteOutcome{
+		s3_lifecycle_pb.LifecycleDeleteOutcome_DONE,
+		s3_lifecycle_pb.LifecycleDeleteOutcome_BLOCKED,
+		s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, // would be a bug if reached
+	}}
+	cfg := Config{Client: client}
+	_, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{}, matches)
+	require.NoError(t, err)
+	require.True(t, halted, "BLOCKED outcome must halt the event loop")
+	assert.Equal(t, []string{"ok", "blocked"}, client.seenObjects(),
+		"would-be-next must NOT be dispatched after the halt")
+}
+
+func TestProcessMatches_EmptyMatchesIsNoop(t *testing.T) {
+	runNow := time.Now()
+	client := &recordingClient{}
+	cfg := Config{Client: client}
+	skipped, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{}, nil)
+	require.NoError(t, err)
+	require.False(t, halted)
+	require.False(t, skipped)
+	assert.Empty(t, client.seenObjects())
+}
+
+func TestProcessMatches_AllDueNoSkippedFlag(t *testing.T) {
+	// All matches past their DueTime — skippedAny must be false so the
+	// caller can advance the cursor past this event.
+	runNow := time.Now()
+	rh := [8]byte{0xcc}
+	rule := s3lifecycle.ActionKey{Bucket: "b", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}
+	matches := []router.Match{
+		{Key: rule, Bucket: "b", ObjectKey: "a", DueTime: runNow.Add(-time.Hour)},
+		{Key: rule, Bucket: "b", ObjectKey: "b", DueTime: runNow.Add(-30 * time.Minute)},
+	}
+	client := &recordingClient{}
+	cfg := Config{Client: client}
+	skipped, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{}, matches)
+	require.NoError(t, err)
+	require.False(t, halted)
+	require.False(t, skipped, "every match was past DueTime; skippedAny must be false")
+}
+
+func TestProcessMatches_DispatchCounterIncrements(t *testing.T) {
+	// Pin that dispatched matches increment S3LifecycleDispatchCounter
+	// with the outcome label so a refactor doesn't silently drop the
+	// observability hook. Use a bucket/kind no other test touches to
+	// keep the read-after-write stable.
+	runNow := time.Now()
+	rh := [8]byte{0xfe}
+	rule := s3lifecycle.ActionKey{Bucket: "metrics-pin-bkt", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}
+	matches := []router.Match{
+		{Key: rule, Bucket: "metrics-pin-bkt", ObjectKey: "obj", DueTime: runNow.Add(-time.Hour)},
+	}
+	client := &recordingClient{} // default DONE
+	before := dispatchCounterValue("metrics-pin-bkt", "expiration_days", "DONE")
+	// Delete the label row on exit so this test doesn't leak into the
+	// in-process Prometheus registry that other tests share.
+	defer stats.S3LifecycleDispatchCounter.DeleteLabelValues("metrics-pin-bkt", "expiration_days", "DONE")
+	cfg := Config{Client: client}
+	_, _, err := processMatches(context.Background(), cfg, runNow, &reader.Event{}, matches)
+	require.NoError(t, err)
+	after := dispatchCounterValue("metrics-pin-bkt", "expiration_days", "DONE")
+	assert.Equal(t, before+1, after, "DONE outcome must increment the dispatch counter")
+}
+
+// dispatchCounterValue reads the current value of the shared
+// S3LifecycleDispatchCounter for the given (bucket, kind, outcome).
+func dispatchCounterValue(bucket, kind, outcome string) float64 {
+	m := stats.S3LifecycleDispatchCounter.WithLabelValues(bucket, kind, outcome)
+	var pm dto.Metric
+	if err := m.Write(&pm); err != nil {
+		return 0
+	}
+	if pm.Counter == nil {
+		return 0
+	}
+	return pm.Counter.GetValue()
+}

weed/s3api/s3lifecycle/dailyrun/run.go

@@ -0,0 +1,772 @@
+package dailyrun
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"golang.org/x/time/rate"
+)
+
+// LifecycleClient mirrors dispatcher's contract; duplicated to avoid an
+// import cycle.
+type LifecycleClient interface {
+	LifecycleDelete(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error)
+}
+
+// WalkerFunc handles the per-shard bucket walk for a given engine view.
+// Phase 4b uses it on the recovery branch (rule-content edit / partition
+// flip) so already-due objects across the rewritten rule set get caught
+// before the cursor rewinds.
+type WalkerFunc func(ctx context.Context, view *engine.Snapshot, shardID int) error
+
+type Config struct {
+	Shards      []int
+	BucketsPath string
+	Engine      *engine.Engine
+	FilerClient filer_pb.SeaweedFilerClient
+	Client      LifecycleClient
+	Persister   CursorPersister
+	Lister      router.SiblingLister
+
+	// Workers <= 0 -> 1 (serial).
+	Workers int
+
+	// nil -> no rate limit. Shared across all shard goroutines.
+	Limiter *rate.Limiter
+
+	// Meta-log retention boundary. Rules whose effective TTL exceeds
+	// this can't be serviced by replay alone and get partitioned into
+	// the walk view (engine.PromotedHash). 0 falls back to maxTTL,
+	// which keeps PromotedHash empty and the partition-flip recovery
+	// trigger dormant.
+	RetentionWindow time.Duration
+
+	// Walker is invoked on the recovery branch (rule-content edit or
+	// partition flip) before the cursor rewind. It receives the
+	// engine.RecoveryView so only the rules that need bulk re-evaluation
+	// are walked, and the per-shard ID so the implementation can filter
+	// entries. nil disables walker invocation entirely — the cursor
+	// still rewinds, matching Phase 4a behavior.
+	Walker WalkerFunc
+
+	// WalkerInterval throttles the steady-state and empty-replay walker
+	// invocations: the walker fires only when the time since the
+	// per-shard cursor's LastWalkedNs exceeds this value. Cold-start
+	// and recovery walker fires (RecoveryView) are unconditional —
+	// those are bounded events that must run once when the trigger
+	// condition is met. 0 means "fire on every pass" (the prior
+	// behavior, suitable for s3tests-style rapid-cadence drivers and
+	// for in-repo integration tests).
+	//
+	// Production deployments should set this to roughly the walk cost
+	// budget per shard per cluster: e.g., 1h for a small cluster, 6h
+	// for a large one. The walker reads the whole bucket subtree it
+	// covers, so a too-short interval crushes the filer; a too-long
+	// interval delays ExpirationDate and ExpiredObjectDeleteMarker
+	// dispatches.
+	WalkerInterval time.Duration
+
+	ClientName string
+	// 0 -> randomized per-run.
+	ClientID int32
+
+	// nil -> time.Now.
+	Now func() time.Time
+
+	// 0 -> unbounded.
+	EventBudget int
+
+	// 0 -> defaultSubscriptionReceiveTimeout.
+	SubscriptionReceiveTimeout time.Duration
+}
+
+// Above the filer's 15m maxGapStall, so a subscriber parked on a gap is
+// not mistaken for a stalled one.
+const defaultSubscriptionReceiveTimeout = 20 * time.Minute
+
+// Run executes the daily replay for every shard in cfg.Shards
+// concurrently. Returns the first shard error; the rest log and run to
+// completion so one shard's transient failure doesn't lose other shards'
+// progress.
+//
+// All cfg.Shards share one meta-log subscription. The Reader's
+// ShardPredicate accepts any shard in cfg.Shards, and a fan-out
+// goroutine routes events to per-shard channels by ev.ShardID. This
+// replaces the earlier per-shard Reader that opened 16 SubscribeMetadata
+// streams to filer per pass.
+func Run(ctx context.Context, cfg Config) error {
+	if err := validate(cfg); err != nil {
+		return err
+	}
+	now := cfg.Now
+	if now == nil {
+		now = func() time.Time { return time.Now().UTC() }
+	}
+	// Freeze "now" so shards agree on the boundary.
+	runNow := now()
+	startedAt := time.Now()
+
+	// Capture once so a mid-run Compile can't make shards disagree.
+	snap := cfg.Engine.Snapshot()
+	rsh := engine.ReplayContentHash(snap)
+	maxTTL := engine.MaxEffectiveTTL(snap)
+
+	// Since all shards share one subscription, every shard goroutine
+	// must be live to drain its channel — capping concurrency would
+	// stall the fan-out. cfg.Workers no longer gates shard concurrency
+	// (dispatch is throttled via cfg.Limiter); accepted for backwards
+	// compatibility but otherwise inert.
+	_ = cfg.Workers
+
+	// rsh==[32]byte{} means no replay-eligible rules — runShard's
+	// walker-only branch fires and saves; no subscription needed.
+	var (
+		shardEvents     map[int]chan *reader.Event
+		readerDone      chan error
+		fanoutDone      chan struct{}
+		cancelRead      context.CancelFunc
+		globalStartTsNs int64
+		// Set by the reader goroutine when it stops for a reason we asked
+		// for, rather than a broken stream.
+		stoppedOnPurpose atomic.Bool
+	)
+	if rsh != [32]byte{} {
+		// Pre-load all per-shard cursors so the shared subscription
+		// can start from min(per-shard startTsNs). Using the cold-start
+		// floor (runNow - maxTTL) globally would skip past pending
+		// events on shards whose cursor is older than the floor —
+		// exactly the case where a rule's TTL equals maxTTL and an
+		// older event still has DueTime <= runNow.
+		globalStartTsNs = computeGlobalStartTsNs(ctx, cfg, runNow, maxTTL)
+		shardEvents, readerDone, fanoutDone, cancelRead = startSharedSubscription(ctx, cfg, runNow, globalStartTsNs, &stoppedOnPurpose)
+		defer cancelRead()
+	}
+
+	var wg sync.WaitGroup
+	errCh := make(chan error, len(cfg.Shards))
+	for _, sh := range cfg.Shards {
+		sh := sh
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			var ch <-chan *reader.Event
+			if shardEvents != nil {
+				ch = shardEvents[sh]
+			}
+			if err := runShard(ctx, cfg, snap, runNow, sh, ch); err != nil {
+				errCh <- err
+			}
+			// A halted dispatch returns runShard mid-stream. Keep
+			// discarding, or the fan-out blocks on this shard's full
+			// buffer and starves every other drain.
+			if ch != nil {
+				for range ch {
+				}
+			}
+		}()
+	}
+	wg.Wait()
+	close(errCh)
+
+	// Tear down the shared subscription. cancelRead unblocks both the
+	// reader's gRPC stream and the fan-out's send loop; we wait on both
+	// so their goroutines don't outlive Run.
+	var readerErr error
+	if cancelRead != nil {
+		cancelRead()
+		<-fanoutDone
+		if rerr := <-readerDone; rerr != nil && !stoppedOnPurpose.Load() {
+			// The drains ended on a closed channel, so the pass is
+			// truncated. Cursors hold what was processed; what matters is
+			// that the job doesn't go green on a dead subscription.
+			readerErr = fmt.Errorf("shared meta-log subscription: %w", rerr)
+			glog.Warningf("daily_run: %v", readerErr)
+		}
+	}
+
+	var first error
+	errCount := 0
+	for err := range errCh {
+		errCount++
+		if first == nil {
+			first = err
+		} else {
+			glog.V(1).Infof("daily_run: additional shard error: %v", err)
+		}
+	}
+	if readerErr != nil {
+		errCount++
+		if first == nil {
+			first = readerErr
+		}
+	}
+	status := "ok"
+	if first != nil {
+		status = "error"
+	}
+	// Summary line is parsed by operator scripts and CI log greps; keep
+	// the existing key=value tokens stable and append new ones at the
+	// end. cursor_lag_max / walked_max_age are the per-pass observability
+	// floor — for finer detail, scrape the per-shard gauges.
+	lagSummary := summarizeShardCursorLag(ctx, cfg, runNow)
+	glog.V(0).Infof("daily_run: status=%s shards=%d errors=%d duration=%s %s",
+		status, len(cfg.Shards), errCount, time.Since(startedAt).Round(time.Millisecond), lagSummary)
+	return first
+}
+
+// summarizeShardCursorLag walks the persisted cursors one more time
+// after the run, computes the worst (largest) lag and walker-age across
+// the cfg.Shards set, and renders them as key=value tokens. Returns
+// a string suitable for appending to the heartbeat line. Load errors
+// short-circuit to a marker token so the heartbeat doesn't lie about
+// the state.
+func summarizeShardCursorLag(ctx context.Context, cfg Config, runNow time.Time) string {
+	var (
+		maxLag    time.Duration
+		maxAge    time.Duration
+		anyCursor bool
+		anyWalked bool
+	)
+	for _, sh := range cfg.Shards {
+		c, found, err := cfg.Persister.Load(ctx, sh)
+		if err != nil || !found {
+			continue
+		}
+		if c.TsNs > 0 {
+			lag := runNow.Sub(time.Unix(0, c.TsNs))
+			if !anyCursor || lag > maxLag {
+				maxLag = lag
+				anyCursor = true
+			}
+		}
+		if c.LastWalkedNs > 0 {
+			age := runNow.Sub(time.Unix(0, c.LastWalkedNs))
+			if !anyWalked || age > maxAge {
+				maxAge = age
+				anyWalked = true
+			}
+		}
+	}
+	lagStr := "cursor_lag_max=cold"
+	if anyCursor {
+		lagStr = fmt.Sprintf("cursor_lag_max=%s", maxLag.Round(time.Second))
+	}
+	ageStr := "walked_max_age=cold"
+	if anyWalked {
+		ageStr = fmt.Sprintf("walked_max_age=%s", maxAge.Round(time.Second))
+	}
+	return lagStr + " " + ageStr
+}
+
+// computeGlobalStartTsNs scans every shard's persisted cursor and
+// returns the minimum startTsNs the shared subscription must cover.
+// For each shard the start point is the persisted cursor (steady state)
+// or runNow - maxTTL (cold start, when no cursor exists). Load errors
+// downgrade to the cold-start floor for that shard — failing closed
+// would be worse than re-scanning maxTTL of events.
+func computeGlobalStartTsNs(ctx context.Context, cfg Config, runNow time.Time, maxTTL time.Duration) int64 {
+	floor := runNow.Add(-maxTTL).UnixNano()
+	min := int64(0)
+	first := true
+	for _, sh := range cfg.Shards {
+		var start int64
+		if persisted, found, err := cfg.Persister.Load(ctx, sh); err == nil && found {
+			start = persisted.TsNs
+		} else {
+			start = floor
+		}
+		if first || start < min {
+			min = start
+			first = false
+		}
+	}
+	if first {
+		// No shards (validate() should have caught this, but defensive).
+		return floor
+	}
+	return min
+}
+
+// startSharedSubscription opens one SubscribeMetadata stream covering
+// every shard in cfg.Shards and fans events out to per-shard channels.
+// Subscription floor is the caller-supplied globalStartTsNs (typically
+// min over per-shard cursors). Shards whose own startTsNs is fresher
+// filter out already-past events themselves inside drainShardEvents.
+// Events arriving with TsNs > runUpTo (the pass boundary) cause the
+// fan-out to cancel the reader and close all per-shard channels,
+// ending the pass.
+func startSharedSubscription(ctx context.Context, cfg Config, runNow time.Time, globalStartTsNs int64, stoppedOnPurpose *atomic.Bool) (map[int]chan *reader.Event, chan error, chan struct{}, context.CancelFunc) {
+	// Set by the fan-out when it ends the pass on an event past runUpTo.
+	var pastBoundary atomic.Bool
+	shardSet := make(map[int]bool, len(cfg.Shards))
+	shardEvents := make(map[int]chan *reader.Event, len(cfg.Shards))
+	for _, sh := range cfg.Shards {
+		shardSet[sh] = true
+		// Per-shard channel size has to absorb event bursts between
+		// drains' iterations without backpressuring the fan-out.
+		// 256 covers a busy test bucket; production tuning can lift
+		// this further.
+		shardEvents[sh] = make(chan *reader.Event, 256)
+	}
+
+	clientName := cfg.ClientName
+	if clientName == "" {
+		clientName = "worker-s3-lifecycle-daily"
+	}
+	clientID := cfg.ClientID
+	if clientID == 0 {
+		clientID = int32(util.RandomInt32())
+	}
+
+	runUpTo := runNow.UnixNano()
+	receiveTimeout := cfg.SubscriptionReceiveTimeout
+	if receiveTimeout == 0 {
+		receiveTimeout = defaultSubscriptionReceiveTimeout
+	}
+	events := make(chan *reader.Event, 4*len(cfg.Shards))
+	rd := &reader.Reader{
+		ShardPredicate: func(id int) bool { return shardSet[id] },
+		BucketsPath:    cfg.BucketsPath,
+		StartTsNs:      globalStartTsNs,
+		// The pass covers (globalStartTsNs, runUpTo]. Unbounded, it ends
+		// only when an unrelated write pushes an event past runUpTo — so
+		// a quiet cluster wedges it.
+		UntilTsNs:      runUpTo,
+		Events:         events,
+		EventBudget:    cfg.EventBudget,
+		ReceiveTimeout: receiveTimeout,
+	}
+
+	readerCtx, cancelReader := context.WithCancel(ctx)
+	readerDone := make(chan error, 1)
+	go func() {
+		rerr := rd.Run(readerCtx, cfg.FilerClient, clientName, clientID)
+		// Not decidable from the error: a stream we cancel and one the
+		// filer cancels both arrive as codes.Canceled. Snapshot intent
+		// here rather than after teardown, or a deadline expiring during
+		// the drains and cursor saves masks a real failure.
+		stoppedOnPurpose.Store(ctx.Err() != nil || pastBoundary.Load())
+		readerDone <- rerr
+		// Only sender. Closing is what ends the fan-out, and through it
+		// every drain, when the stream finishes on its own.
+		close(events)
+	}()
+
+	fanoutDone := make(chan struct{})
+	go func() {
+		defer close(fanoutDone)
+		defer func() {
+			for _, ch := range shardEvents {
+				close(ch)
+			}
+		}()
+		for {
+			select {
+			case <-readerCtx.Done():
+				return
+			case ev, ok := <-events:
+				if !ok {
+					return
+				}
+				if ev == nil {
+					continue
+				}
+				// Meta-log events arrive in TsNs order; the first
+				// event past runUpTo means everything after is past
+				// too. Cancel the reader so subsequent passes don't
+				// pay for stream tail we'd drop anyway.
+				if ev.TsNs > runUpTo {
+					pastBoundary.Store(true)
+					cancelReader()
+					return
+				}
+				ch := shardEvents[ev.ShardID]
+				if ch == nil {
+					continue
+				}
+				select {
+				case <-readerCtx.Done():
+					return
+				case ch <- ev:
+				}
+			}
+		}
+	}()
+
+	return shardEvents, readerDone, fanoutDone, cancelReader
+}
+
+func validate(cfg Config) error {
+	if cfg.Engine == nil {
+		return errors.New("daily_run: nil Engine")
+	}
+	if cfg.FilerClient == nil {
+		return errors.New("daily_run: nil FilerClient")
+	}
+	if cfg.Client == nil {
+		return errors.New("daily_run: nil Client (LifecycleDelete RPC)")
+	}
+	if cfg.Persister == nil {
+		return errors.New("daily_run: nil Persister")
+	}
+	if cfg.Lister == nil {
+		return errors.New("daily_run: nil Lister")
+	}
+	if cfg.BucketsPath == "" {
+		return errors.New("daily_run: empty BucketsPath")
+	}
+	// A negative WalkerInterval would silently fall through walkerDue's
+	// `interval <= 0` branch and re-enable "walk every pass", defeating
+	// the throttle whose whole point is bounding filer load. The
+	// admin-config parser already clamps negative input to zero
+	// (worker/tasks/s3_lifecycle/config.go), but callers using
+	// dailyrun.Config directly (tests, embedders, future drivers) get
+	// a loud failure instead.
+	if cfg.WalkerInterval < 0 {
+		return fmt.Errorf("daily_run: negative WalkerInterval %v (0 = unthrottled, positive values throttle)", cfg.WalkerInterval)
+	}
+	// Negative would reach the reader as "disabled" by accident.
+	if cfg.SubscriptionReceiveTimeout < 0 {
+		return fmt.Errorf("daily_run: negative SubscriptionReceiveTimeout %v (0 = default %v)", cfg.SubscriptionReceiveTimeout, defaultSubscriptionReceiveTimeout)
+	}
+	for _, sh := range cfg.Shards {
+		if sh < 0 || sh >= s3lifecycle.ShardCount {
+			return fmt.Errorf("daily_run: shard %d out of [0, %d)", sh, s3lifecycle.ShardCount)
+		}
+	}
+	return nil
+}
+
+// runShard executes one daily-replay pass; see DESIGN.md for algorithm.
+// Three walker invocation paths under cfg.Walker (when set):
+//   - cold start / recovery: RecoveryView. Unconditional — these are
+//     bounded events (first run for a shard, or rule-content edit / partition
+//     flip) and the walker must run once when the trigger condition is met.
+//   - steady state: RulesForShard's walk view, throttled by cfg.WalkerInterval
+//     against persisted.LastWalkedNs so the walker fires on its own cadence
+//     independently of how often Run() is invoked.
+//   - empty replay: same throttling — buckets with only walker-bound rules
+//     use the walk view, and a too-fast Run() driver would otherwise burn
+//     the filer with a full subtree scan per tick.
+func runShard(ctx context.Context, cfg Config, snap *engine.Snapshot, runNow time.Time, shardID int, events <-chan *reader.Event) error {
+	shardLabel := strconv.Itoa(shardID)
+	shardStart := time.Now()
+	defer func() {
+		stats.S3LifecycleDailyRunShardDurationSeconds.WithLabelValues(shardLabel).Observe(time.Since(shardStart).Seconds())
+	}()
+	persisted, found, err := cfg.Persister.Load(ctx, shardID)
+	if err != nil {
+		return fmt.Errorf("shard=%d: load cursor: %w", shardID, err)
+	}
+
+	// engine.ReplayContentHash has a different byte layout than the
+	// Phase-2 local helper. First post-upgrade run mismatches every
+	// cursor and drops into the rule-change branch below — bounded
+	// one-time rewind to runNow - maxTTL, self-healing on save.
+	rsh := engine.ReplayContentHash(snap)
+	maxTTL := engine.MaxEffectiveTTL(snap)
+	// Operator-supplied retention falls back to maxTTL. In steady
+	// state every active replay rule has TTL <= maxTTL by construction,
+	// so promoted is empty and the partition-flip trigger is dormant.
+	// During bootstrap (rules compiled but not yet active) maxTTL is
+	// 0, retentionWindow is 0, and every rule with TTL > 0 lands in
+	// the walk partition; the resulting non-empty promoted forces a
+	// recovery walk on the first run after rules activate, which is
+	// the intended bootstrap behavior. Once the handler plumbs the
+	// real meta-log retention here, PromotedHash starts catching
+	// retention-driven partition flips in addition.
+	retentionWindow := cfg.RetentionWindow
+	if retentionWindow <= 0 {
+		retentionWindow = maxTTL
+	}
+	promoted := engine.PromotedHash(snap, retentionWindow)
+
+	// lastWalkedNs threads the steady-state walker's clock through the
+	// cursor saves below. Start with the persisted value; only update
+	// it when a steady-state / empty-replay walker fire actually
+	// happens. Cold-start and recovery walker fires also bump it so
+	// the next pass's throttle has a fresh anchor.
+	//
+	// walkedThisPass is the in-pass guard: walkerDue answers the
+	// persisted-state question ("has enough time elapsed?"), but a
+	// cold-start or recovery branch above the steady-state check
+	// already fired the walker with RecoveryView, which is a superset
+	// of every per-shard partition. Keeping the within-pass guard out
+	// of walkerDue lets that function stay pure (interval/never-walked
+	// logic only) and side-steps the test-injectable-runNow ambiguity
+	// where two distinct passes happen to share a UnixNano.
+	lastWalkedNs := persisted.LastWalkedNs
+	walkedThisPass := false
+
+	if rsh == [32]byte{} {
+		// No replay-eligible rules. Walker-only rules
+		// (ExpirationDate / ExpiredDeleteMarker / NewerNoncurrent)
+		// or scan_only-promoted rules might still need a walk; run
+		// the walker (throttled by WalkerInterval) before persisting
+		// the empty cursor and returning. Without this, a bucket whose
+		// only rule is walker-bound would never have it dispatched —
+		// the bug TestLifecycleExpirationDateInThePast caught.
+		if cfg.Walker != nil && walkerDue(lastWalkedNs, runNow, cfg.WalkerInterval) {
+			if _, walkView := snap.RulesForShard(shardID, retentionWindow); walkView != nil && len(walkView.AllActions()) > 0 {
+				if werr := cfg.Walker(ctx, walkView, shardID); werr != nil {
+					return fmt.Errorf("shard=%d: steady walk (empty replay): %w", shardID, werr)
+				}
+				lastWalkedNs = runNow.UnixNano()
+				walkedThisPass = true
+			}
+		}
+		_ = walkedThisPass // empty-replay branch returns; no downstream walker.
+		return saveCursorAndPublish(ctx, cfg.Persister, shardID, Cursor{
+			TsNs:         0,
+			RuleSetHash:  rsh,
+			PromotedHash: promoted,
+			LastWalkedNs: lastWalkedNs,
+		})
+	}
+
+	// Recovery / cold-start walker:
+	//   - found && hashes mismatch: rule edit or partition flip — walk
+	//     the rewritten rule set so already-due objects fire before the
+	//     cursor rewinds, then rewind for meta-log replay.
+	//   - !found: first run for this shard. Pre-existing objects PUT
+	//     before the rule was added live OUTSIDE the meta-log scan
+	//     window (TsNs > runNow - maxTTL) and would never replay; the
+	//     walker has to discover them. The streaming worker did this
+	//     via BucketBootstrapper; daily-replay needs the same.
+	mustWalkRecovery := found && (persisted.RuleSetHash != rsh || persisted.PromotedHash != promoted)
+	mustWalkColdStart := !found
+	if mustWalkRecovery || mustWalkColdStart {
+		if cfg.Walker != nil {
+			if werr := cfg.Walker(ctx, engine.RecoveryView(snap), shardID); werr != nil {
+				return fmt.Errorf("shard=%d: recovery walk: %w", shardID, werr)
+			}
+			lastWalkedNs = runNow.UnixNano()
+			walkedThisPass = true
+		}
+		if mustWalkRecovery {
+			// Rule changed: rewind cursor so the sliding replay re-scans
+			// the new max-TTL window and the persisted hashes match the
+			// new rule set.
+			next := Cursor{
+				TsNs:         runNow.Add(-maxTTL).UnixNano(),
+				RuleSetHash:  rsh,
+				PromotedHash: promoted,
+				LastWalkedNs: lastWalkedNs,
+			}
+			return saveCursorAndPublish(ctx, cfg.Persister, shardID, next)
+		}
+		// Cold start: keep TsNs=0 so the drain below floors to
+		// runNow - maxTTL and the cursor is saved fresh after the run.
+	}
+
+	// Steady-state walker for walker-bound and scan_only-promoted rules.
+	// RulesForShard splits the snapshot using the same retentionWindow
+	// PromotedHash used, so the walk view is exactly the partition the
+	// hash already accounted for. Empty walk view (no rules need walking
+	// today) skips the call so non-versioned, replay-only deployments
+	// don't pay an O(N) bucket-walk per run.
+	//
+	// Two-stage gate: walkedThisPass suppresses a double walk when the
+	// cold-start branch above already fired (RecoveryView covered it).
+	// walkerDue applies the persisted-state throttle for the case
+	// where this is a normal pass that didn't trigger cold-start.
+	if cfg.Walker != nil && !walkedThisPass && walkerDue(lastWalkedNs, runNow, cfg.WalkerInterval) {
+		if _, walkView := snap.RulesForShard(shardID, retentionWindow); walkView != nil && len(walkView.AllActions()) > 0 {
+			if werr := cfg.Walker(ctx, walkView, shardID); werr != nil {
+				return fmt.Errorf("shard=%d: steady walk: %w", shardID, werr)
+			}
+			lastWalkedNs = runNow.UnixNano()
+		}
+	}
+
+	// Cold start: scan from now-maxTTL so already-due objects within
+	// meta-log retention still expire. In steady state honor the
+	// cursor as-is: the drain freezes the cursor at the last pre-skip
+	// event so pending matches with DueTime == TsNs+maxTTL stay in
+	// scope across passes. Bumping forward to runNow-maxTTL would
+	// orphan exactly those events (the test_lifecyclev2_expiration
+	// regression: cursor saved at the no-match event right before
+	// the not-yet-due expire3 PUT, then floor at runNow=PUT+maxTTL
+	// equals PUT — bumping past the expire3 event itself).
+	startTsNs := persisted.TsNs
+	if !found {
+		startTsNs = runNow.Add(-maxTTL).UnixNano()
+	}
+
+	lastOK, _, drainErr := drainShardEvents(ctx, cfg, runNow, shardID, snap, startTsNs, events)
+	// Fresh ctx: a wall-clock cap on the pass (the shell driver's
+	// -runtime) cancels the drain's, and saving with it would drop the
+	// cursor and re-replay from the same floor next pass.
+	saveCtx, saveCancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer saveCancel()
+	if drainErr != nil {
+		_ = saveCursorAndPublish(saveCtx, cfg.Persister, shardID, Cursor{TsNs: lastOK, RuleSetHash: rsh, PromotedHash: promoted, LastWalkedNs: lastWalkedNs})
+		// Cut short by its wall-clock cap: truncated, not failed — the
+		// cursor above carries progress forward.
+		if errors.Is(drainErr, context.DeadlineExceeded) || errors.Is(drainErr, context.Canceled) {
+			return nil
+		}
+		return fmt.Errorf("shard=%d: drain: %w", shardID, drainErr)
+	}
+
+	return saveCursorAndPublish(saveCtx, cfg.Persister, shardID, Cursor{
+		TsNs:         lastOK,
+		RuleSetHash:  rsh,
+		PromotedHash: promoted,
+		LastWalkedNs: lastWalkedNs,
+	})
+}
+
+// saveCursorAndPublish persists the cursor and, on success, updates the
+// per-shard Prometheus gauges so operators can read cursor lag
+// (now - cursor_min_ts_ns) and walker freshness (now - last_walked_ns)
+// without having to read the cursor file directly. On save failure the
+// gauges are left untouched: their last successful-save value is more
+// useful than a value that doesn't match what's on the filer.
+func saveCursorAndPublish(ctx context.Context, p CursorPersister, shardID int, c Cursor) error {
+	if err := p.Save(ctx, shardID, c); err != nil {
+		return err
+	}
+	label := strconv.Itoa(shardID)
+	stats.S3LifecycleCursorMinTsNs.WithLabelValues(label).Set(float64(c.TsNs))
+	stats.S3LifecycleDailyRunLastWalkedNs.WithLabelValues(label).Set(float64(c.LastWalkedNs))
+	return nil
+}
+
+// walkerDue answers the persisted-state throttle: has enough time
+// elapsed since the last walker fire on this shard? interval == 0
+// means "fire every pass" (the prior, unconditional behavior).
+// lastWalkedNs == 0 means "never walked steady-state" — fire to
+// establish the throttle anchor. Otherwise gate on
+// (runNow - lastWalkedNs) >= interval.
+//
+// Within-pass double-fire suppression (cold-start / recovery walker
+// already ran this pass; steady-state branch must not re-walk a
+// superset partition) lives in runShard's walkedThisPass local flag
+// rather than here. Keeping walkerDue pure means the function answers
+// one question and a test that injects two distinct passes with the
+// same runNow doesn't get falsely throttled.
+func walkerDue(lastWalkedNs int64, runNow time.Time, interval time.Duration) bool {
+	if interval <= 0 || lastWalkedNs == 0 {
+		return true
+	}
+	return runNow.UnixNano()-lastWalkedNs >= int64(interval)
+}
+
+// drainShardEvents reads pre-fanned-out events for this shard from the
+// shared meta-log subscription and dispatches matches whose due_time is
+// past runNow. cursorAdvanceTo stops growing at the first event with a
+// not-yet-due match so that event is re-scanned in a later run.
+// halted=true marks an unresolved dispatch outcome.
+//
+// The subscription itself lives at the Run() level — one filer stream
+// fans out to per-shard channels via shardEventDispatcher. Each shard
+// drains its own channel and advances its own cursor; future-TsNs
+// events (relative to this pass's runNow) are filtered out by the
+// dispatcher before they reach this channel.
+func drainShardEvents(ctx context.Context, cfg Config, runNow time.Time, shardID int, snap *engine.Snapshot, startTsNs int64, events <-chan *reader.Event) (int64, bool, error) {
+	cursorAdvanceTo := startTsNs
+	stuck := false
+	halted := false
+
+drain:
+	for {
+		select {
+		case <-ctx.Done():
+			return cursorAdvanceTo, true, ctx.Err()
+		case ev, ok := <-events:
+			if !ok {
+				break drain
+			}
+			if ev == nil {
+				continue
+			}
+			// The global subscription starts at min(per-shard startTsNs),
+			// so this shard may receive events that are already past its
+			// own cursor. Skip them rather than re-dispatching.
+			if ev.TsNs <= startTsNs {
+				continue
+			}
+			stats.S3LifecycleDailyRunEventsScanned.WithLabelValues(strconv.Itoa(shardID)).Inc()
+			matches := router.Route(ctx, snap, ev, runNow, cfg.Lister)
+			eventSkipped, eventHalted, eventErr := processMatches(ctx, cfg, runNow, ev, matches)
+			if eventErr != nil {
+				return cursorAdvanceTo, true, eventErr
+			}
+			if eventHalted {
+				halted = true
+				break drain
+			}
+			if eventSkipped {
+				stuck = true
+				continue
+			}
+			if !stuck {
+				cursorAdvanceTo = ev.TsNs
+			}
+		}
+	}
+
+	return cursorAdvanceTo, halted, nil
+}
+
+// processMatches dispatches matches from one event. A future-DueTime
+// match is silently skipped (per-match, not per-rule): one event can
+// emit multiple matches for the same ActionKey across noncurrent
+// siblings, so a not-yet-due sibling must not gate a due one. The
+// cursor-advance gate upstream re-scans skipped events tomorrow.
+func processMatches(ctx context.Context, cfg Config, runNow time.Time, ev *reader.Event, matches []router.Match) (skippedAny, halted bool, err error) {
+	for _, m := range matches {
+		if m.DueTime.After(runNow) {
+			skippedAny = true
+			continue
+		}
+		if cfg.Limiter != nil {
+			waitStart := time.Now()
+			if waitErr := cfg.Limiter.Wait(ctx); waitErr != nil {
+				return skippedAny, true, waitErr
+			}
+			stats.S3LifecycleDispatchLimiterWaitSeconds.Observe(time.Since(waitStart).Seconds())
+		}
+		outcome, dispatchErr := dispatchWithRetry(ctx, cfg.Client, m)
+		if dispatchErr != nil {
+			glog.V(1).Infof("daily_run: transport error on %s/%s %s: %v",
+				m.Bucket, m.ObjectKey, m.Key.ActionKind, dispatchErr)
+			// "RPC_ERROR" matches the streaming dispatcher's label
+			// (dispatcher/dispatcher.go) so transport failures
+			// aggregate under one outcome key across paths.
+			stats.S3LifecycleDispatchCounter.WithLabelValues(m.Bucket, m.Key.ActionKind.String(), "RPC_ERROR").Inc()
+			return skippedAny, true, nil
+		}
+		stats.S3LifecycleDispatchCounter.WithLabelValues(m.Bucket, m.Key.ActionKind.String(), outcome.String()).Inc()
+		switch outcome {
+		case s3_lifecycle_pb.LifecycleDeleteOutcome_DONE,
+			s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED,
+			s3_lifecycle_pb.LifecycleDeleteOutcome_SKIPPED_OBJECT_LOCK:
+		case s3_lifecycle_pb.LifecycleDeleteOutcome_RETRY_LATER,
+			s3_lifecycle_pb.LifecycleDeleteOutcome_BLOCKED:
+			glog.V(1).Infof("daily_run: %s on %s/%s %s",
+				outcome, m.Bucket, m.ObjectKey, m.Key.ActionKind)
+			return skippedAny, true, nil
+		default:
+			glog.V(1).Infof("daily_run: unknown outcome %v on %s/%s", outcome, m.Bucket, m.ObjectKey)
+			return skippedAny, true, nil
+		}
+		_ = ev
+	}
+	return skippedAny, false, nil
+}

weed/s3api/s3lifecycle/dailyrun/run_bounded_test.go

@@ -0,0 +1,297 @@
+package dailyrun
+
+import (
+	"context"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// idleSubscribeStream: nothing to deliver past the pass boundary. A
+// bounded subscription gets EOF; an unbounded one blocks forever.
+type idleSubscribeStream struct {
+	grpc.ClientStream
+	ctx     context.Context
+	untilNs int64
+}
+
+func (s *idleSubscribeStream) Recv() (*filer_pb.SubscribeMetadataResponse, error) {
+	if s.untilNs != 0 {
+		return nil, io.EOF
+	}
+	<-s.ctx.Done()
+	return nil, s.ctx.Err()
+}
+
+type idleSubscribeClient struct {
+	filer_pb.SeaweedFilerClient
+	gotReq chan *filer_pb.SubscribeMetadataRequest
+}
+
+func (c *idleSubscribeClient) SubscribeMetadata(ctx context.Context, req *filer_pb.SubscribeMetadataRequest, _ ...grpc.CallOption) (filer_pb.SeaweedFiler_SubscribeMetadataClient, error) {
+	select {
+	case c.gotReq <- req:
+	default:
+	}
+	return &idleSubscribeStream{ctx: ctx, untilNs: req.UntilNs}, nil
+}
+
+// The reported wedge: the pass ended only when an unrelated write pushed
+// an event past its boundary, so a quiet cluster starved every drain and
+// Run's WaitGroup never reached zero.
+func TestRun_EndsOnIdleSubscription(t *testing.T) {
+	e := engine.New()
+	e.Compile([]engine.CompileInput{
+		{Bucket: "b1", Rules: []*s3lifecycle.Rule{
+			{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 30},
+		}},
+	}, engine.CompileOptions{})
+	require.NotEqual(t, [32]byte{}, engine.ReplayContentHash(e.Snapshot()),
+		"rule must be replay-eligible or Run skips the subscription entirely")
+
+	client := &idleSubscribeClient{gotReq: make(chan *filer_pb.SubscribeMetadataRequest, 1)}
+	runNow := time.Now().UTC()
+	cfg := Config{
+		Shards:      []int{0, 1, 2},
+		BucketsPath: "/buckets",
+		Engine:      e,
+		FilerClient: client,
+		Client:      stubLifecycleClient{},
+		Persister:   newMemPersister(),
+		Lister:      stubSiblingLister{},
+		Now:         func() time.Time { return runNow },
+	}
+
+	done := make(chan error, 1)
+	go func() { done <- Run(context.Background(), cfg) }()
+	select {
+	case err := <-done:
+		require.NoError(t, err)
+	case <-time.After(10 * time.Second):
+		t.Fatal("Run did not return on an idle subscription")
+	}
+
+	req := <-client.gotReq
+	assert.Equal(t, runNow.UnixNano(), req.UntilNs, "subscription must stop at the pass boundary")
+}
+
+// failingSubscribeClient fails the subscribe call outright.
+type failingSubscribeClient struct {
+	filer_pb.SeaweedFilerClient
+	err error
+}
+
+func (c *failingSubscribeClient) SubscribeMetadata(_ context.Context, _ *filer_pb.SubscribeMetadataRequest, _ ...grpc.CallOption) (filer_pb.SeaweedFiler_SubscribeMetadataClient, error) {
+	return nil, c.err
+}
+
+// Flip side of closing the event channel: the drains now end cleanly on
+// a dead subscription, so the pass would otherwise go green having
+// processed nothing.
+func TestRun_ReaderFailureFailsThePass(t *testing.T) {
+	e := engine.New()
+	e.Compile([]engine.CompileInput{
+		{Bucket: "b1", Rules: []*s3lifecycle.Rule{
+			{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 30},
+		}},
+	}, engine.CompileOptions{})
+
+	cfg := Config{
+		Shards:      []int{0, 1},
+		BucketsPath: "/buckets",
+		Engine:      e,
+		FilerClient: &failingSubscribeClient{err: status.Error(codes.Unavailable, "filer down")},
+		Client:      stubLifecycleClient{},
+		Persister:   newMemPersister(),
+		Lister:      stubSiblingLister{},
+	}
+
+	done := make(chan error, 1)
+	go func() { done <- Run(context.Background(), cfg) }()
+	select {
+	case err := <-done:
+		require.Error(t, err, "a dead subscription must not report a successful pass")
+		assert.Contains(t, err.Error(), "subscription")
+	case <-time.After(10 * time.Second):
+		t.Fatal("Run did not return after the subscription failed")
+	}
+}
+
+// blockingSubscribeStream never delivers; only a cap ends the pass.
+type blockingSubscribeStream struct {
+	grpc.ClientStream
+	ctx context.Context
+}
+
+func (s *blockingSubscribeStream) Recv() (*filer_pb.SubscribeMetadataResponse, error) {
+	<-s.ctx.Done()
+	// A canceled gRPC stream returns a status code, not context.Canceled.
+	return nil, status.Error(codes.Canceled, "context canceled")
+}
+
+type blockingSubscribeClient struct {
+	filer_pb.SeaweedFilerClient
+}
+
+func (c *blockingSubscribeClient) SubscribeMetadata(ctx context.Context, _ *filer_pb.SubscribeMetadataRequest, _ ...grpc.CallOption) (filer_pb.SeaweedFiler_SubscribeMetadataClient, error) {
+	return &blockingSubscribeStream{ctx: ctx}, nil
+}
+
+// Counterweight to TestRun_ServerSideCancelFailsThePass: same
+// codes.Canceled from the reader, opposite verdict. Only intent
+// separates them, which is why the decision reads ctx not the error.
+func TestRun_CappedPassIsNotAFailure(t *testing.T) {
+	e := engine.New()
+	e.Compile([]engine.CompileInput{
+		{Bucket: "b1", Rules: []*s3lifecycle.Rule{
+			{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 30},
+		}},
+	}, engine.CompileOptions{})
+
+	cfg := Config{
+		Shards:      []int{0, 1},
+		BucketsPath: "/buckets",
+		Engine:      e,
+		FilerClient: &blockingSubscribeClient{},
+		Client:      stubLifecycleClient{},
+		Persister:   newMemPersister(),
+		Lister:      stubSiblingLister{},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- Run(ctx, cfg) }()
+	select {
+	case err := <-done:
+		require.NoError(t, err, "a pass truncated by its own wall-clock cap is not a failure")
+	case <-time.After(10 * time.Second):
+		t.Fatal("Run did not return after the pass was capped")
+	}
+}
+
+// Neither UntilNs nor keepalive reaches this: a healthy connection whose
+// filer has stopped producing.
+func TestRun_StalledSubscriptionTimesOutThePass(t *testing.T) {
+	e := engine.New()
+	e.Compile([]engine.CompileInput{
+		{Bucket: "b1", Rules: []*s3lifecycle.Rule{
+			{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 30},
+		}},
+	}, engine.CompileOptions{})
+
+	cfg := Config{
+		Shards:                     []int{0, 1},
+		BucketsPath:                "/buckets",
+		Engine:                     e,
+		FilerClient:                &blockingSubscribeClient{},
+		Client:                     stubLifecycleClient{},
+		Persister:                  newMemPersister(),
+		Lister:                     stubSiblingLister{},
+		SubscriptionReceiveTimeout: 50 * time.Millisecond,
+	}
+
+	done := make(chan error, 1)
+	go func() { done <- Run(context.Background(), cfg) }()
+	select {
+	case err := <-done:
+		require.ErrorIs(t, err, reader.ErrReceiveTimeout)
+	case <-time.After(10 * time.Second):
+		t.Fatal("Run did not return on a stalled subscription")
+	}
+}
+
+func TestValidate_RejectsNegativeSubscriptionReceiveTimeout(t *testing.T) {
+	cfg := validatableConfig()
+	cfg.SubscriptionReceiveTimeout = -time.Second
+	require.ErrorContains(t, validate(cfg), "SubscriptionReceiveTimeout")
+	cfg.SubscriptionReceiveTimeout = 0
+	require.NoError(t, validate(cfg))
+}
+
+// slowSavePersister stretches teardown past the caller's deadline.
+type slowSavePersister struct {
+	*memPersister
+	delay time.Duration
+}
+
+func (p *slowSavePersister) Save(ctx context.Context, shardID int, c Cursor) error {
+	time.Sleep(p.delay)
+	return p.memPersister.Save(ctx, shardID, c)
+}
+
+// A reader that fails while the caller's deadline is still live, on a
+// pass whose teardown then outlives that deadline. Sampling ctx.Err()
+// after the drains and cursor saves would read the late deadline as
+// intent and report the truncated pass as a success.
+func TestRun_LateDeadlineDoesNotMaskReaderFailure(t *testing.T) {
+	e := engine.New()
+	e.Compile([]engine.CompileInput{
+		{Bucket: "b1", Rules: []*s3lifecycle.Rule{
+			{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 30},
+		}},
+	}, engine.CompileOptions{})
+
+	cfg := Config{
+		Shards:      []int{0, 1},
+		BucketsPath: "/buckets",
+		Engine:      e,
+		FilerClient: &failingSubscribeClient{err: status.Error(codes.Unavailable, "filer down")},
+		Client:      stubLifecycleClient{},
+		Persister:   &slowSavePersister{memPersister: newMemPersister(), delay: 300 * time.Millisecond},
+		Lister:      stubSiblingLister{},
+	}
+
+	// Deadline lands during the cursor saves, well after the reader failed.
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- Run(ctx, cfg) }()
+	select {
+	case err := <-done:
+		require.Error(t, err, "a deadline expiring during teardown must not excuse an earlier reader failure")
+		require.NotNil(t, ctx.Err(), "test is meaningless unless the deadline did expire")
+	case <-time.After(10 * time.Second):
+		t.Fatal("Run did not return")
+	}
+}
+
+// The case status-code classification could not express: the filer
+// cancels while the caller's context is untouched.
+func TestRun_ServerSideCancelFailsThePass(t *testing.T) {
+	e := engine.New()
+	e.Compile([]engine.CompileInput{
+		{Bucket: "b1", Rules: []*s3lifecycle.Rule{
+			{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 30},
+		}},
+	}, engine.CompileOptions{})
+
+	cfg := Config{
+		Shards:      []int{0, 1},
+		BucketsPath: "/buckets",
+		Engine:      e,
+		FilerClient: &failingSubscribeClient{err: status.Error(codes.Canceled, "context canceled")},
+		Client:      stubLifecycleClient{},
+		Persister:   newMemPersister(),
+		Lister:      stubSiblingLister{},
+	}
+
+	done := make(chan error, 1)
+	go func() { done <- Run(context.Background(), cfg) }()
+	select {
+	case err := <-done:
+		require.Error(t, err, "a peer-canceled stream truncates the pass and must not report success")
+	case <-time.After(10 * time.Second):
+		t.Fatal("Run did not return after the subscription was canceled")
+	}
+}

weed/s3api/s3lifecycle/dailyrun/run_fanout_test.go

@@ -0,0 +1,148 @@
+package dailyrun
+
+import (
+	"context"
+	"io"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+)
+
+// scriptedSubscribeStream replays a fixed list, then EOFs like a filer
+// honoring UntilNs.
+type scriptedSubscribeStream struct {
+	grpc.ClientStream
+	responses []*filer_pb.SubscribeMetadataResponse
+	next      int
+}
+
+func (s *scriptedSubscribeStream) Recv() (*filer_pb.SubscribeMetadataResponse, error) {
+	if s.next >= len(s.responses) {
+		return nil, io.EOF
+	}
+	resp := s.responses[s.next]
+	s.next++
+	return resp, nil
+}
+
+type scriptedSubscribeClient struct {
+	filer_pb.SeaweedFilerClient
+	responses []*filer_pb.SubscribeMetadataResponse
+}
+
+func (c *scriptedSubscribeClient) SubscribeMetadata(_ context.Context, _ *filer_pb.SubscribeMetadataRequest, _ ...grpc.CallOption) (filer_pb.SeaweedFiler_SubscribeMetadataClient, error) {
+	return &scriptedSubscribeStream{responses: c.responses}, nil
+}
+
+func subscribeResponse(bucket, key string, mtime time.Time, tsNs int64) *filer_pb.SubscribeMetadataResponse {
+	return &filer_pb.SubscribeMetadataResponse{
+		TsNs: tsNs,
+		EventNotification: &filer_pb.EventNotification{
+			NewParentPath: "/buckets/" + bucket,
+			NewEntry: &filer_pb.Entry{
+				Name:       key,
+				Attributes: &filer_pb.FuseAttributes{Mtime: mtime.Unix(), FileSize: 1},
+			},
+		},
+	}
+}
+
+// keyForShard finds an object key in bucket that hashes to shardID.
+func keyForShard(t *testing.T, bucket string, shardID int) string {
+	t.Helper()
+	for i := 0; i < 10000; i++ {
+		key := "k" + strconv.Itoa(i)
+		if s3lifecycle.ShardID(bucket, key) == shardID {
+			return key
+		}
+	}
+	t.Fatalf("no key hashes to shard %d", shardID)
+	return ""
+}
+
+// The second wedge: a drain halted by a BLOCKED dispatch stops reading
+// its channel, the fan-out blocks once the 256 buffer fills, and every
+// other shard starves behind it.
+func TestRun_HaltedShardDoesNotStarveOthers(t *testing.T) {
+	const bucket = "bk"
+	rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
+	prior := map[s3lifecycle.ActionKey]engine.PriorState{}
+	ruleHash := s3lifecycle.RuleHash(rule)
+	for _, k := range s3lifecycle.RuleActionKinds(rule) {
+		prior[s3lifecycle.ActionKey{Bucket: bucket, RuleHash: ruleHash, ActionKind: k}] = engine.PriorState{
+			BootstrapComplete: true,
+			Mode:              engine.ModeEventDriven,
+		}
+	}
+	e := engine.New()
+	snap := e.Compile([]engine.CompileInput{{Bucket: bucket, Rules: []*s3lifecycle.Rule{rule}}},
+		engine.CompileOptions{PriorStates: prior})
+
+	haltedShard, starvedShard := 0, 1
+	haltedKey := keyForShard(t, bucket, haltedShard)
+	starvedKey := keyForShard(t, bucket, starvedShard)
+
+	runNow := time.Now().UTC()
+	evTime := runNow.Add(-48 * time.Hour) // past the 1-day expiration
+	// Past the 256 buffer, so the fan-out blocks on the halted shard
+	// before reaching the starved shard's event.
+	var responses []*filer_pb.SubscribeMetadataResponse
+	for i := 0; i < 300; i++ {
+		responses = append(responses, subscribeResponse(bucket, haltedKey, evTime, evTime.UnixNano()+int64(i)))
+	}
+	responses = append(responses, subscribeResponse(bucket, starvedKey, evTime, runNow.Add(-time.Hour).UnixNano()))
+
+	// Pinned per object, not call index: the shards dispatch concurrently.
+	client := &recordingClient{outcomeByObject: map[string]s3_lifecycle_pb.LifecycleDeleteOutcome{
+		haltedKey: s3_lifecycle_pb.LifecycleDeleteOutcome_BLOCKED,
+	}}
+
+	// Resume from before the events; the cold-start floor would skip them.
+	// Hashes must match or runShard takes the recovery branch instead.
+	persister := newMemPersister()
+	seeded := Cursor{
+		TsNs:         runNow.Add(-72 * time.Hour).UnixNano(),
+		RuleSetHash:  engine.ReplayContentHash(snap),
+		PromotedHash: engine.PromotedHash(snap, engine.MaxEffectiveTTL(snap)),
+	}
+	for _, sh := range []int{haltedShard, starvedShard} {
+		require.NoError(t, persister.Save(context.Background(), sh, seeded))
+	}
+
+	cfg := Config{
+		Shards:      []int{haltedShard, starvedShard},
+		BucketsPath: "/buckets",
+		Engine:      e,
+		FilerClient: &scriptedSubscribeClient{responses: responses},
+		Client:      client,
+		Persister:   persister,
+		Lister:      stubSiblingLister{},
+		Now:         func() time.Time { return runNow },
+	}
+
+	done := make(chan error, 1)
+	go func() { done <- Run(context.Background(), cfg) }()
+	select {
+	case err := <-done:
+		require.NoError(t, err)
+	case <-time.After(10 * time.Second):
+		t.Fatal("Run wedged: the fan-out blocked on a shard that had stopped draining")
+	}
+
+	// Returning isn't enough: dropping the starved shard's events would
+	// return just as cleanly.
+	assert.Contains(t, client.seenObjects(), starvedKey,
+		"the shard behind the halted one must still get its events")
+	starvedCursor, found, err := persister.Load(context.Background(), starvedShard)
+	require.NoError(t, err)
+	require.True(t, found)
+	assert.Greater(t, starvedCursor.TsNs, seeded.TsNs, "starved shard's cursor must advance")
+}

weed/s3api/s3lifecycle/dailyrun/walk_buckets.go

@@ -0,0 +1,76 @@
+package dailyrun
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
+)
+
+// WalkBuckets runs bootstrap.Walk for each bucket using a per-bucket
+// ListFunc wrapped to drop entries whose ShardID doesn't belong to
+// shardID. Returns the first bucket error; remaining buckets log and
+// continue so one bucket's filer error doesn't kill the whole walk.
+//
+// Composable shape: callers (the worker handler) supply a real
+// filer-backed ListFunc and a real Dispatcher (WalkerDispatcher).
+// Tests pass bootstrap.EntryCallback and a stub Dispatcher.
+func WalkBuckets(ctx context.Context, view *engine.Snapshot, shardID int, buckets []string, list bootstrap.ListFunc, dispatch bootstrap.Dispatcher) error {
+	if view == nil {
+		return errors.New("WalkBuckets: nil view")
+	}
+	if list == nil {
+		return errors.New("WalkBuckets: nil list")
+	}
+	if dispatch == nil {
+		return errors.New("WalkBuckets: nil dispatch")
+	}
+	filtered := perShardListFunc(list, shardID)
+	var firstErr error
+	for _, b := range buckets {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		if _, err := bootstrap.Walk(ctx, view, b, filtered, dispatch, bootstrap.WalkOptions{}); err != nil {
+			if firstErr == nil {
+				firstErr = fmt.Errorf("walk %s: %w", b, err)
+			} else {
+				glog.V(1).Infof("walker: additional bucket %s: %v", b, err)
+			}
+		}
+	}
+	return firstErr
+}
+
+// perShardListFunc wraps an inner ListFunc so only entries whose
+// logical-key shard matches shardID reach the callback. The walker
+// emits one entry per logical object — versioned siblings share the
+// same logical key so they're either all in-shard or all out — and
+// MPU init records carry the user's destination in DestKey, so the
+// shard predicate must use DestKey there to match what the dispatcher
+// will send.
+func perShardListFunc(inner bootstrap.ListFunc, shardID int) bootstrap.ListFunc {
+	return func(ctx context.Context, bucket, start string, cb func(*bootstrap.Entry) error) error {
+		return inner(ctx, bucket, start, func(e *bootstrap.Entry) error {
+			if e == nil {
+				return nil
+			}
+			if entryShardID(bucket, e) != shardID {
+				return nil
+			}
+			return cb(e)
+		})
+	}
+}
+
+func entryShardID(bucket string, e *bootstrap.Entry) int {
+	key := e.Path
+	if e.IsMPUInit && e.DestKey != "" {
+		key = e.DestKey
+	}
+	return s3lifecycle.ShardID(bucket, key)
+}

weed/s3api/s3lifecycle/dailyrun/walk_buckets_test.go

@@ -0,0 +1,175 @@
+package dailyrun
+
+import (
+	"context"
+	"errors"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// recordingDispatcher captures every (action, entry) pair the walker
+// emits so tests can assert dispatch order and shard filtering.
+type recordingDispatcher struct {
+	calls []recordedDispatch
+	err   error
+}
+
+type recordedDispatch struct {
+	bucket string
+	path   string
+}
+
+func (d *recordingDispatcher) Delete(_ context.Context, action *engine.CompiledAction, entry *bootstrap.Entry) error {
+	d.calls = append(d.calls, recordedDispatch{bucket: action.Bucket, path: entry.Path})
+	return d.err
+}
+
+// findShardForPath returns the shard ID for an entry with given path
+// in given bucket. Helper for tests that want to force entries onto
+// a known shard.
+func findShardForPath(bucket, path string) int {
+	return s3lifecycle.ShardID(bucket, path)
+}
+
+// fixedShardEntries returns a slice of bootstrap.Entries whose paths
+// all live in the same shard. Used so a per-shard filter test has
+// deterministic expectations.
+func fixedShardEntries(t *testing.T, bucket string, shardID int, count int) []*bootstrap.Entry {
+	t.Helper()
+	var out []*bootstrap.Entry
+	for i := 0; tries(i, count); i++ {
+		path := "obj-" + intToStr(i)
+		if findShardForPath(bucket, path) == shardID {
+			out = append(out, &bootstrap.Entry{
+				Path: path,
+				// Old enough to expire under a 7-day ExpirationDays rule.
+				ModTime:  time.Now().Add(-90 * 24 * time.Hour),
+				Size:     1,
+				IsLatest: true,
+			})
+			if len(out) >= count {
+				return out
+			}
+		}
+	}
+	t.Fatalf("could not find %d entries in shard %d for bucket %s after 4096 attempts", count, shardID, bucket)
+	return nil
+}
+
+func intToStr(i int) string {
+	const hex = "0123456789abcdef"
+	if i == 0 {
+		return "0"
+	}
+	var out []byte
+	for n := i; n > 0; n /= 16 {
+		out = append([]byte{hex[n%16]}, out...)
+	}
+	return string(out)
+}
+
+func tries(i, count int) bool { return i < count*256+256 }
+
+func snapshotForBucketRule(t *testing.T, bucket string, days int) *engine.Snapshot {
+	t.Helper()
+	e := engine.New()
+	e.Compile([]engine.CompileInput{
+		{Bucket: bucket, Rules: []*s3lifecycle.Rule{
+			{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: days},
+		}},
+	}, engine.CompileOptions{})
+	snap := e.Snapshot()
+	for _, a := range snap.AllActions() {
+		snap.MarkActive(a.Key)
+	}
+	return snap
+}
+
+func TestWalkBuckets_DispatchesOnlyShardMatchingEntries(t *testing.T) {
+	bucket := "b1"
+	const targetShard = 0
+	// Build entries: half in shard 0, half elsewhere. The walker's
+	// per-shard filter must drop the non-matching ones.
+	inShard := fixedShardEntries(t, bucket, targetShard, 3)
+	otherShard := fixedShardEntries(t, bucket, (targetShard+1)%16, 3)
+	all := append([]*bootstrap.Entry{}, inShard...)
+	all = append(all, otherShard...)
+
+	snap := snapshotForBucketRule(t, bucket, 7) // 30-day-old objects expire
+	d := &recordingDispatcher{}
+	err := WalkBuckets(context.Background(), snap, targetShard, []string{bucket},
+		bootstrap.EntryCallback(all), d)
+	require.NoError(t, err)
+
+	gotPaths := make([]string, 0, len(d.calls))
+	for _, c := range d.calls {
+		gotPaths = append(gotPaths, c.path)
+		assert.Equal(t, bucket, c.bucket)
+	}
+	sort.Strings(gotPaths)
+
+	wantPaths := make([]string, 0, len(inShard))
+	for _, e := range inShard {
+		wantPaths = append(wantPaths, e.Path)
+	}
+	sort.Strings(wantPaths)
+	assert.Equal(t, wantPaths, gotPaths, "walker must dispatch exactly the in-shard entries")
+}
+
+func TestWalkBuckets_NilGuards(t *testing.T) {
+	require.Error(t, WalkBuckets(context.Background(), nil, 0, nil, bootstrap.EntryCallback(nil), &recordingDispatcher{}))
+	require.Error(t, WalkBuckets(context.Background(), snapshotForBucketRule(t, "b", 7), 0, nil, nil, &recordingDispatcher{}))
+	require.Error(t, WalkBuckets(context.Background(), snapshotForBucketRule(t, "b", 7), 0, nil, bootstrap.EntryCallback(nil), nil))
+}
+
+func TestWalkBuckets_OneBucketErrorDoesNotStopOthers(t *testing.T) {
+	// Two buckets, first ListFunc errors. WalkBuckets should still
+	// process the second bucket but return the first bucket's error.
+	listErr := errors.New("filer flake")
+	list := func(_ context.Context, bucket, _ string, _ func(*bootstrap.Entry) error) error {
+		if bucket == "bad" {
+			return listErr
+		}
+		return nil
+	}
+
+	snap := snapshotForBucketRule(t, "good", 7)
+	err := WalkBuckets(context.Background(), snap, 0, []string{"bad", "good"}, list, &recordingDispatcher{})
+	require.Error(t, err)
+	assert.ErrorIs(t, err, listErr)
+}
+
+func TestWalkBuckets_HonorsContextCancellationBetweenBuckets(t *testing.T) {
+	// Pre-cancel ctx before invocation. Even an empty bucket list
+	// must surface ctx.Err early so a long walk in progress can
+	// short-circuit.
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	err := WalkBuckets(ctx, snapshotForBucketRule(t, "b", 7), 0, []string{"b"},
+		bootstrap.EntryCallback(nil), &recordingDispatcher{})
+	assert.ErrorIs(t, err, context.Canceled)
+}
+
+func TestEntryShardID_MPUUsesDestKey(t *testing.T) {
+	bucket := "bkt"
+	// Force a (Path, DestKey) pair that maps to different shards
+	// so we can prove the filter picks the right one.
+	for i := 0; i < 4096; i++ {
+		path := ".uploads/" + intToStr(i)
+		destKey := "user/" + intToStr(i)
+		if s3lifecycle.ShardID(bucket, path) != s3lifecycle.ShardID(bucket, destKey) {
+			e := &bootstrap.Entry{Path: path, DestKey: destKey, IsMPUInit: true}
+			assert.Equal(t, s3lifecycle.ShardID(bucket, destKey), entryShardID(bucket, e),
+				"MPU init must use DestKey for shard, not the .uploads/<id> path")
+			return
+		}
+	}
+	t.Fatal("could not find an MPU path/destkey pair with diverging shards in 4096 attempts")
+}

weed/s3api/s3lifecycle/dailyrun/walker_dispatcher.go

@@ -0,0 +1,103 @@
+package dailyrun
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"golang.org/x/time/rate"
+)
+
+// WalkerDispatcher adapts LifecycleClient to bootstrap.Dispatcher so
+// the Phase 4b walker can drive the same LifecycleDelete RPC the
+// meta-log replay path uses. No CAS witness is supplied; the server's
+// identityMatches treats nil ExpectedIdentity as "bootstrap call, skip
+// witness" (see weed/s3api/s3api_internal_lifecycle.go), which is the
+// right contract for a full-tree walk that has just observed the entry.
+type WalkerDispatcher struct {
+	Client LifecycleClient
+	// Limiter throttles LifecycleDelete dispatch under the cluster's
+	// shared deletes/sec budget. Should be the same *rate.Limiter the
+	// daily-run's processMatches uses so the walker and replay paths
+	// can't combine to burst past the cap. nil disables throttling.
+	Limiter *rate.Limiter
+}
+
+// Compile-time check.
+var _ bootstrap.Dispatcher = (*WalkerDispatcher)(nil)
+
+// Delete classifies non-DONE server outcomes as errors so the walker
+// halts and the caller persists progress under the bootstrap
+// checkpoint rather than silently skipping objects.
+func (d *WalkerDispatcher) Delete(ctx context.Context, action *engine.CompiledAction, entry *bootstrap.Entry) error {
+	if d == nil || d.Client == nil {
+		return fmt.Errorf("walker dispatch: nil client")
+	}
+	if action == nil || entry == nil {
+		return fmt.Errorf("walker dispatch: nil action or entry")
+	}
+	objectPath := entry.Path
+	if entry.IsMPUInit {
+		// Rule-prefix matching uses DestKey (the user's intended
+		// object key); dispatch uses entry.Path (.uploads/<id>),
+		// which is what the server's ABORT_MPU handler expects in
+		// req.ObjectPath — it strips the .uploads/ prefix to get
+		// the upload id and reads the init record from that
+		// directory. DestKey is the dispatch ANTI-pattern here: it
+		// looks like a regular object path, the server's check for
+		// the .uploads/ prefix fails, and the dispatch comes back
+		// as BLOCKED FATAL_EVENT_ERROR.
+		if entry.DestKey == "" {
+			return fmt.Errorf("walker dispatch: MPU init entry with empty DestKey: %s", entry.Path)
+		}
+	}
+	rh := action.Key.RuleHash
+	req := &s3_lifecycle_pb.LifecycleDeleteRequest{
+		Bucket:     action.Bucket,
+		ObjectPath: objectPath,
+		VersionId:  entry.VersionID,
+		RuleHash:   rh[:],
+		ActionKind: toProtoActionKind(action.Key.ActionKind),
+		// ExpectedIdentity intentionally nil; server bootstraps from
+		// the live entry on this code path.
+	}
+	kindLabel := action.Key.ActionKind.String()
+	if d.Limiter != nil {
+		waitStart := time.Now()
+		if waitErr := d.Limiter.Wait(ctx); waitErr != nil {
+			return fmt.Errorf("walker dispatch %s/%s %s: limiter: %w", action.Bucket, objectPath, action.Key.ActionKind, waitErr)
+		}
+		stats.S3LifecycleDispatchLimiterWaitSeconds.Observe(time.Since(waitStart).Seconds())
+	}
+	resp, err := d.Client.LifecycleDelete(ctx, req)
+	if err != nil {
+		// "RPC_ERROR" matches the streaming dispatcher and processMatches
+		// labels so transport-class failures aggregate under one key.
+		stats.S3LifecycleDispatchCounter.WithLabelValues(action.Bucket, kindLabel, "RPC_ERROR").Inc()
+		return fmt.Errorf("walker dispatch %s/%s %s: %w", action.Bucket, objectPath, action.Key.ActionKind, err)
+	}
+	if resp == nil {
+		// A misbehaving server stub returning (nil, nil) would panic on
+		// the switch below. Bucket under RPC_ERROR rather than a new
+		// label — operationally it's the same class of failure.
+		stats.S3LifecycleDispatchCounter.WithLabelValues(action.Bucket, kindLabel, "RPC_ERROR").Inc()
+		return fmt.Errorf("walker dispatch %s/%s %s: nil response", action.Bucket, objectPath, action.Key.ActionKind)
+	}
+	stats.S3LifecycleDispatchCounter.WithLabelValues(action.Bucket, kindLabel, resp.Outcome.String()).Inc()
+	switch resp.Outcome {
+	case s3_lifecycle_pb.LifecycleDeleteOutcome_DONE,
+		s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED,
+		s3_lifecycle_pb.LifecycleDeleteOutcome_SKIPPED_OBJECT_LOCK:
+		return nil
+	default:
+		// RETRY_LATER / BLOCKED / UNSPECIFIED: surface as error so the
+		// walk halts at this entry and resumes from
+		// Checkpoint.LastScannedPath on the next run.
+		return fmt.Errorf("walker dispatch %s/%s %s: outcome=%s reason=%s",
+			action.Bucket, objectPath, action.Key.ActionKind, resp.Outcome, resp.Reason)
+	}
+}

weed/s3api/s3lifecycle/dailyrun/walker_dispatcher_test.go

@@ -0,0 +1,195 @@
+package dailyrun
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/time/rate"
+)
+
+// walkerStubClient captures the last LifecycleDeleteRequest so tests
+// can assert on the request shape produced by WalkerDispatcher.
+type walkerStubClient struct {
+	lastReq *s3_lifecycle_pb.LifecycleDeleteRequest
+	outcome s3_lifecycle_pb.LifecycleDeleteOutcome
+	err     error
+	reason  string
+	nilResp bool // return (nil, nil) — pin the dispatcher's defensive guard
+}
+
+func (c *walkerStubClient) LifecycleDelete(_ context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
+	c.lastReq = req
+	if c.err != nil {
+		return nil, c.err
+	}
+	if c.nilResp {
+		return nil, nil
+	}
+	return &s3_lifecycle_pb.LifecycleDeleteResponse{Outcome: c.outcome, Reason: c.reason}, nil
+}
+
+func sampleAction(t *testing.T, kind s3lifecycle.ActionKind) *engine.CompiledAction {
+	t.Helper()
+	var rh [8]byte
+	for i := range rh {
+		rh[i] = byte(0xa0 + i)
+	}
+	return &engine.CompiledAction{
+		Bucket: "bkt",
+		Key:    s3lifecycle.ActionKey{Bucket: "bkt", ActionKind: kind, RuleHash: rh},
+		Mode:   engine.ModeEventDriven,
+	}
+}
+
+func TestWalkerDispatcher_NonVersionedSendsExpectedRequest(t *testing.T) {
+	c := &walkerStubClient{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}
+	d := &WalkerDispatcher{Client: c}
+	a := sampleAction(t, s3lifecycle.ActionKindExpirationDays)
+	err := d.Delete(context.Background(), a, &bootstrap.Entry{Path: "obj"})
+	require.NoError(t, err)
+	require.NotNil(t, c.lastReq)
+	assert.Equal(t, "bkt", c.lastReq.Bucket)
+	assert.Equal(t, "obj", c.lastReq.ObjectPath)
+	assert.Equal(t, "", c.lastReq.VersionId)
+	assert.Equal(t, a.Key.RuleHash[:], c.lastReq.RuleHash)
+	assert.Equal(t, s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, c.lastReq.ActionKind)
+	// Bootstrap-style call: server skips CAS witness when nil.
+	assert.Nil(t, c.lastReq.ExpectedIdentity)
+}
+
+func TestWalkerDispatcher_VersionedPassesVersionID(t *testing.T) {
+	c := &walkerStubClient{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}
+	d := &WalkerDispatcher{Client: c}
+	a := sampleAction(t, s3lifecycle.ActionKindNoncurrentDays)
+	err := d.Delete(context.Background(), a, &bootstrap.Entry{Path: "obj", VersionID: "v-abc"})
+	require.NoError(t, err)
+	assert.Equal(t, "v-abc", c.lastReq.VersionId)
+}
+
+func TestWalkerDispatcher_MPUInitUsesUploadsPath(t *testing.T) {
+	// Rule-prefix matching uses DestKey to decide IF this MPU init
+	// matches; dispatch uses Path (.uploads/<id>) because the server's
+	// ABORT_MPU handler strips the .uploads/ prefix to get the upload
+	// id. Sending DestKey here would BLOCK with FATAL_EVENT_ERROR.
+	c := &walkerStubClient{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}
+	d := &WalkerDispatcher{Client: c}
+	a := sampleAction(t, s3lifecycle.ActionKindAbortMPU)
+	err := d.Delete(context.Background(), a, &bootstrap.Entry{
+		Path:      ".uploads/abc123",
+		DestKey:   "user/path/object",
+		IsMPUInit: true,
+	})
+	require.NoError(t, err)
+	assert.Equal(t, ".uploads/abc123", c.lastReq.ObjectPath)
+}
+
+func TestWalkerDispatcher_MPUInitEmptyDestKeyErrors(t *testing.T) {
+	// An MPU init record with no DestKey is mid-write before metadata
+	// landed; skipping silently in the walker is fine, but the
+	// dispatcher must not invent a path.
+	c := &walkerStubClient{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}
+	d := &WalkerDispatcher{Client: c}
+	a := sampleAction(t, s3lifecycle.ActionKindAbortMPU)
+	err := d.Delete(context.Background(), a, &bootstrap.Entry{
+		Path:      ".uploads/abc123",
+		IsMPUInit: true,
+	})
+	require.Error(t, err)
+	assert.Nil(t, c.lastReq, "no RPC should be sent when DestKey is empty")
+}
+
+func TestWalkerDispatcher_AcceptsAllResolvedOutcomes(t *testing.T) {
+	for _, oc := range []s3_lifecycle_pb.LifecycleDeleteOutcome{
+		s3_lifecycle_pb.LifecycleDeleteOutcome_DONE,
+		s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED,
+		s3_lifecycle_pb.LifecycleDeleteOutcome_SKIPPED_OBJECT_LOCK,
+	} {
+		c := &walkerStubClient{outcome: oc}
+		d := &WalkerDispatcher{Client: c}
+		err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
+		assert.NoError(t, err, "outcome %s must be treated as resolved", oc)
+	}
+}
+
+func TestWalkerDispatcher_UnresolvedOutcomeReturnsError(t *testing.T) {
+	// RETRY_LATER, BLOCKED, and UNSPECIFIED all halt the walk so it
+	// resumes from Checkpoint.LastScannedPath on the next run.
+	for _, oc := range []s3_lifecycle_pb.LifecycleDeleteOutcome{
+		s3_lifecycle_pb.LifecycleDeleteOutcome_RETRY_LATER,
+		s3_lifecycle_pb.LifecycleDeleteOutcome_BLOCKED,
+		s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED,
+	} {
+		c := &walkerStubClient{outcome: oc, reason: "server said so"}
+		d := &WalkerDispatcher{Client: c}
+		err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
+		require.Error(t, err, "outcome %s must halt the walk", oc)
+		assert.Contains(t, err.Error(), oc.String())
+	}
+}
+
+func TestWalkerDispatcher_TransportErrorReturnsWrappedError(t *testing.T) {
+	c := &walkerStubClient{err: errors.New("transport boom")}
+	d := &WalkerDispatcher{Client: c}
+	err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "transport boom")
+}
+
+func TestWalkerDispatcher_NilResponseReturnsError(t *testing.T) {
+	// A server returning (nil, nil) would otherwise panic on the
+	// outcome switch.
+	c := &walkerStubClient{nilResp: true}
+	d := &WalkerDispatcher{Client: c}
+	err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "nil response")
+}
+
+func TestWalkerDispatcher_LimiterWaitsBeforeDispatch(t *testing.T) {
+	// Build a tiny limiter (1 token, slow refill) and pre-drain it so
+	// the next Wait blocks until the deadline. Dispatcher must respect
+	// the limiter — without the wait the test passes trivially.
+	c := &walkerStubClient{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}
+	lim := rate.NewLimiter(rate.Every(50*time.Millisecond), 1)
+	_ = lim.AllowN(time.Now(), 1) // burn the burst token
+	d := &WalkerDispatcher{Client: c, Limiter: lim}
+
+	start := time.Now()
+	err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
+	elapsed := time.Since(start)
+	require.NoError(t, err)
+	assert.GreaterOrEqual(t, elapsed, 30*time.Millisecond,
+		"limiter must throttle the dispatch; elapsed=%v", elapsed)
+}
+
+func TestWalkerDispatcher_LimiterContextCancelHaltsWalker(t *testing.T) {
+	// Pre-drained limiter + canceled ctx. Limiter.Wait returns the
+	// cancel error; walker must surface it (not silently dispatch).
+	c := &walkerStubClient{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}
+	lim := rate.NewLimiter(rate.Every(time.Hour), 1)
+	_ = lim.AllowN(time.Now(), 1)
+	d := &WalkerDispatcher{Client: c, Limiter: lim}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	err := d.Delete(ctx, sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
+	require.Error(t, err)
+	assert.Nil(t, c.lastReq, "no RPC should be sent when ctx was cancelled in Wait")
+}
+
+func TestWalkerDispatcher_NilGuardsReturnError(t *testing.T) {
+	d := &WalkerDispatcher{Client: &walkerStubClient{}}
+	require.Error(t, d.Delete(context.Background(), nil, &bootstrap.Entry{Path: "obj"}))
+	require.Error(t, d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), nil))
+
+	nilClient := &WalkerDispatcher{}
+	require.Error(t, nilClient.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"}))
+}

weed/s3api/s3lifecycle/dailyrun/walker_interval_test.go

@@ -0,0 +1,276 @@
+package dailyrun
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// readerEventAlias keeps the test free of the reader import collision
+// in the recovery test file when we need a typed nil channel.
+type readerEventAlias = reader.Event
+
+// TestWalkerDue covers the throttle decision matrix in isolation.
+func TestWalkerDue(t *testing.T) {
+	runNow := time.Unix(1_700_000_000, 0).UTC()
+	cases := []struct {
+		name         string
+		lastWalkedNs int64
+		interval     time.Duration
+		want         bool
+	}{
+		// interval=0 keeps the prior "fire every pass" semantics — the
+		// in-repo integration tests and the s3tests fast driver rely on
+		// this so the rule-change-then-walk behavior surfaces within a
+		// single test runtime. Within-pass double-fire suppression
+		// lives in runShard's walkedThisPass local; walkerDue answers
+		// the persisted-state question only.
+		{"interval zero always due", runNow.UnixNano(), 0, true},
+		// LastWalkedNs=0 marks "never walked steady-state" — the post-
+		// upgrade cold-start case for cursor files that predate the
+		// LastWalkedNs field. Fire so the throttle anchor gets seeded.
+		{"never walked is due", 0, time.Hour, true},
+		// Throttle on: interval elapsed → due.
+		{"interval elapsed", runNow.Add(-2 * time.Hour).UnixNano(), time.Hour, true},
+		{"interval exactly elapsed", runNow.Add(-time.Hour).UnixNano(), time.Hour, true},
+		// Throttle on: not yet → skip.
+		{"interval not yet elapsed", runNow.Add(-30 * time.Minute).UnixNano(), time.Hour, false},
+		// Future LastWalkedNs (clock skew or replay of an older runNow
+		// against a newer cursor) must not panic and must not fire —
+		// treat as "throttled" because the cursor claims a future walk.
+		{"future lastWalked", runNow.Add(time.Hour).UnixNano(), time.Hour, false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.want, walkerDue(tc.lastWalkedNs, runNow, tc.interval))
+		})
+	}
+}
+
+// TestRunShard_WalkerThrottle confirms two back-to-back runShard
+// invocations only fire the steady-state walker once when the second
+// pass starts inside the WalkerInterval window. The interval-zero
+// control case keeps firing on every pass — same shape, different
+// expectation — pinning the prior behavior so a default-zero config
+// (tests, in-repo integration) doesn't regress.
+func TestRunShard_WalkerThrottle(t *testing.T) {
+	cases := []struct {
+		name            string
+		interval        time.Duration
+		secondPassAfter time.Duration
+		wantTotalCalls  int
+		wantSecondAdvNs bool // did LastWalkedNs change between pass 1 and pass 2?
+	}{
+		{"interval=0 fires every pass", 0, 30 * time.Second, 2, true},
+		{"throttled: second pass within interval", time.Hour, 30 * time.Second, 1, false},
+		{"throttled: second pass past interval", time.Hour, 2 * time.Hour, 2, true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			snap := snapshotWithRule(t, 30) // replay-eligible only; steady-state walker would normally skip
+			// Force a walker-only partition so the steady-state branch
+			// has something to walk: set retentionWindow=0 in cfg, which
+			// makes promoted non-empty and walkView non-empty.
+			p := newMemPersister()
+			calls := 0
+			cfg := Config{
+				Persister: p,
+				Walker: func(_ context.Context, _ *engine.Snapshot, _ int) error {
+					calls++
+					return nil
+				},
+				RetentionWindow: -1, // negative sentinel: falls back to maxTTL — leaves walkView empty
+				WalkerInterval:  tc.interval,
+			}
+			// Pre-seed cursor matching snap's hashes so cold-start /
+			// recovery branches don't fire and we measure ONLY the
+			// steady-state walker.
+			rsh := engine.ReplayContentHash(snap)
+			promoted := engine.PromotedHash(snap, engine.MaxEffectiveTTL(snap))
+			runNow := time.Unix(1_700_000_000, 0).UTC()
+			require.NoError(t, p.Save(context.Background(), 0, Cursor{
+				TsNs:         runNow.UnixNano(),
+				RuleSetHash:  rsh,
+				PromotedHash: promoted,
+			}))
+
+			// With the snapshot's only rule being replay-eligible, the
+			// steady-state walkView is empty and the walker won't fire
+			// regardless of throttle. Use a snapshot that has BOTH a
+			// replay rule AND a scan-only walker rule by forcing
+			// retentionWindow to a very small value via cfg.
+			cfg.RetentionWindow = time.Nanosecond
+			// Recompute promoted with the test retention so persisted
+			// hashes match what runShard sees on each pass.
+			snapPromoted := engine.PromotedHash(snap, time.Nanosecond)
+			require.NoError(t, p.Save(context.Background(), 0, Cursor{
+				TsNs:         runNow.UnixNano(),
+				RuleSetHash:  rsh,
+				PromotedHash: snapPromoted,
+			}))
+
+			// runShard reaches drainShardEvents after the steady-state
+			// walker fires (replay-eligible rules present). A nil events
+			// channel would block forever; a closed one returns
+			// immediately so drain exits cleanly and the cursor save
+			// still records LastWalkedNs.
+			closedEvents := make(chan *readerEventAlias)
+			close(closedEvents)
+
+			// Pass 1.
+			require.NoError(t, runShard(context.Background(), cfg, snap, runNow, 0, closedEvents))
+			afterPass1, _, _ := p.Load(context.Background(), 0)
+			require.Equal(t, 1, calls, "pass 1 must fire the walker (cold-start anchor)")
+			require.Equal(t, runNow.UnixNano(), afterPass1.LastWalkedNs)
+
+			// Pass 2.
+			runNow2 := runNow.Add(tc.secondPassAfter)
+			closedEvents2 := make(chan *readerEventAlias)
+			close(closedEvents2)
+			require.NoError(t, runShard(context.Background(), cfg, snap, runNow2, 0, closedEvents2))
+			afterPass2, _, _ := p.Load(context.Background(), 0)
+			assert.Equal(t, tc.wantTotalCalls, calls, "walker calls after 2 passes")
+			if tc.wantSecondAdvNs {
+				assert.Equal(t, runNow2.UnixNano(), afterPass2.LastWalkedNs, "throttle allowed the second walk")
+			} else {
+				assert.Equal(t, runNow.UnixNano(), afterPass2.LastWalkedNs, "throttle suppressed the second walk; anchor unchanged")
+			}
+		})
+	}
+}
+
+// TestValidate_RejectsNegativeWalkerInterval pins the loud-failure
+// contract for an embedder or test that constructs dailyrun.Config
+// directly with a negative WalkerInterval. The admin-config parser
+// clamps negative input to zero (worker/tasks/s3_lifecycle/config.go),
+// but a caller bypassing that parser would otherwise get the silent
+// fall-through-to-walk-every-pass behavior the throttle is trying to
+// prevent.
+func TestValidate_RejectsNegativeWalkerInterval(t *testing.T) {
+	// validate only checks for nil fields, not behavior — type-assert
+	// each interface to its zero value via a stub. The stubs never run.
+	cfg := validatableConfig()
+	cfg.WalkerInterval = -time.Hour
+	err := validate(cfg)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "WalkerInterval")
+	// And the zero sentinel still passes.
+	cfg.WalkerInterval = 0
+	require.NoError(t, validate(cfg))
+}
+
+// validatableConfig builds a minimal dailyrun.Config that passes the
+// nil checks in validate() so individual fields can be poked. The
+// stubs intentionally don't implement any meaningful behavior — they
+// only need to be non-nil interface values.
+func validatableConfig() Config {
+	return Config{
+		Engine:      engine.New(),
+		FilerClient: stubFilerClient{},
+		Client:      stubLifecycleClient{},
+		Persister:   newMemPersister(),
+		Lister:      stubSiblingLister{},
+		BucketsPath: "/buckets",
+	}
+}
+
+type stubFilerClient struct{ filer_pb.SeaweedFilerClient }
+type stubLifecycleClient struct{}
+
+func (stubLifecycleClient) LifecycleDelete(_ context.Context, _ *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
+	return nil, nil
+}
+
+type stubSiblingLister struct{}
+
+func (stubSiblingLister) Survivors(_ context.Context, _, _ string) (router.Survivors, error) {
+	return router.Survivors{}, nil
+}
+func (stubSiblingLister) LookupVersion(_ context.Context, _, _, _ string) (*filer_pb.Entry, error) {
+	return nil, nil
+}
+func (stubSiblingLister) ListVersions(_ context.Context, _, _ string) ([]*filer_pb.Entry, error) {
+	return nil, nil
+}
+func (stubSiblingLister) LookupNullVersion(_ context.Context, _, _ string) (*filer_pb.Entry, bool, error) {
+	return nil, false, nil
+}
+
+// TestRunShard_ColdStartDoesNotDoubleWalk pins the within-pass guard
+// in runShard's walkedThisPass local. Before the guard, a cold-start
+// pass with WalkerInterval=0 fired the walker twice in one pass:
+// once via the mustWalkColdStart branch (with RecoveryView) and again
+// immediately via the steady-state branch (with the per-shard walk
+// view, which is a subset of RecoveryView). The second walk added no
+// coverage and burned a full bucket scan. Guard against regression.
+func TestRunShard_ColdStartDoesNotDoubleWalk(t *testing.T) {
+	snap := snapshotWithRule(t, 30)
+	p := newMemPersister()
+	// No persisted cursor → mustWalkColdStart=true. WalkerInterval=0
+	// would, before the fix, allow the steady-state branch to fire
+	// again with no time elapsed.
+	calls := 0
+	cfg := Config{
+		Persister: p,
+		Walker: func(_ context.Context, _ *engine.Snapshot, _ int) error {
+			calls++
+			return nil
+		},
+		WalkerInterval: 0,
+	}
+	runNow := time.Unix(1_700_000_000, 0).UTC()
+
+	closedEvents := make(chan *readerEventAlias)
+	close(closedEvents)
+	require.NoError(t, runShard(context.Background(), cfg, snap, runNow, 0, closedEvents))
+
+	assert.Equal(t, 1, calls, "cold-start walker must fire exactly once per pass even with interval=0")
+}
+
+// TestRunShard_RecoveryWalkerSetsLastWalkedAnchor pins that the
+// unconditional recovery-branch walker fire still updates the
+// LastWalkedNs anchor so the NEXT pass's throttle starts counting
+// from this walk rather than re-firing immediately.
+func TestRunShard_RecoveryWalkerSetsLastWalkedAnchor(t *testing.T) {
+	snap := snapshotWithRule(t, 30)
+	p := newMemPersister()
+	var stale [32]byte
+	for i := range stale {
+		stale[i] = 0xAA
+	}
+	require.NoError(t, p.Save(context.Background(), 5, Cursor{
+		TsNs:         1234,
+		RuleSetHash:  stale,
+		LastWalkedNs: 0, // never walked anchor
+	}))
+
+	calls := 0
+	cfg := Config{
+		Persister: p,
+		Walker: func(_ context.Context, _ *engine.Snapshot, _ int) error {
+			calls++
+			return nil
+		},
+		WalkerInterval: time.Hour,
+	}
+	runNow := time.Unix(1_700_000_000, 0).UTC()
+	require.NoError(t, runShard(context.Background(), cfg, snap, runNow, 5, nil))
+
+	require.Equal(t, 1, calls, "recovery branch fires the walker unconditionally")
+	got, ok, _ := p.Load(context.Background(), 5)
+	require.True(t, ok)
+	assert.Equal(t, runNow.UnixNano(), got.LastWalkedNs,
+		"recovery walk must update LastWalkedNs so the steady-state pass after it can throttle")
+}
+
+// Compile-time check that the documented sentinel passes the type system.
+var _ = s3lifecycle.ShardCount

weed/s3api/s3lifecycle/dailyrun/walker_recovery_test.go

@@ -0,0 +1,150 @@
+package dailyrun
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// memPersister is a minimal in-memory CursorPersister. Phase 4b tests
+// drive runShard directly; the recovery-branch path returns before
+// drainShardEvents so the heavier filer/client/lister fakes aren't
+// needed here. Locked because Run fans out a goroutine per shard and
+// they all load and save through the same persister.
+type memPersister struct {
+	mu    sync.Mutex
+	store map[int]Cursor
+}
+
+func newMemPersister() *memPersister { return &memPersister{store: map[int]Cursor{}} }
+
+func (p *memPersister) Load(_ context.Context, shardID int) (Cursor, bool, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	c, ok := p.store[shardID]
+	return c, ok, nil
+}
+
+func (p *memPersister) Save(_ context.Context, shardID int, c Cursor) error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.store[shardID] = c
+	return nil
+}
+
+func snapshotWithRule(t *testing.T, days int) *engine.Snapshot {
+	t.Helper()
+	e := engine.New()
+	e.Compile([]engine.CompileInput{
+		{Bucket: "b1", Rules: []*s3lifecycle.Rule{
+			{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: days},
+		}},
+	}, engine.CompileOptions{})
+	snap := e.Snapshot()
+	for _, a := range snap.AllActions() {
+		snap.MarkActive(a.Key)
+	}
+	return snap
+}
+
+func TestRunShard_WalkerInvokedOnRecoveryBranch(t *testing.T) {
+	snap := snapshotWithRule(t, 30)
+	p := newMemPersister()
+	// Seed a persisted cursor whose RuleSetHash differs from snap's, so
+	// the rule-change branch fires.
+	var stale [32]byte
+	for i := range stale {
+		stale[i] = 0xAA
+	}
+	require.NoError(t, p.Save(context.Background(), 3, Cursor{TsNs: 1234, RuleSetHash: stale}))
+
+	var gotView *engine.Snapshot
+	var gotShard int
+	calls := 0
+	cfg := Config{
+		Persister: p,
+		Walker: func(_ context.Context, view *engine.Snapshot, shardID int) error {
+			calls++
+			gotView = view
+			gotShard = shardID
+			return nil
+		},
+	}
+	runNow := time.Unix(1_700_000_000, 0).UTC()
+	require.NoError(t, runShard(context.Background(), cfg, snap, runNow, 3, nil))
+
+	assert.Equal(t, 1, calls, "walker must fire exactly once on recovery")
+	require.NotNil(t, gotView, "walker received the RecoveryView")
+	assert.Equal(t, 3, gotShard)
+
+	// Cursor rewound to runNow - maxTTL with the new hashes persisted.
+	got, ok, err := p.Load(context.Background(), 3)
+	require.NoError(t, err)
+	require.True(t, ok)
+	expectedFloor := runNow.Add(-engine.MaxEffectiveTTL(snap)).UnixNano()
+	assert.Equal(t, expectedFloor, got.TsNs, "cursor must rewind to runNow - maxTTL after recovery walk")
+	assert.NotEqual(t, stale, got.RuleSetHash, "stale hash must be replaced")
+}
+
+func TestRunShard_NilWalkerOnRecoveryIsNoop(t *testing.T) {
+	// Phase 4a behavior: a nil Walker must not crash and must still
+	// rewind the cursor.
+	snap := snapshotWithRule(t, 30)
+	p := newMemPersister()
+	var stale [32]byte
+	for i := range stale {
+		stale[i] = 0xBB
+	}
+	require.NoError(t, p.Save(context.Background(), 0, Cursor{TsNs: 9999, RuleSetHash: stale}))
+
+	cfg := Config{Persister: p} // Walker nil
+	runNow := time.Unix(1_700_000_000, 0).UTC()
+	require.NoError(t, runShard(context.Background(), cfg, snap, runNow, 0, nil))
+
+	got, ok, err := p.Load(context.Background(), 0)
+	require.NoError(t, err)
+	require.True(t, ok)
+	assert.Equal(t, runNow.Add(-engine.MaxEffectiveTTL(snap)).UnixNano(), got.TsNs)
+}
+
+func TestRunShard_WalkerErrorPropagates(t *testing.T) {
+	// A walker failure must surface as the runShard error so the daily
+	// run treats the shard as interrupted, and must NOT advance the
+	// cursor (the seeded cursor stays).
+	snap := snapshotWithRule(t, 30)
+	p := newMemPersister()
+	var stale [32]byte
+	for i := range stale {
+		stale[i] = 0xCC
+	}
+	require.NoError(t, p.Save(context.Background(), 7, Cursor{TsNs: 42, RuleSetHash: stale}))
+
+	cfg := Config{
+		Persister: p,
+		Walker: func(_ context.Context, _ *engine.Snapshot, _ int) error {
+			return errors.New("walker boom")
+		},
+	}
+	runNow := time.Unix(1_700_000_000, 0).UTC()
+	err := runShard(context.Background(), cfg, snap, runNow, 7, nil)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "walker boom")
+
+	// Cursor untouched.
+	got, _, _ := p.Load(context.Background(), 7)
+	assert.Equal(t, int64(42), got.TsNs, "walker failure must leave cursor unchanged")
+	assert.Equal(t, stale, got.RuleSetHash)
+}
+
+// The "matching cursor doesn't invoke walker" case is implicit: the
+// walker call is inside the rule-change / partition-flip `if` and
+// can't be reached otherwise. Exercising it end-to-end requires the
+// full filer + lister + client harness; covered by the integration
+// tests once Phase 4b is wired into the handler.