snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/s3api/s3lifecycle/reader/composition_test.go

@@ -0,0 +1,162 @@
+package reader
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Layer 2 contracts the daily-replay drain relies on. These augment
+// reader_test.go (extract + dispatch) and cursor_test.go (Advance +
+// Freeze) by pinning the cursor composition surface that the daily-run
+// drain depends on for resume-point selection, checkpoint persistence,
+// and Run-time input validation.
+
+func TestCursorMinTsNsWithFrozenKeysIncluded(t *testing.T) {
+	// MinTsNs is the resume point Reader.Run feeds to SubscribeMetadata
+	// when StartTsNs is zero. A freeze pins a key at its current TsNs;
+	// the min must still see that pinned position so the subscription
+	// doesn't rewind past it. Mixing frozen + non-frozen keys exercises
+	// the path where the smallest position belongs to a frozen key.
+	c := NewCursor()
+	frozenKey := key("a", s3lifecycle.ActionKindExpirationDays)
+	movingKey := key("b", s3lifecycle.ActionKindNoncurrentDays)
+	otherKey := key("c", s3lifecycle.ActionKindAbortMPU)
+
+	c.Advance(frozenKey, 100)
+	c.Advance(movingKey, 500)
+	c.Advance(otherKey, 300)
+	c.Freeze(frozenKey, 100)
+
+	// Frozen key holds at 100; Advance must not move it.
+	c.Advance(frozenKey, 9_999)
+	require.Equal(t, int64(100), c.Get(frozenKey))
+
+	// MinTsNs reflects the frozen key because it is still the smallest.
+	assert.Equal(t, int64(100), c.MinTsNs())
+
+	// Non-frozen keys keep advancing; min stays anchored to the freeze.
+	c.Advance(movingKey, 700)
+	c.Advance(otherKey, 600)
+	assert.Equal(t, int64(100), c.MinTsNs())
+}
+
+func TestCursorSnapshotIsDeepCopy(t *testing.T) {
+	// Snapshot is persisted via the persister; callers must be able to
+	// hold the returned map without observing live cursor mutations,
+	// and the cursor must not observe caller-side edits to the map.
+	c := NewCursor()
+	a := key("a", s3lifecycle.ActionKindExpirationDays)
+	b := key("b", s3lifecycle.ActionKindNoncurrentDays)
+	c.Advance(a, 100)
+	c.Advance(b, 200)
+
+	snap := c.Snapshot()
+	require.Equal(t, int64(100), snap[a])
+	require.Equal(t, int64(200), snap[b])
+
+	// Mutating the returned map must not bleed into the cursor.
+	snap[a] = 9_999
+	delete(snap, b)
+	snap[key("z", s3lifecycle.ActionKindAbortMPU)] = 42
+	assert.Equal(t, int64(100), c.Get(a), "cursor must be insulated from snapshot writes")
+	assert.Equal(t, int64(200), c.Get(b), "cursor must retain key the caller deleted from snapshot")
+	assert.Equal(t, int64(0), c.Get(key("z", s3lifecycle.ActionKindAbortMPU)), "cursor must not see keys added to snapshot")
+
+	// Mutating the cursor after Snapshot must not bleed into the map
+	// the caller is still holding (e.g., between Snapshot and Save).
+	snap2 := c.Snapshot()
+	c.Advance(a, 1_000)
+	c.Advance(key("d", s3lifecycle.ActionKindAbortMPU), 50)
+	assert.Equal(t, int64(100), snap2[a], "snapshot must not see post-snapshot Advance")
+	_, hasNew := snap2[key("d", s3lifecycle.ActionKindAbortMPU)]
+	assert.False(t, hasNew, "snapshot must not see post-snapshot key insertions")
+}
+
+func TestCursorRestoreReplacesNotMerges(t *testing.T) {
+	// On startup the persister loads the last checkpoint and Restores
+	// the cursor. A subsequent Restore (e.g., after a re-bootstrap)
+	// must fully replace the in-memory map: keys present before but
+	// absent in the new map must vanish, otherwise stale resume
+	// points would survive across restores.
+	c := NewCursor()
+	a := key("a", s3lifecycle.ActionKindExpirationDays)
+	b := key("b", s3lifecycle.ActionKindNoncurrentDays)
+	cc := key("c", s3lifecycle.ActionKindAbortMPU)
+
+	c.Restore(map[s3lifecycle.ActionKey]int64{a: 100, b: 200, cc: 300})
+	require.Equal(t, int64(100), c.Get(a))
+	require.Equal(t, int64(200), c.Get(b))
+	require.Equal(t, int64(300), c.Get(cc))
+
+	// Freeze a key before second Restore; Restore must clear frozen state
+	// alongside the value map so a stale freeze doesn't survive a reload.
+	c.Freeze(a, 100)
+	require.True(t, c.IsFrozen(a))
+
+	// Second Restore drops keys b and c entirely; a is replaced.
+	c.Restore(map[s3lifecycle.ActionKey]int64{a: 50})
+	assert.Equal(t, int64(50), c.Get(a))
+	assert.False(t, c.IsFrozen(a), "Restore must clear frozen state")
+	assert.Equal(t, int64(0), c.Get(b), "key absent from second Restore must be removed")
+	assert.Equal(t, int64(0), c.Get(cc), "key absent from second Restore must be removed")
+
+	// MinTsNs reflects only the surviving key.
+	assert.Equal(t, int64(50), c.MinTsNs())
+
+	// Restoring an empty map clears all state.
+	c.Restore(map[s3lifecycle.ActionKey]int64{})
+	assert.Equal(t, int64(0), c.Get(a))
+	assert.Equal(t, int64(0), c.MinTsNs())
+}
+
+func TestReaderRunValidatesInputsBeforeSubscribing(t *testing.T) {
+	// Run guards: out-of-range ShardID (no predicate), nil Events
+	// channel, empty BucketsPath. Each error path must fire before any
+	// client call, so a nil client suffices to drive the assertions.
+	out := make(chan *Event, 1)
+
+	cases := []struct {
+		name    string
+		reader  *Reader
+		wantSub string
+	}{
+		{
+			name:    "shard id below zero",
+			reader:  &Reader{ShardID: -1, BucketsPath: "/buckets", Events: out},
+			wantSub: "shard_id",
+		},
+		{
+			name:    "shard id at or above ShardCount",
+			reader:  &Reader{ShardID: s3lifecycle.ShardCount, BucketsPath: "/buckets", Events: out},
+			wantSub: "shard_id",
+		},
+		{
+			name:    "nil events channel",
+			reader:  &Reader{ShardID: 0, BucketsPath: "/buckets", Events: nil},
+			wantSub: "nil Events",
+		},
+		{
+			name:    "empty buckets path",
+			reader:  &Reader{ShardID: 0, BucketsPath: "", Events: out},
+			wantSub: "empty BucketsPath",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Bound the call so a regression that lets Run reach the
+			// nil client's blocking subscribe surfaces as a test failure
+			// instead of hanging the suite.
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			err := tc.reader.Run(ctx, nil, "test", 0)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.wantSub)
+		})
+	}
+}

weed/s3api/s3lifecycle/reader/cursor.go

@@ -0,0 +1,123 @@
+package reader
+
+import (
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+)
+
+// Cursor tracks the meta-log position for each ActionKey within one shard.
+// Position = tsNs of the last event the shard's reader+downstream considers
+// "fully resolved" for this ActionKey. Subscription resumes at the minimum
+// position across all keys.
+//
+// A frozen key is pinned at its current position (BLOCKED outcome): Advance
+// is a no-op until Unfreeze. The meta-log retention bounds how long a freeze
+// can last before the entry expires from the log; operator action via the
+// blocker-resolve flow lifts the freeze.
+type Cursor struct {
+	mu     sync.RWMutex
+	state  map[s3lifecycle.ActionKey]int64
+	frozen map[s3lifecycle.ActionKey]struct{}
+}
+
+func NewCursor() *Cursor {
+	return &Cursor{
+		state:  map[s3lifecycle.ActionKey]int64{},
+		frozen: map[s3lifecycle.ActionKey]struct{}{},
+	}
+}
+
+// MinTsNs returns the smallest position across all keys; zero if empty.
+// Used as the resume point for the meta-log subscription.
+func (c *Cursor) MinTsNs() int64 {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if len(c.state) == 0 {
+		return 0
+	}
+	var min int64 = -1
+	for _, v := range c.state {
+		if min < 0 || v < min {
+			min = v
+		}
+	}
+	if min < 0 {
+		return 0
+	}
+	return min
+}
+
+// Get returns the current position for key, or 0 if unset.
+func (c *Cursor) Get(key s3lifecycle.ActionKey) int64 {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	return c.state[key]
+}
+
+// Advance moves key's position forward to tsNs. No-op if key is frozen, if
+// tsNs <= current, or if tsNs <= 0.
+func (c *Cursor) Advance(key s3lifecycle.ActionKey, tsNs int64) {
+	if tsNs <= 0 {
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if _, isFrozen := c.frozen[key]; isFrozen {
+		return
+	}
+	if cur := c.state[key]; tsNs > cur {
+		c.state[key] = tsNs
+	}
+}
+
+// Freeze pins key at its current position. If key has no recorded position
+// yet, it is set to tsNs.
+func (c *Cursor) Freeze(key s3lifecycle.ActionKey, tsNs int64) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if _, exists := c.state[key]; !exists {
+		c.state[key] = tsNs
+	}
+	c.frozen[key] = struct{}{}
+}
+
+// Unfreeze releases a freeze. Subsequent Advance calls take effect.
+func (c *Cursor) Unfreeze(key s3lifecycle.ActionKey) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	delete(c.frozen, key)
+}
+
+// IsFrozen reports whether key is currently pinned.
+func (c *Cursor) IsFrozen(key s3lifecycle.ActionKey) bool {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	_, ok := c.frozen[key]
+	return ok
+}
+
+// Snapshot copies the cursor map for persistence. Frozen keys are not
+// distinguished in the snapshot; callers persist freezes via the separate
+// blocker-record store.
+func (c *Cursor) Snapshot() map[s3lifecycle.ActionKey]int64 {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	out := make(map[s3lifecycle.ActionKey]int64, len(c.state))
+	for k, v := range c.state {
+		out[k] = v
+	}
+	return out
+}
+
+// Restore replaces the cursor map. Freezes are not restored here; the
+// caller re-applies them from blocker records on startup.
+func (c *Cursor) Restore(state map[s3lifecycle.ActionKey]int64) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.state = make(map[s3lifecycle.ActionKey]int64, len(state))
+	for k, v := range state {
+		c.state[k] = v
+	}
+	c.frozen = map[s3lifecycle.ActionKey]struct{}{}
+}

weed/s3api/s3lifecycle/reader/cursor_test.go

@@ -0,0 +1,109 @@
+package reader
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+)
+
+func key(b string, kind s3lifecycle.ActionKind) s3lifecycle.ActionKey {
+	return s3lifecycle.ActionKey{Bucket: b, ActionKind: kind}
+}
+
+func TestCursorMinTsNsEmpty(t *testing.T) {
+	c := NewCursor()
+	if got := c.MinTsNs(); got != 0 {
+		t.Fatalf("empty MinTsNs=%d, want 0", got)
+	}
+}
+
+func TestCursorAdvanceMonotonic(t *testing.T) {
+	c := NewCursor()
+	k := key("b", s3lifecycle.ActionKindExpirationDays)
+	c.Advance(k, 100)
+	c.Advance(k, 50)  // backward, ignored
+	c.Advance(k, 100) // equal, ignored
+	c.Advance(k, 200)
+	if got := c.Get(k); got != 200 {
+		t.Fatalf("Get=%d, want 200", got)
+	}
+}
+
+func TestCursorAdvanceIgnoresZero(t *testing.T) {
+	c := NewCursor()
+	k := key("b", s3lifecycle.ActionKindExpirationDays)
+	c.Advance(k, 0)
+	c.Advance(k, -1)
+	if got := c.Get(k); got != 0 {
+		t.Fatalf("Get=%d, want 0", got)
+	}
+}
+
+func TestCursorMinTsNsAcrossKeys(t *testing.T) {
+	c := NewCursor()
+	c.Advance(key("a", s3lifecycle.ActionKindExpirationDays), 100)
+	c.Advance(key("b", s3lifecycle.ActionKindExpirationDays), 50)
+	c.Advance(key("c", s3lifecycle.ActionKindAbortMPU), 200)
+	if got := c.MinTsNs(); got != 50 {
+		t.Fatalf("MinTsNs=%d, want 50", got)
+	}
+}
+
+func TestCursorFreeze(t *testing.T) {
+	c := NewCursor()
+	k := key("b", s3lifecycle.ActionKindExpirationDays)
+	c.Advance(k, 100)
+	c.Freeze(k, 100)
+	c.Advance(k, 200) // frozen — ignored
+	if got := c.Get(k); got != 100 {
+		t.Fatalf("Get after freeze=%d, want 100", got)
+	}
+	if !c.IsFrozen(k) {
+		t.Fatal("IsFrozen=false, want true")
+	}
+	c.Unfreeze(k)
+	c.Advance(k, 200)
+	if got := c.Get(k); got != 200 {
+		t.Fatalf("Get after unfreeze=%d, want 200", got)
+	}
+}
+
+func TestCursorFreezeOnUnsetKeySeedsPosition(t *testing.T) {
+	// Freezing a key with no recorded position seeds it at the freeze tsNs;
+	// subsequent MinTsNs reflects the freeze, so the subscription doesn't
+	// rewind past it.
+	c := NewCursor()
+	k := key("b", s3lifecycle.ActionKindExpirationDays)
+	c.Freeze(k, 500)
+	if got := c.Get(k); got != 500 {
+		t.Fatalf("Get after freeze-on-unset=%d, want 500", got)
+	}
+	if got := c.MinTsNs(); got != 500 {
+		t.Fatalf("MinTsNs=%d, want 500", got)
+	}
+}
+
+func TestCursorSnapshotRestore(t *testing.T) {
+	c := NewCursor()
+	a := key("a", s3lifecycle.ActionKindExpirationDays)
+	b := key("b", s3lifecycle.ActionKindAbortMPU)
+	c.Advance(a, 100)
+	c.Advance(b, 200)
+
+	snap := c.Snapshot()
+	if len(snap) != 2 || snap[a] != 100 || snap[b] != 200 {
+		t.Fatalf("Snapshot=%v", snap)
+	}
+
+	c2 := NewCursor()
+	c2.Restore(snap)
+	if got := c2.Get(a); got != 100 {
+		t.Fatalf("restored Get(a)=%d, want 100", got)
+	}
+	if got := c2.Get(b); got != 200 {
+		t.Fatalf("restored Get(b)=%d, want 200", got)
+	}
+	if c2.IsFrozen(a) {
+		t.Fatal("Restore should not carry freezes")
+	}
+}

weed/s3api/s3lifecycle/reader/event_predicates_test.go

@@ -0,0 +1,47 @@
+package reader
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/assert"
+)
+
+// IsCreate / IsDelete are the small but routing-critical predicates
+// the dispatcher uses to decide which match path applies. Both were
+// previously exercised only through end-to-end Tick/Match tests; pin
+// them directly here.
+
+func TestEventIsCreate_PopulatedNewEntryNoOldEntry(t *testing.T) {
+	e := &Event{NewEntry: &filer_pb.Entry{Name: "k"}}
+	assert.True(t, e.IsCreate())
+	assert.False(t, e.IsDelete())
+}
+
+func TestEventIsDelete_PopulatedOldEntryNoNewEntry(t *testing.T) {
+	e := &Event{OldEntry: &filer_pb.Entry{Name: "k"}}
+	assert.True(t, e.IsDelete())
+	assert.False(t, e.IsCreate())
+}
+
+func TestEventBothEntries_NeitherCreateNorDelete(t *testing.T) {
+	// An update event carries both old and new; the predicates are
+	// strict (one or the other, not both) so the router can route
+	// updates through the dedicated path rather than treating them
+	// as creates or deletes.
+	e := &Event{
+		OldEntry: &filer_pb.Entry{Name: "k"},
+		NewEntry: &filer_pb.Entry{Name: "k"},
+	}
+	assert.False(t, e.IsCreate(), "update event must not classify as create")
+	assert.False(t, e.IsDelete(), "update event must not classify as delete")
+}
+
+func TestEventNoEntries_NeitherCreateNorDelete(t *testing.T) {
+	// A degenerate event with neither side populated must not classify
+	// as either; otherwise a metadata-only filer event with no bucket
+	// payload could trigger spurious dispatches.
+	e := &Event{}
+	assert.False(t, e.IsCreate())
+	assert.False(t, e.IsDelete())
+}

weed/s3api/s3lifecycle/reader/log_startup_test.go

@@ -0,0 +1,44 @@
+package reader
+
+import (
+	"testing"
+)
+
+// LogStartup writes a single glog line summarising the reader's
+// resume position; the only behavioral output is a side effect on the
+// log sink, but exercising both branches still pins compile-time
+// shape (e.g. that ShardPredicate-set readers don't trip on a missing
+// ShardID and vice versa) and lets coverage actually visit the code.
+
+func TestLogStartup_ShardIDOnly(t *testing.T) {
+	// Single-shard configuration: ShardID is set, ShardPredicate is nil,
+	// no Cursor, no StartTsNs. The function must run without panic.
+	r := &Reader{ShardID: 7, EventBudget: 100}
+	r.LogStartup()
+}
+
+func TestLogStartup_ShardPredicate(t *testing.T) {
+	// ShardPredicate-set readers take a different log branch; pinning
+	// the call here catches a regression that returns or panics.
+	r := &Reader{
+		ShardPredicate: func(int) bool { return true },
+		EventBudget:    100,
+	}
+	r.LogStartup()
+}
+
+func TestLogStartup_StartTsNsOverridesCursor(t *testing.T) {
+	// Explicit StartTsNs takes precedence over Cursor.MinTsNs; this
+	// branch is otherwise only hit when a worker is replaying a
+	// specific position. Run it through to make sure the override is
+	// honored without consulting the Cursor.
+	r := &Reader{ShardID: 0, StartTsNs: 1700000000_000_000_000, Cursor: NewCursor()}
+	r.LogStartup()
+}
+
+func TestLogStartup_CursorMinFallback(t *testing.T) {
+	// StartTsNs=0 with a non-nil Cursor falls back to Cursor.MinTsNs.
+	c := NewCursor()
+	r := &Reader{ShardID: 0, Cursor: c}
+	r.LogStartup()
+}

weed/s3api/s3lifecycle/reader/persister.go

@@ -0,0 +1,55 @@
+package reader
+
+import (
+	"context"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+)
+
+// Persister loads and stores per-shard cursor state. The contract:
+//   - Load returns an empty map (not an error) when no prior state exists.
+//   - Save replaces the prior state atomically; partial writes must not be
+//     observable on a later Load.
+//   - Concurrent Save calls for the same shardID are serialized by the
+//     caller (one Reader per shard at a time).
+//
+// The filer-backed implementation lives in the worker package alongside the
+// rest of the persistence wiring; this package only ships the in-memory
+// variant used by tests.
+type Persister interface {
+	Load(ctx context.Context, shardID int) (map[s3lifecycle.ActionKey]int64, error)
+	Save(ctx context.Context, shardID int, state map[s3lifecycle.ActionKey]int64) error
+}
+
+// InMemoryPersister is a Persister for tests. Save copies the input so
+// later mutations by the caller don't leak into stored state.
+type InMemoryPersister struct {
+	mu      sync.Mutex
+	storage map[int]map[s3lifecycle.ActionKey]int64
+}
+
+func NewInMemoryPersister() *InMemoryPersister {
+	return &InMemoryPersister{storage: map[int]map[s3lifecycle.ActionKey]int64{}}
+}
+
+func (p *InMemoryPersister) Load(ctx context.Context, shardID int) (map[s3lifecycle.ActionKey]int64, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	out := map[s3lifecycle.ActionKey]int64{}
+	for k, v := range p.storage[shardID] {
+		out[k] = v
+	}
+	return out, nil
+}
+
+func (p *InMemoryPersister) Save(ctx context.Context, shardID int, state map[s3lifecycle.ActionKey]int64) error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	copied := make(map[s3lifecycle.ActionKey]int64, len(state))
+	for k, v := range state {
+		copied[k] = v
+	}
+	p.storage[shardID] = copied
+	return nil
+}

weed/s3api/s3lifecycle/reader/persister_test.go

@@ -0,0 +1,161 @@
+package reader
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// InMemoryPersister is the in-memory test double other lifecycle tests
+// rely on for cursor checkpointing. The deep-copy contract documented
+// on the Persister interface is what insulates each test's state from
+// the others — these tests pin every direction of that contract so a
+// regression here can't silently corrupt downstream tests.
+
+func TestInMemoryPersister_LoadOnUnknownShardReturnsEmptyMap(t *testing.T) {
+	// Contract: Load returns an empty map (not an error) when no prior
+	// state exists. The dispatcher startup path relies on this so a
+	// fresh worker doesn't have to special-case the "no checkpoint yet"
+	// branch.
+	p := NewInMemoryPersister()
+	got, err := p.Load(context.Background(), 0)
+	require.NoError(t, err)
+	assert.NotNil(t, got)
+	assert.Empty(t, got)
+}
+
+func TestInMemoryPersister_SaveLoadRoundTrip(t *testing.T) {
+	p := NewInMemoryPersister()
+	want := map[s3lifecycle.ActionKey]int64{
+		key("a", s3lifecycle.ActionKindExpirationDays): 100,
+		key("b", s3lifecycle.ActionKindNoncurrentDays): 200,
+	}
+	require.NoError(t, p.Save(context.Background(), 7, want))
+
+	got, err := p.Load(context.Background(), 7)
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestInMemoryPersister_SaveCopiesInput(t *testing.T) {
+	// Caller mutating the map after Save must not bleed into stored
+	// state — the next Load must return what was saved.
+	p := NewInMemoryPersister()
+	in := map[s3lifecycle.ActionKey]int64{
+		key("a", s3lifecycle.ActionKindExpirationDays): 100,
+	}
+	require.NoError(t, p.Save(context.Background(), 0, in))
+
+	in[key("a", s3lifecycle.ActionKindExpirationDays)] = 9_999
+	in[key("z", s3lifecycle.ActionKindAbortMPU)] = 42
+	delete(in, key("a", s3lifecycle.ActionKindExpirationDays))
+
+	got, err := p.Load(context.Background(), 0)
+	require.NoError(t, err)
+	assert.Equal(t, int64(100), got[key("a", s3lifecycle.ActionKindExpirationDays)],
+		"caller-side mutation must not bleed into stored state")
+	assert.NotContains(t, got, key("z", s3lifecycle.ActionKindAbortMPU),
+		"caller-side insertion must not bleed into stored state")
+}
+
+func TestInMemoryPersister_LoadReturnsACopy(t *testing.T) {
+	// Caller mutating the returned map must not bleed back; otherwise
+	// a test reusing the persister across cases could see drift.
+	p := NewInMemoryPersister()
+	want := map[s3lifecycle.ActionKey]int64{
+		key("a", s3lifecycle.ActionKindExpirationDays): 100,
+	}
+	require.NoError(t, p.Save(context.Background(), 0, want))
+
+	first, err := p.Load(context.Background(), 0)
+	require.NoError(t, err)
+	first[key("a", s3lifecycle.ActionKindExpirationDays)] = 9_999
+	first[key("z", s3lifecycle.ActionKindAbortMPU)] = 42
+
+	second, err := p.Load(context.Background(), 0)
+	require.NoError(t, err)
+	assert.Equal(t, int64(100), second[key("a", s3lifecycle.ActionKindExpirationDays)],
+		"caller-side mutation of the first snapshot must not bleed back")
+	assert.NotContains(t, second, key("z", s3lifecycle.ActionKindAbortMPU),
+		"caller-side insertion into the first snapshot must not bleed back")
+}
+
+func TestInMemoryPersister_SaveReplacesNotMerges(t *testing.T) {
+	// Documented contract: Save replaces prior state atomically. A
+	// merge would silently keep stale resume points across restarts.
+	p := NewInMemoryPersister()
+	require.NoError(t, p.Save(context.Background(), 0, map[s3lifecycle.ActionKey]int64{
+		key("a", s3lifecycle.ActionKindExpirationDays): 100,
+		key("b", s3lifecycle.ActionKindNoncurrentDays): 200,
+	}))
+	require.NoError(t, p.Save(context.Background(), 0, map[s3lifecycle.ActionKey]int64{
+		key("c", s3lifecycle.ActionKindAbortMPU): 300,
+	}))
+
+	got, err := p.Load(context.Background(), 0)
+	require.NoError(t, err)
+	assert.Equal(t, map[s3lifecycle.ActionKey]int64{
+		key("c", s3lifecycle.ActionKindAbortMPU): 300,
+	}, got, "second Save must replace prior state, not merge")
+}
+
+func TestInMemoryPersister_DifferentShardsIsolated(t *testing.T) {
+	// State for shardID=1 must not leak into Load(shardID=2).
+	p := NewInMemoryPersister()
+	require.NoError(t, p.Save(context.Background(), 1, map[s3lifecycle.ActionKey]int64{
+		key("a", s3lifecycle.ActionKindExpirationDays): 100,
+	}))
+	require.NoError(t, p.Save(context.Background(), 2, map[s3lifecycle.ActionKey]int64{
+		key("b", s3lifecycle.ActionKindNoncurrentDays): 200,
+	}))
+
+	got1, err := p.Load(context.Background(), 1)
+	require.NoError(t, err)
+	got2, err := p.Load(context.Background(), 2)
+	require.NoError(t, err)
+	assert.Equal(t, int64(100), got1[key("a", s3lifecycle.ActionKindExpirationDays)])
+	assert.NotContains(t, got1, key("b", s3lifecycle.ActionKindNoncurrentDays))
+	assert.Equal(t, int64(200), got2[key("b", s3lifecycle.ActionKindNoncurrentDays)])
+	assert.NotContains(t, got2, key("a", s3lifecycle.ActionKindExpirationDays))
+}
+
+func TestInMemoryPersister_SaveEmptyMapClearsState(t *testing.T) {
+	// An empty map is valid input; Save must accept it and a subsequent
+	// Load must return an empty map (not the prior contents).
+	p := NewInMemoryPersister()
+	require.NoError(t, p.Save(context.Background(), 0, map[s3lifecycle.ActionKey]int64{
+		key("a", s3lifecycle.ActionKindExpirationDays): 100,
+	}))
+	require.NoError(t, p.Save(context.Background(), 0, map[s3lifecycle.ActionKey]int64{}))
+	got, err := p.Load(context.Background(), 0)
+	require.NoError(t, err)
+	assert.Empty(t, got)
+}
+
+func TestInMemoryPersister_ConcurrentSaveLoadDoesNotRace(t *testing.T) {
+	// The dispatcher fans cursor reads/writes across many goroutines;
+	// the persister must hold under load without livelocking or
+	// data-racing (-race detector enforces that).
+	p := NewInMemoryPersister()
+	var wg sync.WaitGroup
+	const N = 64
+	wg.Add(N * 2)
+	for i := 0; i < N; i++ {
+		i := i
+		go func() {
+			defer wg.Done()
+			_ = p.Save(context.Background(), i%4, map[s3lifecycle.ActionKey]int64{
+				key("k", s3lifecycle.ActionKindExpirationDays): int64(i),
+			})
+		}()
+		go func() {
+			defer wg.Done()
+			_, _ = p.Load(context.Background(), i%4)
+		}()
+	}
+	wg.Wait()
+}

weed/s3api/s3lifecycle/reader/reader.go

@@ -0,0 +1,338 @@
+package reader
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+)
+
+// Event is one in-shard meta-log event delivered to the router.
+//
+// BootstrapVersion is set only by the bucket bootstrapper when it
+// expands a .versions/ directory; the meta-log path leaves it nil.
+// Carries pre-computed sibling state so the router can fire
+// NoncurrentDays / NewerNoncurrent without listing again.
+type Event struct {
+	TsNs             int64
+	Bucket           string
+	Key              string
+	ShardID          int
+	OldEntry         *filer_pb.Entry
+	NewEntry         *filer_pb.Entry
+	NewParent        string
+	BootstrapVersion *BootstrapVersion
+}
+
+// BootstrapVersion is the per-version state computed once per
+// .versions/<key>/ directory at bootstrap time. Key fields shape
+// EvaluateAction: IsLatest gates current vs. noncurrent rules,
+// NoncurrentIndex gates NewerNoncurrentVersions retention,
+// SuccessorModTime sets the noncurrent clock (when this version was
+// replaced).
+type BootstrapVersion struct {
+	LogicalKey       string
+	VersionID        string
+	IsLatest         bool
+	IsDeleteMarker   bool
+	NumVersions      int
+	NoncurrentIndex  int // 0 = newest noncurrent
+	SuccessorModTime time.Time
+}
+
+// IsDelete reports whether this event removes an entry.
+func (e *Event) IsDelete() bool {
+	return e.NewEntry == nil && e.OldEntry != nil
+}
+
+// IsCreate reports whether this event creates an entry.
+func (e *Event) IsCreate() bool {
+	return e.OldEntry == nil && e.NewEntry != nil
+}
+
+// Reader subscribes to the filer meta-log and emits in-range Events to a
+// channel. One subscription handles a contiguous span (or arbitrary set)
+// of shards via ShardPredicate; the downstream router/dispatcher consume
+// events and ack-advance the per-shard cursor for matched ActionKeys
+// when their actions complete.
+type Reader struct {
+	// ShardID and ShardPredicate are alternatives — set at most one.
+	// ShardPredicate wins if both are populated.
+	ShardID        int            // [0, s3lifecycle.ShardCount); used when ShardPredicate is nil
+	ShardPredicate func(int) bool // accepts an event when true; nil falls back to ShardID equality
+
+	BucketsPath string // e.g. "/buckets"
+	// Cursor is the single-shard cursor used for SinceNs when StartTsNs is 0.
+	// Range callers pass StartTsNs directly and leave Cursor nil; SinceNs=0
+	// then means "subscribe from the start of the meta-log".
+	Cursor    *Cursor
+	StartTsNs int64
+	Events    chan<- *Event
+
+	// UntilTsNs ends the stream once the filer has delivered through this
+	// timestamp. Zero follows forever, which on an idle cluster parks a
+	// bounded caller in Recv until something unrelated is written.
+	UntilTsNs int64
+
+	// EventBudget caps how many events Run processes before returning nil.
+	// Zero = unbounded; the run continues until ctx cancellation or stream
+	// error. Used by the worker scheduler to bound a single READ task.
+	EventBudget int
+
+	// ReceiveTimeout bounds the wait for each response, covering a filer
+	// that stops producing while the transport still answers keepalives.
+	// Also opts into idle heartbeats so a caught-up stream stays alive.
+	// Keep above the filer's 15m maxGapStall — a subscriber parked on a
+	// gap sends nothing and is not stuck. Zero disables it.
+	ReceiveTimeout time.Duration
+
+	// bucketsPathSlash is BucketsPath with a guaranteed trailing slash,
+	// computed once on Run and reused per event to avoid recomputing the
+	// normalized prefix in extractBucketKey.
+	bucketsPathSlash string
+}
+
+// ErrReceiveTimeout: stream still open, but no events and no heartbeats.
+var ErrReceiveTimeout = errors.New("reader: metadata receive timeout")
+
+type receiveResult struct {
+	resp *filer_pb.SubscribeMetadataResponse
+	err  error
+}
+
+// awaitResponse waits for the next response under ReceiveTimeout. Started
+// per call, so it times the filer only — a slow Events consumer blocks in
+// dispatchOne, outside this window, and can't trip the watchdog.
+func (r *Reader) awaitResponse(ctx context.Context, received <-chan receiveResult) (*filer_pb.SubscribeMetadataResponse, error, bool) {
+	var timeout <-chan time.Time
+	if r.ReceiveTimeout > 0 {
+		timer := time.NewTimer(r.ReceiveTimeout)
+		defer timer.Stop()
+		timeout = timer.C
+	}
+	select {
+	case result := <-received:
+		return result.resp, result.err, false
+	case <-timeout:
+		return nil, nil, true
+	case <-ctx.Done():
+		return nil, ctx.Err(), false
+	}
+}
+
+// Run subscribes via SubscribeMetadata starting at the configured position,
+// filters to the configured shard set, and emits Events. Returns on
+// ctx.Done(), io.EOF, or stream error. Caller is responsible for closing
+// Events if it owns it.
+func (r *Reader) Run(ctx context.Context, client filer_pb.SeaweedFilerClient, clientName string, clientID int32) error {
+	if r.ShardPredicate == nil {
+		if r.ShardID < 0 || r.ShardID >= s3lifecycle.ShardCount {
+			return fmt.Errorf("reader: shard_id %d out of range and no ShardPredicate", r.ShardID)
+		}
+	}
+	if r.Events == nil {
+		return errors.New("reader: nil Events channel")
+	}
+	if r.BucketsPath == "" {
+		return errors.New("reader: empty BucketsPath")
+	}
+	if r.ReceiveTimeout < 0 {
+		return fmt.Errorf("reader: negative ReceiveTimeout %v", r.ReceiveTimeout)
+	}
+	r.bucketsPathSlash = r.BucketsPath
+	if !strings.HasSuffix(r.bucketsPathSlash, "/") {
+		r.bucketsPathSlash += "/"
+	}
+
+	sinceNs := r.StartTsNs
+	if sinceNs == 0 && r.Cursor != nil {
+		sinceNs = r.Cursor.MinTsNs()
+	}
+	// Own context: aborting the RPC is the only way to unblock Recv.
+	streamCtx, cancelStream := context.WithCancel(ctx)
+	defer cancelStream()
+	stream, err := client.SubscribeMetadata(streamCtx, &filer_pb.SubscribeMetadataRequest{
+		ClientName:             clientName,
+		PathPrefix:             r.BucketsPath,
+		SinceNs:                sinceNs,
+		UntilNs:                r.UntilTsNs,
+		ClientId:               clientID,
+		ClientSupportsBatching: true,
+		// dispatchOne drops these, but arriving at all is the point.
+		ClientSupportsIdleHeartbeat: r.ReceiveTimeout > 0,
+	})
+	if err != nil {
+		return fmt.Errorf("subscribe: %w", err)
+	}
+
+	// Buffered so a response landing as the watchdog fires doesn't strand
+	// this goroutine.
+	received := make(chan receiveResult, 1)
+	go func() {
+		for {
+			resp, recvErr := stream.Recv()
+			select {
+			case received <- receiveResult{resp: resp, err: recvErr}:
+			case <-streamCtx.Done():
+				return
+			}
+			if recvErr != nil {
+				return
+			}
+		}
+	}()
+
+	processed := 0
+	for {
+		resp, recvErr, timedOut := r.awaitResponse(streamCtx, received)
+		if timedOut {
+			cancelStream() // unwind the goroutine blocked in Recv
+			return fmt.Errorf("%w after %s", ErrReceiveTimeout, r.ReceiveTimeout)
+		}
+		if recvErr == io.EOF {
+			return nil
+		}
+		if recvErr != nil {
+			return recvErr
+		}
+
+		// First event in resp is the primary; resp.Events is the batched tail.
+		if err := r.dispatchOne(ctx, resp, &processed); err != nil {
+			return err
+		}
+		for _, ev := range resp.Events {
+			if err := r.dispatchOne(ctx, ev, &processed); err != nil {
+				return err
+			}
+		}
+		if r.EventBudget > 0 && processed >= r.EventBudget {
+			return nil
+		}
+	}
+}
+
+func (r *Reader) dispatchOne(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, processed *int) error {
+	if resp == nil || resp.EventNotification == nil {
+		return nil
+	}
+	bucket, key, ok := r.extractBucketKey(resp)
+	if !ok {
+		return nil
+	}
+	shardID := s3lifecycle.ShardID(bucket, key)
+	if r.ShardPredicate != nil {
+		if !r.ShardPredicate(shardID) {
+			return nil
+		}
+	} else if shardID != r.ShardID {
+		return nil
+	}
+
+	ev := &Event{
+		TsNs:      resp.TsNs,
+		Bucket:    bucket,
+		Key:       key,
+		ShardID:   shardID,
+		OldEntry:  resp.EventNotification.OldEntry,
+		NewEntry:  resp.EventNotification.NewEntry,
+		NewParent: resp.EventNotification.NewParentPath,
+	}
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case r.Events <- ev:
+		*processed++
+		return nil
+	}
+}
+
+// extractBucketKey turns a meta-log event's path into (bucket, key) when the
+// event lies under BucketsPath. Returns ok=false for events outside that
+// subtree (cluster admin entries, system files, etc.) so the reader can skip
+// them without engaging the routing index.
+//
+// The path is reconstructed as DirectoryPath/Name, where DirectoryPath comes
+// from the entry context and Name from old_entry/new_entry. We prefer
+// new_entry on creates/updates and old_entry on deletes; both carry the same
+// Name on renames where new_parent_path differs.
+func (r *Reader) extractBucketKey(resp *filer_pb.SubscribeMetadataResponse) (string, string, bool) {
+	notif := resp.EventNotification
+	dir := notif.NewParentPath
+	if dir == "" {
+		// On deletes, NewParentPath may be empty; the directory is encoded
+		// in resp.Directory.
+		dir = resp.Directory
+	}
+	var name string
+	switch {
+	case notif.NewEntry != nil:
+		name = notif.NewEntry.Name
+	case notif.OldEntry != nil:
+		name = notif.OldEntry.Name
+	default:
+		return "", "", false
+	}
+
+	// Pre-normalized prefix (BucketsPath with trailing slash) is computed
+	// once in Run; bucket-root events arrive as either "/buckets" or
+	// "/buckets/", so accept both. The fallback path mirrors Run's
+	// normalization for tests that call extractBucketKey directly.
+	prefix := r.bucketsPathSlash
+	if prefix == "" {
+		prefix = r.BucketsPath
+		if !strings.HasSuffix(prefix, "/") {
+			prefix += "/"
+		}
+	}
+	bare := strings.TrimSuffix(prefix, "/")
+	var rest string
+	switch {
+	case dir == bare || dir == prefix:
+		// Bucket create/delete at /buckets root: bucket name is the entry name.
+		if name == "" {
+			return "", "", false
+		}
+		return name, "", true
+	case strings.HasPrefix(dir, prefix):
+		rest = dir[len(prefix):]
+	default:
+		return "", "", false
+	}
+	// rest = "<bucket>" or "<bucket>/<sub>/<sub>..."
+	slash := strings.IndexByte(rest, '/')
+	var bucket, parentInBucket string
+	if slash < 0 {
+		bucket = rest
+	} else {
+		bucket = rest[:slash]
+		parentInBucket = rest[slash+1:]
+	}
+	if bucket == "" {
+		return "", "", false
+	}
+	if parentInBucket != "" {
+		return bucket, parentInBucket + "/" + name, true
+	}
+	return bucket, name, true
+}
+
+// LogStartup is a small helper for callers that want a one-line readable
+// description of where the reader is starting.
+func (r *Reader) LogStartup() {
+	sinceNs := r.StartTsNs
+	if sinceNs == 0 && r.Cursor != nil {
+		sinceNs = r.Cursor.MinTsNs()
+	}
+	if r.ShardPredicate != nil {
+		glog.V(1).Infof("lifecycle reader: shard=range sinceNs=%d budget=%d", sinceNs, r.EventBudget)
+		return
+	}
+	glog.V(1).Infof("lifecycle reader: shard=%d sinceNs=%d budget=%d",
+		r.ShardID, sinceNs, r.EventBudget)
+}

weed/s3api/s3lifecycle/reader/reader_test.go

@@ -0,0 +1,184 @@
+package reader
+
+import (
+	"context"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+)
+
+func TestExtractBucketKeyCreate(t *testing.T) {
+	r := &Reader{BucketsPath: "/buckets"}
+	resp := &filer_pb.SubscribeMetadataResponse{
+		EventNotification: &filer_pb.EventNotification{
+			NewParentPath: "/buckets/mybucket/path/to",
+			NewEntry:      &filer_pb.Entry{Name: "obj.txt"},
+		},
+	}
+	b, k, ok := r.extractBucketKey(resp)
+	if !ok || b != "mybucket" || k != "path/to/obj.txt" {
+		t.Fatalf("got (%q,%q,%v), want (mybucket,path/to/obj.txt,true)", b, k, ok)
+	}
+}
+
+func TestExtractBucketKeyTopLevelObject(t *testing.T) {
+	r := &Reader{BucketsPath: "/buckets"}
+	resp := &filer_pb.SubscribeMetadataResponse{
+		EventNotification: &filer_pb.EventNotification{
+			NewParentPath: "/buckets/mybucket",
+			NewEntry:      &filer_pb.Entry{Name: "a.txt"},
+		},
+	}
+	b, k, ok := r.extractBucketKey(resp)
+	if !ok || b != "mybucket" || k != "a.txt" {
+		t.Fatalf("got (%q,%q,%v), want (mybucket,a.txt,true)", b, k, ok)
+	}
+}
+
+func TestExtractBucketKeyDeleteUsesOldEntry(t *testing.T) {
+	r := &Reader{BucketsPath: "/buckets"}
+	resp := &filer_pb.SubscribeMetadataResponse{
+		EventNotification: &filer_pb.EventNotification{
+			NewParentPath: "/buckets/b/dir",
+			OldEntry:      &filer_pb.Entry{Name: "gone.txt"},
+		},
+	}
+	b, k, ok := r.extractBucketKey(resp)
+	if !ok || b != "b" || k != "dir/gone.txt" {
+		t.Fatalf("got (%q,%q,%v), want (b,dir/gone.txt,true)", b, k, ok)
+	}
+}
+
+func TestExtractBucketKeyDeleteWithEmptyParent(t *testing.T) {
+	// Some delete events carry the directory in resp.Directory rather than
+	// EventNotification.NewParentPath.
+	r := &Reader{BucketsPath: "/buckets"}
+	resp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/buckets/b/dir",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{Name: "gone.txt"},
+		},
+	}
+	b, k, ok := r.extractBucketKey(resp)
+	if !ok || b != "b" || k != "dir/gone.txt" {
+		t.Fatalf("got (%q,%q,%v), want (b,dir/gone.txt,true)", b, k, ok)
+	}
+}
+
+func TestExtractBucketKeyOutsideBucketsSkipped(t *testing.T) {
+	r := &Reader{BucketsPath: "/buckets"}
+	resp := &filer_pb.SubscribeMetadataResponse{
+		EventNotification: &filer_pb.EventNotification{
+			NewParentPath: "/etc/something",
+			NewEntry:      &filer_pb.Entry{Name: "config"},
+		},
+	}
+	if _, _, ok := r.extractBucketKey(resp); ok {
+		t.Fatal("expected skip for non-bucket path")
+	}
+}
+
+func TestExtractBucketKeyBucketCreateAtRoot(t *testing.T) {
+	// Creating a new bucket emits an event at /buckets/ with the bucket as
+	// the new entry's Name.
+	r := &Reader{BucketsPath: "/buckets"}
+	resp := &filer_pb.SubscribeMetadataResponse{
+		EventNotification: &filer_pb.EventNotification{
+			NewParentPath: "/buckets/",
+			NewEntry:      &filer_pb.Entry{Name: "newbucket"},
+		},
+	}
+	b, k, ok := r.extractBucketKey(resp)
+	if !ok || b != "newbucket" || k != "" {
+		t.Fatalf("got (%q,%q,%v), want (newbucket,,true)", b, k, ok)
+	}
+}
+
+func TestExtractBucketKeyBucketCreateBareRoot(t *testing.T) {
+	// Some delete and create events emit the parent without a trailing
+	// slash. /buckets (no trailing slash) must also resolve to a bucket-
+	// root event with the entry name as the bucket.
+	r := &Reader{BucketsPath: "/buckets"}
+	resp := &filer_pb.SubscribeMetadataResponse{
+		EventNotification: &filer_pb.EventNotification{
+			NewParentPath: "/buckets",
+			NewEntry:      &filer_pb.Entry{Name: "newbucket"},
+		},
+	}
+	b, k, ok := r.extractBucketKey(resp)
+	if !ok || b != "newbucket" || k != "" {
+		t.Fatalf("bare /buckets parent: got (%q,%q,%v), want (newbucket,,true)", b, k, ok)
+	}
+}
+
+func TestDispatchOneFiltersByShard(t *testing.T) {
+	// Pick a bucket+key whose shard is known, set Reader to a different
+	// shard, expect skip; same shard, expect emit.
+	bucket, key := "mybucket", "thing.txt"
+	target := s3lifecycle.ShardID(bucket, key)
+	other := (target + 1) % s3lifecycle.ShardCount
+
+	resp := &filer_pb.SubscribeMetadataResponse{
+		TsNs: 12345,
+		EventNotification: &filer_pb.EventNotification{
+			NewParentPath: "/buckets/" + bucket,
+			NewEntry:      &filer_pb.Entry{Name: key},
+		},
+	}
+
+	out := make(chan *Event, 1)
+	r := &Reader{
+		ShardID:     other,
+		BucketsPath: "/buckets",
+		Cursor:      NewCursor(),
+		Events:      out,
+	}
+	processed := 0
+	if err := r.dispatchOne(context.Background(), resp, &processed); err != nil {
+		t.Fatalf("dispatchOne err: %v", err)
+	}
+	if processed != 0 || len(out) != 0 {
+		t.Fatalf("wrong-shard event should be skipped: processed=%d emitted=%d", processed, len(out))
+	}
+
+	r.ShardID = target
+	if err := r.dispatchOne(context.Background(), resp, &processed); err != nil {
+		t.Fatalf("dispatchOne err: %v", err)
+	}
+	if processed != 1 || len(out) != 1 {
+		t.Fatalf("matching-shard event should be emitted: processed=%d emitted=%d", processed, len(out))
+	}
+	ev := <-out
+	if ev.Bucket != bucket || ev.Key != key || ev.TsNs != 12345 {
+		t.Fatalf("emitted event mismatch: %+v", ev)
+	}
+}
+
+func TestDispatchOneRespectsCtxCancel(t *testing.T) {
+	// Buffered channel of size 0 (unbuffered) blocks the send; ctx cancel
+	// should unblock and return ctx.Err().
+	bucket, key := "mybucket", "thing.txt"
+	target := s3lifecycle.ShardID(bucket, key)
+	resp := &filer_pb.SubscribeMetadataResponse{
+		TsNs: 1,
+		EventNotification: &filer_pb.EventNotification{
+			NewParentPath: "/buckets/" + bucket,
+			NewEntry:      &filer_pb.Entry{Name: key},
+		},
+	}
+	out := make(chan *Event)
+	r := &Reader{
+		ShardID:     target,
+		BucketsPath: "/buckets",
+		Cursor:      NewCursor(),
+		Events:      out,
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	processed := 0
+	err := r.dispatchOne(ctx, resp, &processed)
+	if err != context.Canceled {
+		t.Fatalf("expected ctx.Canceled, got %v", err)
+	}
+}

weed/s3api/s3lifecycle/reader/receive_timeout_test.go

@@ -0,0 +1,117 @@
+package reader
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+)
+
+// stalledStream stays open and never delivers: keepalives answered, no
+// progress.
+type stalledStream struct {
+	grpc.ClientStream
+	ctx context.Context
+}
+
+func (s *stalledStream) Recv() (*filer_pb.SubscribeMetadataResponse, error) {
+	<-s.ctx.Done()
+	return nil, s.ctx.Err()
+}
+
+type stalledClient struct {
+	filer_pb.SeaweedFilerClient
+	req *filer_pb.SubscribeMetadataRequest
+}
+
+func (c *stalledClient) SubscribeMetadata(ctx context.Context, req *filer_pb.SubscribeMetadataRequest, _ ...grpc.CallOption) (filer_pb.SeaweedFiler_SubscribeMetadataClient, error) {
+	c.req = req
+	return &stalledStream{ctx: ctx}, nil
+}
+
+func TestRun_ReceiveTimeoutOnStalledStream(t *testing.T) {
+	client := &stalledClient{}
+	r := &Reader{
+		ShardPredicate: func(int) bool { return true },
+		BucketsPath:    "/buckets",
+		Events:         make(chan *Event, 1),
+		ReceiveTimeout: 50 * time.Millisecond,
+	}
+
+	done := make(chan error, 1)
+	go func() { done <- r.Run(context.Background(), client, "test", 1) }()
+	select {
+	case err := <-done:
+		require.ErrorIs(t, err, ErrReceiveTimeout)
+	case <-time.After(5 * time.Second):
+		t.Fatal("Run did not time out on a stalled stream")
+	}
+	assert.True(t, client.req.ClientSupportsIdleHeartbeat,
+		"a reader with a receive timeout must ask for heartbeats, or a caught-up stream looks stalled")
+}
+
+// heartbeatStream delivers only heartbeats: a ts, no EventNotification.
+type heartbeatStream struct {
+	grpc.ClientStream
+	every time.Duration
+	sent  int
+	max   int
+}
+
+func (s *heartbeatStream) Recv() (*filer_pb.SubscribeMetadataResponse, error) {
+	if s.sent >= s.max {
+		return nil, context.Canceled
+	}
+	time.Sleep(s.every)
+	s.sent++
+	return &filer_pb.SubscribeMetadataResponse{TsNs: int64(s.sent)}, nil
+}
+
+type heartbeatClient struct {
+	filer_pb.SeaweedFilerClient
+	stream *heartbeatStream
+}
+
+func (c *heartbeatClient) SubscribeMetadata(_ context.Context, _ *filer_pb.SubscribeMetadataRequest, _ ...grpc.CallOption) (filer_pb.SeaweedFiler_SubscribeMetadataClient, error) {
+	return c.stream, nil
+}
+
+// Why the timeout is safe to set: a caught-up stream keeps arriving, so
+// the watchdog fires on real silence, not on an idle cluster.
+func TestRun_HeartbeatsHoldTheStreamOpen(t *testing.T) {
+	stream := &heartbeatStream{every: 20 * time.Millisecond, max: 10}
+	r := &Reader{
+		ShardPredicate: func(int) bool { return true },
+		BucketsPath:    "/buckets",
+		Events:         make(chan *Event, 1),
+		ReceiveTimeout: 200 * time.Millisecond,
+	}
+
+	done := make(chan error, 1)
+	go func() { done <- r.Run(context.Background(), &heartbeatClient{stream: stream}, "test", 1) }()
+	select {
+	case err := <-done:
+		assert.False(t, errors.Is(err, ErrReceiveTimeout),
+			"heartbeats spanning more than one timeout window must keep the stream alive")
+	case <-time.After(5 * time.Second):
+		t.Fatal("Run never returned")
+	}
+	assert.Equal(t, 10, stream.sent, "every heartbeat should have been consumed")
+}
+
+func TestRun_RejectsNegativeReceiveTimeout(t *testing.T) {
+	r := &Reader{
+		ShardPredicate: func(int) bool { return true },
+		BucketsPath:    "/buckets",
+		Events:         make(chan *Event, 1),
+		ReceiveTimeout: -time.Second,
+	}
+	err := r.Run(context.Background(), &stalledClient{}, "test", 1)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "ReceiveTimeout")
+}