snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/s3/lifecycle/Makefile

@@ -0,0 +1,86 @@
+# S3 Lifecycle Test Makefile
+# End-to-end test of the event-driven lifecycle worker, driven by
+# the s3.lifecycle.run-shard shell command.
+
+.PHONY: help build-weed start-server stop-server test test-with-server clean health-check
+
+WEED_BINARY := ../../../weed/weed_binary
+S3_PORT := 8333
+MASTER_PORT := 9333
+VOLUME_PORT := 8080
+FILER_PORT := 8888
+# Pin the filer gRPC port off the FILER_PORT+10000 convention so any
+# code path that assumes the offset breaks here, not in production.
+FILER_GRPC_PORT := 18890
+ADMIN_PORT := 23646
+ACCESS_KEY ?= some_access_key1
+SECRET_KEY ?= some_secret_key1
+TEST_TIMEOUT := 10m
+TEST_PATTERN := TestLifecycle
+SERVER_DIR := ./test-volume-data/server-data
+
+help:
+	@echo "S3 Lifecycle Test Makefile"
+	@echo ""
+	@echo "Targets:"
+	@echo "  build-weed       - build the SeaweedFS binary"
+	@echo "  start-server     - start a local 'weed mini' cluster"
+	@echo "  stop-server      - stop the cluster"
+	@echo "  test             - run tests against an already-running cluster"
+	@echo "  test-with-server - start, run tests, stop"
+	@echo "  health-check     - is the S3 endpoint up?"
+	@echo "  clean            - remove test artifacts"
+
+build-weed:
+	@echo "Building SeaweedFS binary..."
+	@cd ../../../weed && go build -o weed_binary .
+	@chmod +x $(WEED_BINARY)
+
+start-server: build-weed
+	@echo "Starting weed mini..."
+	@rm -f weed-server.pid
+	@mkdir -p $(SERVER_DIR)
+	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \
+		-dir=$(SERVER_DIR) \
+		-s3.port=$(S3_PORT) \
+		-filer.port.grpc=$(FILER_GRPC_PORT) \
+		> weed-test.log 2>&1 & \
+		echo $$! > weed-server.pid
+	@for i in $$(seq 1 90); do \
+		if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
+			echo "server up after $$i s"; exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "server did not start within 90s"; \
+	tail -50 weed-test.log; \
+	exit 1
+
+stop-server:
+	@if [ -f weed-server.pid ]; then \
+		PID=$$(cat weed-server.pid); \
+		kill -TERM $$PID 2>/dev/null || true; \
+		sleep 1; \
+		kill -KILL $$PID 2>/dev/null || true; \
+		rm -f weed-server.pid; \
+	fi
+
+health-check:
+	@curl -s http://localhost:$(S3_PORT) >/dev/null && echo "S3 up on $(S3_PORT)" || (echo "S3 not reachable"; exit 1)
+
+test:
+	@WEED_BINARY=$$(cd ../../../weed && pwd)/weed_binary \
+	S3_ENDPOINT=http://localhost:$(S3_PORT) \
+	S3_GRPC_ENDPOINT=localhost:$$(($(S3_PORT) + 10000)) \
+	MASTER_ENDPOINT=http://localhost:$(MASTER_PORT) \
+	FILER_GRPC_ADDRESS=localhost:$(FILER_GRPC_PORT) \
+	ADMIN_ENDPOINT=http://localhost:$(ADMIN_PORT) \
+	go test -v -timeout $(TEST_TIMEOUT) -run $(TEST_PATTERN)
+
+test-with-server: start-server
+	@$(MAKE) test || (RC=$$?; $(MAKE) stop-server; exit $$RC)
+	@$(MAKE) stop-server
+
+clean: stop-server
+	@rm -rf $(SERVER_DIR) test-volume-data
+	@rm -f weed-test.log weed-server.pid

test/s3/lifecycle/README.md

@@ -0,0 +1,38 @@
+# S3 Lifecycle Integration Tests
+
+End-to-end test of the event-driven S3 lifecycle worker, exercised through
+the `s3.lifecycle.run-shard` shell command.
+
+## Why backdate mtimes?
+
+The S3 API rejects `Expiration.Days < 1`, so a literal "wait one day"
+integration test isn't workable. Each test sets up a 1-day expiration rule,
+puts the target object, then rewrites its filer entry's `Mtime` to ~30 days
+ago via `filer_pb.UpdateEntry`. From the engine's perspective the object is
+past its expiration window the moment the shell command starts.
+
+## Running
+
+```sh
+# build the binary, start a local mini cluster, run tests, stop it
+make test-with-server
+
+# or, if a cluster is already running on the default ports
+make test
+```
+
+The test runs the shell command once with `-shards 0-15` (one filer
+subscription covering all 16 shards) rather than computing the target
+object's shard up front. This keeps the test independent of the
+`ShardID(bucket, key)` hash function — only that some shard reaches the
+deletion within the polling window.
+
+## Environment
+
+| variable             | default                  | description                |
+|----------------------|--------------------------|----------------------------|
+| `WEED_BINARY`        | _required_               | path to `weed_binary`      |
+| `S3_ENDPOINT`        | `http://localhost:8333`  | S3 API URL                 |
+| `S3_GRPC_ENDPOINT`   | `localhost:18333`        | S3 gRPC for lifecycle dispatch |
+| `MASTER_ENDPOINT`    | `http://localhost:9333`  | master HTTP                |
+| `FILER_GRPC_ADDRESS` | `localhost:18888`        | filer gRPC for `UpdateEntry` |

test/s3/lifecycle/s3_lifecycle_admin_dispatch_test.go

@@ -0,0 +1,155 @@
+package lifecycle
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/require"
+)
+
+// Drives the real admin->worker dispatch path via the run endpoint and
+// asserts the dispatch reaches the filer (no dial error) and deletes
+// the backdated object. The Makefile pins filer.port.grpc off the
+// FILER_PORT+10000 convention so a raw-address forwarding regression
+// resurfaces here.
+func TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort(t *testing.T) {
+	adminEndpoint := envOr("ADMIN_ENDPOINT", defaultAdminEndpoint)
+
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("admin-dispatch")
+	mustCreateBucket(t, c, bucket)
+	putExpirationLifecycle(t, c, bucket, "expire/", 1)
+	const oldKey = "expire/old.txt"
+	putObject(t, c, bucket, oldKey, "old")
+	backdateMtime(t, fc, bucket, oldKey, 30)
+
+	waitForLifecycleWorkerReady(t, adminEndpoint)
+
+	payload := runLifecycleJob(t, adminEndpoint, 30)
+
+	require.GreaterOrEqual(t, jsonNumber(t, payload, "detected_count"), 1)
+	require.Equal(t, 0, jsonNumber(t, payload, "error_count"),
+		"dispatched job errored — likely filer_grpc_address was raw host:httpPort.grpcPort")
+
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(oldKey),
+		})
+		return err != nil
+	}, 30*time.Second, 500*time.Millisecond,
+		"expected %s/%s to be deleted after admin-dispatched lifecycle run", bucket, oldKey)
+}
+
+// A worker-dispatched pass with rules in place but nothing due: the
+// invariant is that a pass returns on its own, so the admin never has to
+// cancel it and the executor slot is free for the next one.
+//
+// Not a regression test for the wedge this scenario comes from. Before
+// the subscription was bounded a pass ended only when some write landed
+// past its boundary, and on a shared test cluster something usually
+// does — verified: the whole suite passes on the unfixed build. The
+// deterministic guards are the dailyrun unit tests. This covers the
+// worker path end-to-end and would catch a pass that hangs
+// unconditionally.
+func TestLifecycleAdminDispatchCompletesWithNothingDue(t *testing.T) {
+	adminEndpoint := envOr("ADMIN_ENDPOINT", defaultAdminEndpoint)
+
+	c := s3Client(t)
+	bucket := uniqueBucket("admin-idle")
+	mustCreateBucket(t, c, bucket)
+	putExpirationLifecycle(t, c, bucket, "expire/", 1)
+	// Deliberately not backdated: the rule matches the prefix but nothing
+	// is due, so the pass dispatches nothing.
+	putObject(t, c, bucket, "expire/fresh.txt", "fresh")
+
+	waitForLifecycleWorkerReady(t, adminEndpoint)
+
+	// Let the writes above settle below the pass boundary, or one of them
+	// ends the pass and the wedge is masked.
+	time.Sleep(2 * time.Second)
+
+	payload := runLifecycleJob(t, adminEndpoint, 30)
+
+	require.Equal(t, 0, jsonNumber(t, payload, "canceled_count"),
+		"pass did not return on its own; the admin cancelled it at the run timeout")
+	require.GreaterOrEqual(t, jsonNumber(t, payload, "success_count"), 1,
+		"expected the dispatched pass to complete")
+	require.Equal(t, 0, jsonNumber(t, payload, "skipped_active_count"),
+		"a previous pass still held the executor slot")
+}
+
+// runLifecycleJob drives the admin's run endpoint and returns its decoded
+// response. timeoutSeconds bounds the run admin-side: a pass that never
+// returns comes back as canceled_count=1 after that long.
+func runLifecycleJob(t *testing.T, adminEndpoint string, timeoutSeconds int) map[string]any {
+	t.Helper()
+	body, err := json.Marshal(map[string]any{"timeout_seconds": timeoutSeconds})
+	require.NoError(t, err)
+	req, err := http.NewRequestWithContext(
+		context.Background(), http.MethodPost,
+		adminEndpoint+"/api/plugin/job-types/s3_lifecycle/run",
+		bytes.NewReader(body),
+	)
+	require.NoError(t, err)
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	var payload map[string]any
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&payload))
+	t.Logf("admin /api/plugin/job-types/s3_lifecycle/run response: %v", payload)
+	require.Equal(t, http.StatusOK, resp.StatusCode, "admin run endpoint failed: %v", payload)
+	return payload
+}
+
+func jsonNumber(t *testing.T, payload map[string]any, key string) int {
+	t.Helper()
+	raw, ok := payload[key]
+	require.Truef(t, ok, "response missing key %q: %v", key, payload)
+	switch v := raw.(type) {
+	case float64:
+		return int(v)
+	case json.Number:
+		n, err := v.Int64()
+		require.NoErrorf(t, err, "key %q is not an int: %v", key, raw)
+		return int(n)
+	default:
+		t.Fatalf("key %q has unexpected type %T (%v)", key, raw, raw)
+		return 0
+	}
+}
+
+func waitForLifecycleWorkerReady(t *testing.T, adminEndpoint string) {
+	t.Helper()
+	deadline := time.Now().Add(30 * time.Second)
+	for time.Now().Before(deadline) {
+		req, err := http.NewRequest(http.MethodGet, adminEndpoint+"/api/plugin/scheduler-states", nil)
+		require.NoError(t, err)
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			time.Sleep(250 * time.Millisecond)
+			continue
+		}
+		var payload any
+		_ = json.NewDecoder(resp.Body).Decode(&payload)
+		resp.Body.Close()
+		if resp.StatusCode == http.StatusOK && strings.Contains(fmt.Sprintf("%v", payload), "s3_lifecycle") {
+			return
+		}
+		time.Sleep(250 * time.Millisecond)
+	}
+	t.Fatalf("admin never reported an s3_lifecycle-capable worker")
+}

test/s3/lifecycle/s3_lifecycle_bootstrap_walk_test.go

@@ -0,0 +1,79 @@
+// Bootstrap-walk integration scenario: lifecycle config added AFTER the
+// objects exist must still expire them on the first sweep.
+package lifecycle
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleBootstrapWalkOnExistingObjects: PUT a batch of objects
+// FIRST, backdate them, THEN configure the lifecycle rule, THEN run
+// the worker. The worker must discover the pre-existing objects via
+// the bootstrap walk (BucketBootstrapper) — there were no meta-log
+// events the reader could have observed because the objects predate
+// the rule.
+//
+// Production scenario: an operator enables lifecycle on a bucket that
+// already holds a million objects from before the policy. Without a
+// bootstrap walk, only NEW writes would ever match; the existing
+// content would never expire.
+func TestLifecycleBootstrapWalkOnExistingObjects(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("bootstrap")
+	mustCreateBucket(t, c, bucket)
+
+	// 5 objects so the test exercises the walker iterating multiple
+	// entries in the same bucket directory. PUT FIRST, before any
+	// lifecycle config is set, so the meta-log doesn't carry events
+	// that match any rule.
+	const total = 5
+	keys := make([]string, total)
+	for i := 0; i < total; i++ {
+		keys[i] = fmt.Sprintf("preexisting/obj-%02d.txt", i)
+		putObject(t, c, bucket, keys[i], "content")
+		backdateMtime(t, fc, bucket, keys[i], 30)
+	}
+
+	// One out-of-prefix object that must NOT be expired. Same
+	// pre-existing PUT timing.
+	const survivor = "other/keep.txt"
+	putObject(t, c, bucket, survivor, "keep")
+	backdateMtime(t, fc, bucket, survivor, 30)
+
+	// NOW set the rule. Up to this point the worker (if it were
+	// running event-driven only) would have nothing matching to do.
+	putExpirationLifecycle(t, c, bucket, "preexisting/", 1)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// Every pre-existing in-prefix object must be expired by the
+	// bootstrap walk's discovery + dispatch.
+	for _, k := range keys {
+		k := k
+		require.Eventuallyf(t, func() bool {
+			_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+				Bucket: aws.String(bucket), Key: aws.String(k),
+			})
+			return isS3NotFound(err)
+		}, 30*time.Second, 500*time.Millisecond,
+			"pre-existing %s/%s must be discovered + expired by bootstrap walk", bucket, k)
+	}
+
+	// Out-of-prefix object stays — pin that the bootstrap walk
+	// doesn't ignore the rule's prefix filter.
+	_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(survivor),
+	})
+	require.NoError(t, err, "out-of-prefix object must survive the bootstrap walk")
+}

test/s3/lifecycle/s3_lifecycle_config_update_test.go

@@ -0,0 +1,79 @@
+// Lifecycle config update across sweeps.
+package lifecycle
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleConfigUpdateBetweenSweeps: an operator changes the
+// lifecycle rule between two shell-driven sweeps. The second sweep
+// must respect the NEW rule, not a cached version of the old one.
+//
+// Each `runLifecycleShard` invocation spawns a fresh `weed shell`
+// subprocess, so cached engine state from a previous sweep doesn't
+// persist across runs. This test pins that the freshly-loaded config
+// actually changes routing — under the new prefix only matching
+// objects expire, even if there are still backdated objects sitting
+// under the old prefix.
+func TestLifecycleConfigUpdateBetweenSweeps(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("config-update")
+	mustCreateBucket(t, c, bucket)
+
+	// Sweep 1: rule expires anything under "first/".
+	putExpirationLifecycle(t, c, bucket, "first/", 1)
+
+	const firstKey = "first/initial.txt"
+	putObject(t, c, bucket, firstKey, "first")
+	backdateMtime(t, fc, bucket, firstKey, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("sweep 1 output:\n%s", out)
+
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(firstKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "sweep 1 must expire %s", firstKey)
+
+	// Update the rule to a different prefix. The old "first/" prefix
+	// is no longer covered by any rule; objects under it must NOT be
+	// expired by sweep 2 even when backdated.
+	putExpirationLifecycle(t, c, bucket, "second/", 1)
+
+	const secondKey = "second/new.txt"
+	const oldPrefixKey = "first/post-update.txt"
+	putObject(t, c, bucket, secondKey, "second")
+	putObject(t, c, bucket, oldPrefixKey, "stale rule")
+	backdateMtime(t, fc, bucket, secondKey, 30)
+	backdateMtime(t, fc, bucket, oldPrefixKey, 30)
+
+	out = runLifecycleShard(t)
+	t.Logf("sweep 2 output:\n%s", out)
+
+	// Sweep 2 expires the new-prefix object.
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(secondKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "sweep 2 must expire %s under the new rule", secondKey)
+
+	// Sweep 2 must NOT expire the old-prefix object — the rule was
+	// replaced, not merged. A regression that caches old rules across
+	// PutBucketLifecycleConfiguration calls would fail here.
+	_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(oldPrefixKey),
+	})
+	require.NoError(t, err, "old-prefix object must survive after rule update — config replacement, not merge")
+}

test/s3/lifecycle/s3_lifecycle_delete_config_test.go

@@ -0,0 +1,67 @@
+// Operator-removes-lifecycle-config integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleDeleteBucketLifecycleStopsDispatching: when an operator
+// runs DeleteBucketLifecycle, subsequent sweeps must NOT continue
+// expiring objects under the now-removed rule. The worker reads each
+// bucket's XML config from filer state on every run; if it cached the
+// previous config across PutBucketLifecycleConfiguration → DeleteBucket
+// Lifecycle the bucket would keep losing objects.
+//
+// Setup: positive control (rule active, backdated obj expires), then
+// DeleteBucketLifecycle, then PUT + backdate a fresh object, run the
+// worker again. The fresh object must remain — no rule, no dispatch.
+func TestLifecycleDeleteBucketLifecycleStopsDispatching(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("delete-config")
+	mustCreateBucket(t, c, bucket)
+	putExpirationLifecycle(t, c, bucket, "x/", 1)
+
+	const firstKey = "x/initial.txt"
+	putObject(t, c, bucket, firstKey, "first")
+	backdateMtime(t, fc, bucket, firstKey, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("sweep 1 output (rule active):\n%s", out)
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(firstKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "positive control: %s should expire while rule is active", firstKey)
+
+	// Now remove the rule. The worker must observe the removal on the
+	// next sweep.
+	_, err := c.DeleteBucketLifecycle(context.Background(), &s3.DeleteBucketLifecycleInput{
+		Bucket: aws.String(bucket),
+	})
+	require.NoError(t, err)
+
+	const survivorKey = "x/post-delete.txt"
+	putObject(t, c, bucket, survivorKey, "should-stay")
+	backdateMtime(t, fc, bucket, survivorKey, 30)
+
+	out = runLifecycleShard(t)
+	t.Logf("sweep 2 output (rule deleted):\n%s", out)
+
+	// Wait the same window; survivor MUST still exist after the worker
+	// has had every opportunity to (incorrectly) act on it.
+	time.Sleep(2 * time.Second)
+	_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(survivorKey),
+	})
+	require.NoError(t, err, "after DeleteBucketLifecycle, sweeps must not expire %s", survivorKey)
+}

test/s3/lifecycle/s3_lifecycle_empty_bucket_test.go

@@ -0,0 +1,46 @@
+// Empty bucket sweep — no-op safety integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleEmptyBucketSweepIsNoOp: a bucket carrying lifecycle
+// config but no objects must produce a successful sweep — no hangs,
+// no errors, no dispatches. Pinned because the bootstrap walker
+// iterates bucket directories, and an empty directory is a corner of
+// that traversal that's easy to break (e.g. a slice-bounds bug on
+// the first listing returning zero entries).
+func TestLifecycleEmptyBucketSweepIsNoOp(t *testing.T) {
+	c := s3Client(t)
+	_, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("empty")
+	mustCreateBucket(t, c, bucket)
+	putExpirationLifecycle(t, c, bucket, "anywhere/", 1)
+
+	// No PUTs. The bucket is empty.
+
+	out := runLifecycleShard(t)
+	t.Logf("sweep output:\n%s", out)
+
+	// Sanity: the worker must report it loaded the bucket's config and
+	// completed without errors. The runLifecycleShard helper already
+	// fails on FATAL; pin the success markers here too so a regression
+	// that produces a half-shaped output (no completion) is caught.
+	require.Contains(t, out, "loaded lifecycle for", "worker must report config load")
+	require.Contains(t, out, "shards 0-15 complete", "worker must complete all shards")
+	require.False(t, strings.Contains(out, "FATAL"), "worker must not produce FATAL output")
+
+	// Bucket still exists after the sweep — the worker doesn't drop
+	// empty buckets as a side effect.
+	_, err := c.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: aws.String(bucket)})
+	require.NoError(t, err, "empty bucket must still exist after sweep")
+}

test/s3/lifecycle/s3_lifecycle_expiration_date_test.go

@@ -0,0 +1,75 @@
+// ExpirationDate (date-based, not Days) integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleExpirationDateInThePast: an Expiration{Date: <past>} rule
+// routes through the engine's ScanAtDate mode rather than the
+// EventDriven delay-group path. The worker's dispatcher must still fire
+// when the date has already passed at the time the worker runs. Pinning
+// this path because most tests use Days-based rules; ScanAtDate is a
+// separate compile + dispatch branch (engine.decideMode case
+// ActionKindExpirationDate) that wouldn't be exercised otherwise.
+func TestLifecycleExpirationDateInThePast(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("expdate")
+	mustCreateBucket(t, c, bucket)
+
+	// Date in the past: AWS rejects ExpirationDate in the future from
+	// being processed early but a past date is the natural integration
+	// test — every object hit by the rule is immediately eligible.
+	pastDate := time.Now().Add(-7 * 24 * time.Hour).UTC().Truncate(24 * time.Hour)
+	_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
+		Bucket: aws.String(bucket),
+		LifecycleConfiguration: &types.BucketLifecycleConfiguration{
+			Rules: []types.LifecycleRule{{
+				ID:     aws.String("expire-by-date"),
+				Status: types.ExpirationStatusEnabled,
+				Filter: &types.LifecycleRuleFilter{Prefix: aws.String("d/")},
+				Expiration: &types.LifecycleExpiration{
+					Date: aws.Time(pastDate),
+				},
+			}},
+		},
+	})
+	require.NoError(t, err)
+
+	const oldKey = "d/in-prefix.txt"
+	const otherKey = "other/skip.txt"
+	putObject(t, c, bucket, oldKey, "old")
+	putObject(t, c, bucket, otherKey, "other")
+
+	// Backdate oldKey too — defense-in-depth so the test doesn't
+	// depend on whether the worker also considers MinTriggerAge for
+	// date kinds; a fresh object with a past date should still expire,
+	// but aging it locks the assertion either way.
+	backdateMtime(t, fc, bucket, oldKey, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(oldKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "%s/%s must be expired by past-date rule", bucket, oldKey)
+
+	// Out-of-prefix object stays — the rule's Filter.Prefix gates this.
+	_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(otherKey),
+	})
+	require.NoError(t, err, "object outside the rule's prefix must remain")
+}

test/s3/lifecycle/s3_lifecycle_mpu_test.go

@@ -0,0 +1,136 @@
+// AbortIncompleteMultipartUpload integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/require"
+)
+
+// putAbortMPULifecycle wires AbortIncompleteMultipartUpload on a prefix.
+// AWS rejects DaysAfterInitiation < 1, so the test backdates the upload
+// directory to age it past the threshold.
+func putAbortMPULifecycle(t *testing.T, c *s3.Client, bucket, prefix string, days int32) {
+	t.Helper()
+	_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
+		Bucket: aws.String(bucket),
+		LifecycleConfiguration: &types.BucketLifecycleConfiguration{
+			Rules: []types.LifecycleRule{
+				{
+					ID:     aws.String("abort-stale-mpu"),
+					Status: types.ExpirationStatusEnabled,
+					Filter: &types.LifecycleRuleFilter{Prefix: aws.String(prefix)},
+					AbortIncompleteMultipartUpload: &types.AbortIncompleteMultipartUpload{
+						DaysAfterInitiation: aws.Int32(days),
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err)
+}
+
+// backdateUploadDir ages the .uploads/<uploadID>/ directory entry, which is
+// what the lifecycle worker keys ABORT_MPU off of (the upload's init-time
+// directory carries the destination key in Extended).
+func backdateUploadDir(t *testing.T, fc filer_pb.SeaweedFilerClient, bucket, uploadID string, daysOld int) {
+	t.Helper()
+	dir := bucketsPath + "/" + bucket + "/.uploads"
+	resp, err := fc.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
+		Directory: dir, Name: uploadID,
+	})
+	require.NoError(t, err, "lookup .uploads/%s", uploadID)
+	require.NotNil(t, resp.Entry)
+	require.NotNil(t, resp.Entry.Attributes)
+	require.True(t, resp.Entry.IsDirectory, ".uploads/%s should be a directory", uploadID)
+
+	resp.Entry.Attributes.Mtime = time.Now().Add(-time.Duration(daysOld) * 24 * time.Hour).Unix()
+	resp.Entry.Attributes.MtimeNs = 0
+	_, err = fc.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
+		Directory: dir, Entry: resp.Entry,
+	})
+	require.NoError(t, err)
+}
+
+// TestLifecycleAbortIncompleteMultipartUpload: an MPU left incomplete past
+// the AbortIncompleteMultipartUpload.DaysAfterInitiation threshold must be
+// aborted by the lifecycle worker. Exercises the lifecycleAbortMPU handler
+// path which is otherwise unreachable from the prefix-based expiration
+// tests (the routing keys off of `.uploads/<id>/` directory events, not
+// regular object events).
+func TestLifecycleAbortIncompleteMultipartUpload(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("abort-mpu")
+	mustCreateBucket(t, c, bucket)
+	putAbortMPULifecycle(t, c, bucket, "uploads/", 1)
+
+	// Initiate an MPU, upload a single part, then leave it. CompleteMultipart
+	// would consume the upload; AbortMultipart would explicitly abort. We do
+	// neither so the lifecycle worker is the path that ends it.
+	const key = "uploads/big.bin"
+	createOut, err := c.CreateMultipartUpload(context.Background(), &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucket), Key: aws.String(key),
+	})
+	require.NoError(t, err)
+	uploadID := aws.ToString(createOut.UploadId)
+	require.NotEmpty(t, uploadID)
+
+	// Upload one part so the directory carries the right shape.
+	_, err = c.UploadPart(context.Background(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucket),
+		Key:        aws.String(key),
+		UploadId:   aws.String(uploadID),
+		PartNumber: aws.Int32(1),
+		Body:       strings.NewReader("part 1 contents"),
+	})
+	require.NoError(t, err)
+
+	// Sanity: ListMultipartUploads sees our pending upload before the
+	// worker runs.
+	listOut, err := c.ListMultipartUploads(context.Background(), &s3.ListMultipartUploadsInput{
+		Bucket: aws.String(bucket),
+	})
+	require.NoError(t, err)
+	var foundBefore bool
+	for _, u := range listOut.Uploads {
+		if aws.ToString(u.UploadId) == uploadID {
+			foundBefore = true
+			break
+		}
+	}
+	require.True(t, foundBefore, "upload %s should be visible before the worker runs", uploadID)
+
+	// Backdate the upload directory entry so the rule's DaysAfterInitiation
+	// threshold is satisfied. The router walks .uploads/ and emits an
+	// MPU-init event for each entry.
+	backdateUploadDir(t, fc, bucket, uploadID, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// The upload must no longer appear in ListMultipartUploads.
+	require.Eventuallyf(t, func() bool {
+		listOut, err := c.ListMultipartUploads(context.Background(), &s3.ListMultipartUploadsInput{
+			Bucket: aws.String(bucket),
+		})
+		if err != nil {
+			return false
+		}
+		for _, u := range listOut.Uploads {
+			if aws.ToString(u.UploadId) == uploadID {
+				return false
+			}
+		}
+		return true
+	}, 30*time.Second, 500*time.Millisecond, "upload %s must be aborted by the lifecycle worker", uploadID)
+}

test/s3/lifecycle/s3_lifecycle_multi_bucket_test.go

@@ -0,0 +1,90 @@
+// Multiple-bucket integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+	"github.com/stretchr/testify/require"
+)
+
+// isS3NotFound recognizes a NotFound (NoSuchKey/404) response from the
+// AWS SDK. Treating any HeadObject error as "deleted" lets a transport
+// failure or dead endpoint mask a real bug, so callers that need to
+// prove deletion specifically should use this.
+func isS3NotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	var nsk *types.NoSuchKey
+	if errors.As(err, &nsk) {
+		return true
+	}
+	var notFound *types.NotFound
+	if errors.As(err, &notFound) {
+		return true
+	}
+	var apiErr *smithyhttp.ResponseError
+	if errors.As(err, &apiErr) {
+		return apiErr.HTTPStatusCode() == 404
+	}
+	return false
+}
+
+// TestLifecycleMultipleBucketsInOneSweep: a single shell-driven shard
+// sweep must process every bucket carrying lifecycle config, not just
+// the first one alphabetically. Pinned because the scheduler iterates
+// the buckets directory and a regression that returns early after the
+// first match would silently disable lifecycle for every later bucket.
+//
+// Two buckets, each with its own 1-day prefix-expiration rule and one
+// backdated object. After the worker runs, both objects must be gone.
+func TestLifecycleMultipleBucketsInOneSweep(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucketA := uniqueBucket("multi-a")
+	bucketB := uniqueBucket("multi-b")
+	mustCreateBucket(t, c, bucketA)
+	mustCreateBucket(t, c, bucketB)
+
+	putExpirationLifecycle(t, c, bucketA, "exp/", 1)
+	putExpirationLifecycle(t, c, bucketB, "exp/", 1)
+
+	const keyA = "exp/a.txt"
+	const keyB = "exp/b.txt"
+	putObject(t, c, bucketA, keyA, "a")
+	putObject(t, c, bucketB, keyB, "b")
+	backdateMtime(t, fc, bucketA, keyA, 30)
+	backdateMtime(t, fc, bucketB, keyB, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// Both buckets must have their objects expired in this single sweep.
+	for _, c2 := range []struct {
+		bucket, key string
+	}{
+		{bucketA, keyA},
+		{bucketB, keyB},
+	} {
+		c2 := c2
+		require.Eventuallyf(t, func() bool {
+			_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+				Bucket: aws.String(c2.bucket), Key: aws.String(c2.key),
+			})
+			// Pin: only count 404 / NoSuchKey as deletion. Any other
+			// error (transport failure, dead endpoint, auth) would
+			// otherwise mask a real bug as a "passed" test.
+			return isS3NotFound(err)
+		}, 30*time.Second, 500*time.Millisecond,
+			"%s/%s must be expired by the multi-bucket sweep", c2.bucket, c2.key)
+	}
+}

test/s3/lifecycle/s3_lifecycle_multi_rule_test.go

@@ -0,0 +1,94 @@
+// Multiple-rules-per-bucket integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleMultipleRulesInOneBucket: a single bucket carries two
+// independent Expiration rules with disjoint prefix filters and
+// different Days thresholds. Each rule must fire only on its prefix
+// and ignore the other; objects outside both prefixes must survive.
+//
+// Pinned because compile.Compile builds one CompiledAction per rule
+// per kind, all sharing the same bucket index. A bug that lets one
+// rule's prefix or threshold leak into the other (e.g. last-write-
+// wins on a shared map) would silently expire wrong objects.
+func TestLifecycleMultipleRulesInOneBucket(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("multi-rule")
+	mustCreateBucket(t, c, bucket)
+
+	_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
+		Bucket: aws.String(bucket),
+		LifecycleConfiguration: &types.BucketLifecycleConfiguration{
+			Rules: []types.LifecycleRule{
+				{
+					ID:         aws.String("logs-1d"),
+					Status:     types.ExpirationStatusEnabled,
+					Filter:     &types.LifecycleRuleFilter{Prefix: aws.String("logs/")},
+					Expiration: &types.LifecycleExpiration{Days: aws.Int32(1)},
+				},
+				{
+					ID:         aws.String("tmp-7d"),
+					Status:     types.ExpirationStatusEnabled,
+					Filter:     &types.LifecycleRuleFilter{Prefix: aws.String("tmp/")},
+					Expiration: &types.LifecycleExpiration{Days: aws.Int32(7)},
+				},
+			},
+		},
+	})
+	require.NoError(t, err)
+
+	const logsKey = "logs/access.log"
+	const tmpKey = "tmp/scratch.bin"
+	const otherKey = "data/keep.bin"
+
+	putObject(t, c, bucket, logsKey, "log entry")
+	putObject(t, c, bucket, tmpKey, "scratch")
+	putObject(t, c, bucket, otherKey, "important")
+
+	// Backdate logs/ past the 1d rule, tmp/ past the 7d rule, data/
+	// past both thresholds — but data/ matches no rule, so it must
+	// survive regardless of age.
+	backdateMtime(t, fc, bucket, logsKey, 30)
+	backdateMtime(t, fc, bucket, tmpKey, 30)
+	backdateMtime(t, fc, bucket, otherKey, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// logs/ rule fires.
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(logsKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "logs/ rule must expire %s", logsKey)
+
+	// tmp/ rule fires.
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(tmpKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "tmp/ rule must expire %s", tmpKey)
+
+	// data/ matches NEITHER rule and must survive — pinning that the
+	// per-rule prefix gate is independent and additive, not a global
+	// "any rule's prefix" check.
+	_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(otherKey),
+	})
+	require.NoError(t, err, "object outside both rule prefixes must survive")
+}

test/s3/lifecycle/s3_lifecycle_newer_noncurrent_test.go

@@ -0,0 +1,110 @@
+// NewerNoncurrentVersions integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/require"
+)
+
+// putNewerNoncurrentLifecycle keeps the N most recent noncurrent versions
+// and expires the rest. NoncurrentDays is required by the AWS XML schema
+// alongside NewerNoncurrentVersions; keeping it at 1 means the rule fires
+// as soon as a version moves past the keep-count and is older than 1 day.
+func putNewerNoncurrentLifecycle(t *testing.T, c *s3.Client, bucket, prefix string, keep int32) {
+	t.Helper()
+	_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
+		Bucket: aws.String(bucket),
+		LifecycleConfiguration: &types.BucketLifecycleConfiguration{
+			Rules: []types.LifecycleRule{
+				{
+					ID:     aws.String("keep-newest-noncurrent"),
+					Status: types.ExpirationStatusEnabled,
+					Filter: &types.LifecycleRuleFilter{Prefix: aws.String(prefix)},
+					NoncurrentVersionExpiration: &types.NoncurrentVersionExpiration{
+						NewerNoncurrentVersions: aws.Int32(keep),
+						NoncurrentDays:          aws.Int32(1),
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err)
+}
+
+// TestLifecycleNewerNoncurrentVersions: NewerNoncurrentVersions=1 keeps
+// only the most recent noncurrent version; older noncurrent versions
+// must be deleted. PUT v1, v2, v3, v4 (v4 current; v1-v3 noncurrent),
+// backdate v1, v2, v3, run the worker, expect v3 (newest noncurrent)
+// and v4 (current) to remain, v1 and v2 to be removed.
+//
+// This routes through routePointerTransition's "needs full expansion"
+// path — distinct from the per-version NoncurrentDays path because
+// the rule depends on per-version rank, not just per-version age.
+func TestLifecycleNewerNoncurrentVersions(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("newer-nc")
+	mustCreateBucket(t, c, bucket)
+	putVersioningEnabled(t, c, bucket)
+	putNewerNoncurrentLifecycle(t, c, bucket, "n/", 1)
+
+	const key = "n/obj.txt"
+	puts := make([]string, 4) // versionIds in PUT order
+	for i := range puts {
+		out, err := c.PutObject(context.Background(), &s3.PutObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(key),
+			Body: strings.NewReader(time.Now().Format(time.RFC3339Nano) + "-" + string(rune('a'+i))),
+		})
+		require.NoError(t, err)
+		puts[i] = aws.ToString(out.VersionId)
+		require.NotEmpty(t, puts[i])
+	}
+	v1, v2, v3, v4 := puts[0], puts[1], puts[2], puts[3]
+	require.NotEqual(t, v1, v2)
+	require.NotEqual(t, v3, v4)
+
+	// Backdate every noncurrent (v1, v2, v3) so all three satisfy the
+	// NoncurrentDays>=1 floor; the keep-newest-1 logic then chooses
+	// which to drop.
+	for _, vid := range []string{v1, v2, v3} {
+		backdateVersionedMtime(t, fc, bucket, key, vid, 30)
+	}
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// v1 and v2 (older noncurrent) must be expired.
+	for _, vid := range []string{v1, v2} {
+		vid := vid
+		require.Eventuallyf(t, func() bool {
+			_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+				Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(vid),
+			})
+			return err != nil
+		}, 30*time.Second, 500*time.Millisecond, "older noncurrent version %s must be expired", vid)
+	}
+
+	// v3 (newest noncurrent within the keep=1 window) must remain.
+	_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(v3),
+	})
+	require.NoError(t, err, "v3 (newest noncurrent within keep=1) must survive")
+
+	// v4 (current) must remain — current versions are immune to
+	// NoncurrentVersionExpiration regardless of how the keep-count
+	// arithmetic plays out.
+	currentHead, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key),
+	})
+	require.NoError(t, err, "current version v4 must remain")
+	require.Equal(t, v4, aws.ToString(currentHead.VersionId))
+}

test/s3/lifecycle/s3_lifecycle_object_lock_test.go

@@ -0,0 +1,119 @@
+// Object Lock + lifecycle integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleSkipsObjectLockedObjects: an object under Object Lock
+// retention must NOT be expired by the lifecycle worker. The handler's
+// enforceObjectLockProtections check (s3api_internal_lifecycle.go:47)
+// returns an error which the dispatcher classifies as
+// SKIPPED_OBJECT_LOCK. Pinning the integration end-to-end so a
+// regression in either the lock state lookup or the dispatcher's
+// outcome handling is caught.
+//
+// Companion positive control: a second object in the same bucket
+// without retention is expired by the same sweep — proves the rule
+// is firing AND that the lock check is selective, not a blanket skip.
+func TestLifecycleSkipsObjectLockedObjects(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("objlock")
+	// Object Lock requires the bucket to be created with the flag set;
+	// it can't be enabled retroactively on an existing bucket. Use the
+	// raw API rather than mustCreateBucket here so we can pass the
+	// ObjectLockEnabledForBucket option.
+	_, err := c.CreateBucket(context.Background(), &s3.CreateBucketInput{
+		Bucket:                     aws.String(bucket),
+		ObjectLockEnabledForBucket: aws.Bool(true),
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		// Best-effort cleanup mirrors mustCreateBucket; locked objects
+		// may resist deletion but the test process continues regardless.
+		c.DeleteBucketLifecycle(context.Background(), &s3.DeleteBucketLifecycleInput{Bucket: aws.String(bucket)})
+		listOut, _ := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{Bucket: aws.String(bucket)})
+		if listOut != nil {
+			for _, v := range listOut.Versions {
+				// Bypass governance retention to free the bucket for delete.
+				c.DeleteObject(context.Background(), &s3.DeleteObjectInput{
+					Bucket:                    aws.String(bucket),
+					Key:                       v.Key,
+					VersionId:                 v.VersionId,
+					BypassGovernanceRetention: aws.Bool(true),
+				})
+			}
+			for _, m := range listOut.DeleteMarkers {
+				c.DeleteObject(context.Background(), &s3.DeleteObjectInput{
+					Bucket: aws.String(bucket), Key: m.Key, VersionId: m.VersionId,
+				})
+			}
+		}
+		c.DeleteBucket(context.Background(), &s3.DeleteBucketInput{Bucket: aws.String(bucket)})
+	})
+
+	putExpirationLifecycle(t, c, bucket, "lock/", 1)
+
+	const lockedKey = "lock/protected.txt"
+	const freeKey = "lock/free.txt"
+
+	// Locked object: PUT with GOVERNANCE retention until 1h from now.
+	lockedPut, err := c.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket:                    aws.String(bucket),
+		Key:                       aws.String(lockedKey),
+		Body:                      strings.NewReader("locked"),
+		ObjectLockMode:            types.ObjectLockModeGovernance,
+		ObjectLockRetainUntilDate: aws.Time(time.Now().Add(time.Hour)),
+	})
+	require.NoError(t, err, "PUT with retention must succeed on a lock-enabled bucket")
+	require.NotEmpty(t, aws.ToString(lockedPut.VersionId))
+
+	// Free object: PUT without retention. Object Lock requires
+	// versioning, so the bucket is implicitly versioned and every PUT
+	// produces a versionId. Capture both for the version-aware
+	// backdate path.
+	freePut, err := c.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(freeKey), Body: strings.NewReader("free"),
+	})
+	require.NoError(t, err)
+	freeVersionID := aws.ToString(freePut.VersionId)
+	require.NotEmpty(t, freeVersionID)
+
+	// Backdate both so they would otherwise both expire under the
+	// 1-day rule. The lock check is what distinguishes them.
+	// Versioning-enabled buckets store entries under .versions/v_<id>,
+	// not at the bare key path, so use backdateVersionedMtime.
+	backdateVersionedMtime(t, fc, bucket, lockedKey, aws.ToString(lockedPut.VersionId), 30)
+	backdateVersionedMtime(t, fc, bucket, freeKey, freeVersionID, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// Free object expires (positive control).
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(freeKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "unlocked object must expire (control)")
+
+	// Locked object survives — Object Lock retention overrides the
+	// lifecycle rule. HEAD without versionId returns the still-current
+	// object data, not a 404.
+	_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(lockedKey),
+	})
+	require.NoError(t, err, "object under GOVERNANCE retention must survive the sweep")
+}
+

test/s3/lifecycle/s3_lifecycle_size_filter_test.go

@@ -0,0 +1,71 @@
+// Size-filter integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleSizeFilterGreaterThan: ObjectSizeGreaterThan is a strict >
+// gate (filterAllows in match.go uses ev.Size <= rule.FilterSizeGreaterThan
+// to reject). Two backdated objects on the same prefix — only the one
+// strictly larger than the threshold is expired.
+func TestLifecycleSizeFilterGreaterThan(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("size-filter")
+	mustCreateBucket(t, c, bucket)
+
+	const threshold = 100
+	_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
+		Bucket: aws.String(bucket),
+		LifecycleConfiguration: &types.BucketLifecycleConfiguration{
+			Rules: []types.LifecycleRule{{
+				ID:     aws.String("size-gt"),
+				Status: types.ExpirationStatusEnabled,
+				Filter: &types.LifecycleRuleFilter{
+					And: &types.LifecycleRuleAndOperator{
+						Prefix:                aws.String("sz/"),
+						ObjectSizeGreaterThan: aws.Int64(threshold),
+					},
+				},
+				Expiration: &types.LifecycleExpiration{Days: aws.Int32(1)},
+			}},
+		},
+	})
+	require.NoError(t, err)
+
+	const smallKey = "sz/small.txt"                                   // size <= threshold → must remain
+	const largeKey = "sz/large.txt"                                   // size > threshold → must expire
+	putObject(t, c, bucket, smallKey, strings.Repeat("a", threshold)) // exactly at boundary; gate is strictly >, so rejects
+	putObject(t, c, bucket, largeKey, strings.Repeat("a", threshold+50))
+
+	backdateMtime(t, fc, bucket, smallKey, 30)
+	backdateMtime(t, fc, bucket, largeKey, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// Larger-than-threshold object expired.
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(largeKey),
+		})
+		return err != nil
+	}, 30*time.Second, 500*time.Millisecond, "object larger than threshold must expire")
+
+	// Boundary object (size == threshold) survives — strictly-greater gate.
+	_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(smallKey),
+	})
+	require.NoError(t, err, "object at exact threshold must remain (gate is strictly >)")
+}

test/s3/lifecycle/s3_lifecycle_suspended_test.go

@@ -0,0 +1,117 @@
+// Suspended-versioning Expiration integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/require"
+)
+
+// putVersioningSuspended flips Status=Suspended. PUTs after this point
+// overwrite the null version in place rather than creating new versionIds.
+func putVersioningSuspended(t *testing.T, c *s3.Client, bucket string) {
+	t.Helper()
+	_, err := c.PutBucketVersioning(context.Background(), &s3.PutBucketVersioningInput{
+		Bucket:                  aws.String(bucket),
+		VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusSuspended},
+	})
+	require.NoError(t, err)
+}
+
+// TestLifecycleSuspendedVersioningExpiration: on a suspended-versioning
+// bucket, Expiration takes a different code path than the Enabled and
+// Off cases. lifecycleDispatch's VersioningSuspended branch first
+// deletes the null version (via deleteSpecificObjectVersion(versionId="null"))
+// and then writes a fresh delete marker on top.
+//
+// Setup: enable then suspend versioning so any noncurrent versions can
+// exist alongside the null one; PUT once-with-versioning + once-after-
+// suspend (so we have a noncurrent version + a null version), backdate
+// the null, run the worker. The null must be deleted and replaced by a
+// delete marker; the prior noncurrent version stays addressable.
+func TestLifecycleSuspendedVersioningExpiration(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("suspended")
+	mustCreateBucket(t, c, bucket)
+	putVersioningEnabled(t, c, bucket)
+
+	const key = "s/obj.txt"
+	// First PUT under Enabled: produces a real versionId.
+	putObject(t, c, bucket, key, "v1")
+	headEnabled, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key),
+	})
+	require.NoError(t, err)
+	v1 := aws.ToString(headEnabled.VersionId)
+	require.NotEmpty(t, v1)
+	require.NotEqual(t, "null", v1)
+
+	// Suspend versioning, then PUT again. The new write becomes the
+	// "null" version that overwrites the bare-key entry; v1 demotes to
+	// a noncurrent version.
+	putVersioningSuspended(t, c, bucket)
+	putObject(t, c, bucket, key, "null-version")
+
+	// Set the Expiration rule AFTER the writes so the rule
+	// configuration race against PUTs is deterministic. Days=1 for
+	// AWS-spec compliance; backdating the null entry makes the worker
+	// fire immediately.
+	putExpirationLifecycle(t, c, bucket, "s/", 1)
+
+	// Backdate the bare-key entry (the null version): on suspended
+	// buckets the null lives at the bare path, NOT under .versions/.
+	backdateMtime(t, fc, bucket, key, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// HEAD without versionId hits the latest, which after the worker
+	// runs should be a delete marker (404 NoSuchKey).
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(key),
+		})
+		return err != nil
+	}, 30*time.Second, 500*time.Millisecond, "delete marker must dominate latest after suspended-versioning expiration")
+
+	// The noncurrent v1 (created under Enabled) must remain — Expiration
+	// in the suspended branch only touches the null version, not other
+	// versions in the .versions/ folder.
+	directHead, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(v1),
+	})
+	require.NoError(t, err, "noncurrent v1 must survive — suspended Expiration only touches the null")
+	require.NotNil(t, directHead)
+
+	// ListObjectVersions must show: v1 still present, a fresh delete
+	// marker as latest, and no more "null" version.
+	listOut, err := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucket), Prefix: aws.String(key),
+	})
+	require.NoError(t, err)
+	var sawV1, sawMarker bool
+	for _, v := range listOut.Versions {
+		if aws.ToString(v.Key) == key {
+			vid := aws.ToString(v.VersionId)
+			if vid == v1 {
+				sawV1 = true
+			}
+			require.NotEqual(t, "null", vid, "null version must have been removed")
+		}
+	}
+	for _, m := range listOut.DeleteMarkers {
+		if aws.ToString(m.Key) == key && aws.ToBool(m.IsLatest) {
+			sawMarker = true
+		}
+	}
+	require.True(t, sawV1, "v1 must still appear in ListObjectVersions")
+	require.True(t, sawMarker, "fresh delete marker must dominate latest")
+}

test/s3/lifecycle/s3_lifecycle_test.go

@@ -0,0 +1,377 @@
+// Package lifecycle is the end-to-end test for the event-driven S3
+// lifecycle worker, driven by the s3.lifecycle.run-shard shell command.
+//
+// The S3 API rejects Expiration.Days < 1, so a literal "wait one day"
+// integration test is unworkable. Instead, the test backdates the target
+// object's mtime via the filer's UpdateEntry RPC: from the engine's
+// perspective the object is past its expiration window the moment the
+// shell command starts, and the dispatcher fires immediately.
+//
+// Variables (set by the Makefile):
+//
+//	WEED_BINARY         - path to the built `weed` binary
+//	S3_ENDPOINT         - http://host:port for the S3 API
+//	FILER_GRPC_ADDRESS  - host:port of the filer's gRPC listener
+package lifecycle
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+const (
+	defaultS3Endpoint     = "http://localhost:8333"
+	defaultS3GrpcEndpoint = "localhost:18333"
+	defaultMasterEndpt    = "http://localhost:9333"
+	defaultAdminEndpoint  = "http://localhost:23646"
+	// Pinned off the FILER_PORT+10000 convention; see Makefile.
+	defaultFilerGRPC      = "localhost:18890"
+	bucketLifecycleXMLKey = "s3-bucket-lifecycle-configuration-xml"
+	bucketsPath           = "/buckets"
+	accessKey             = "some_access_key1"
+	secretKey             = "some_secret_key1"
+	region                = "us-east-1"
+)
+
+func envOr(key, def string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return def
+}
+
+func s3Client(t *testing.T) *s3.Client {
+	t.Helper()
+	endpoint := envOr("S3_ENDPOINT", defaultS3Endpoint)
+	cfg, err := config.LoadDefaultConfig(context.Background(),
+		config.WithRegion(region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
+		config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
+			func(service, region string, _ ...interface{}) (aws.Endpoint, error) {
+				return aws.Endpoint{URL: endpoint, SigningRegion: region, HostnameImmutable: true}, nil
+			})),
+	)
+	require.NoError(t, err)
+	client := s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true })
+	ensureClusterWritable(t, client)
+	return client
+}
+
+var clusterWritableOnce sync.Once
+
+// ensureClusterWritable blocks until the cluster can actually serve a write,
+// absorbing the volume-growth warmup window after a fresh start. The Makefile
+// only waits for the server process to be up ("server up after N s"); it does
+// not wait for a writable volume, so the first PutObject can race volume growth
+// and fail with a transient 500 (assign volume: DeadlineExceeded) — the source
+// of the lifecycle-test flakes. Probing one throwaway write here, once per
+// process, warms growth so every test's first real write is past that window.
+// Best-effort: if it never becomes writable, the test's own PutObject surfaces
+// the failure normally.
+func ensureClusterWritable(t *testing.T, c *s3.Client) {
+	t.Helper()
+	clusterWritableOnce.Do(func() {
+		bucket := uniqueBucket("warmup")
+		deadline := time.Now().Add(60 * time.Second)
+
+		// try runs fn under a bounded context so a single hung call can't block.
+		try := func(timeout time.Duration, fn func(ctx context.Context) error) error {
+			ctx, cancel := context.WithTimeout(context.Background(), timeout)
+			defer cancel()
+			return fn(ctx)
+		}
+		// probe is a try whose timeout is clamped to the time left before the
+		// deadline, so the whole warmup stays within the budget; ok=false means
+		// the budget is exhausted and the caller should stop.
+		probe := func(fn func(ctx context.Context) error) (err error, ok bool) {
+			remaining := time.Until(deadline)
+			if remaining <= 0 {
+				return nil, false
+			}
+			if remaining > 10*time.Second {
+				remaining = 10 * time.Second
+			}
+			return try(remaining, fn), true
+		}
+		// backoff sleeps attempt*250ms, never past the deadline.
+		backoff := func(attempt int) {
+			d := time.Duration(attempt) * 250 * time.Millisecond
+			if left := time.Until(deadline); d > left {
+				d = left
+			}
+			if d > 0 {
+				time.Sleep(d)
+			}
+		}
+
+		// CreateBucket is a metadata op, but on a cold cluster the filer itself may
+		// not be ready yet, so retry it within the deadline rather than abandoning
+		// the whole warmup (and the PutObject probe) on the first error.
+		created := false
+		for attempt := 1; ; attempt++ {
+			err, ok := probe(func(ctx context.Context) error {
+				_, e := c.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)})
+				return e
+			})
+			if !ok {
+				break
+			}
+			if err == nil {
+				created = true
+				break
+			}
+			backoff(attempt)
+		}
+		if !created {
+			t.Logf("warmup: could not create probe bucket within 60s; proceeding")
+			return
+		}
+		// Cleanup gets a fresh timeout (not the warmup budget) so teardown runs
+		// even when the probe loop used the full window.
+		defer try(10*time.Second, func(ctx context.Context) error {
+			_, e := c.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)})
+			return e
+		})
+
+		for attempt := 1; ; attempt++ {
+			err, ok := probe(func(ctx context.Context) error {
+				_, e := c.PutObject(ctx, &s3.PutObjectInput{
+					Bucket: aws.String(bucket), Key: aws.String("warmup"), Body: strings.NewReader("ok"),
+				})
+				return e
+			})
+			if !ok {
+				break
+			}
+			if err == nil {
+				try(10*time.Second, func(ctx context.Context) error {
+					_, e := c.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String("warmup")})
+					return e
+				})
+				if attempt > 1 {
+					t.Logf("cluster became writable after %d probe(s)", attempt)
+				}
+				return
+			}
+			backoff(attempt)
+		}
+		t.Logf("warmup: cluster not confirmed writable within 60s; proceeding")
+	})
+}
+
+func filerClient(t *testing.T) (filer_pb.SeaweedFilerClient, func()) {
+	t.Helper()
+	addr := envOr("FILER_GRPC_ADDRESS", defaultFilerGRPC)
+	conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.NoError(t, err)
+	return filer_pb.NewSeaweedFilerClient(conn), func() { conn.Close() }
+}
+
+// uniqueBucket returns a fresh bucket name; the shell command picks up all
+// buckets with lifecycle config, so two parallel-running tests would conflict
+// on cursor state. Each test gets its own bucket.
+func uniqueBucket(prefix string) string {
+	return fmt.Sprintf("lc-%s-%d", prefix, time.Now().UnixNano())
+}
+
+func mustCreateBucket(t *testing.T, c *s3.Client, name string) {
+	t.Helper()
+	_, err := c.CreateBucket(context.Background(), &s3.CreateBucketInput{Bucket: aws.String(name)})
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		// Drop the lifecycle configuration first so any subsequent
+		// shell-driven shard sweep stops loading rules for this bucket
+		// — without this, a later test's run-shard would pick up the
+		// dead bucket's config and produce phantom dispatches.
+		c.DeleteBucketLifecycle(context.Background(), &s3.DeleteBucketLifecycleInput{Bucket: aws.String(name)})
+
+		// Empty every version + delete marker (versioning-aware buckets
+		// hold state that ListObjectsV2 doesn't surface). Best-effort:
+		// errors are tolerated because the bucket might already be
+		// half-torn-down.
+		listOut, _ := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{Bucket: aws.String(name)})
+		if listOut != nil {
+			for _, v := range listOut.Versions {
+				c.DeleteObject(context.Background(), &s3.DeleteObjectInput{
+					Bucket: aws.String(name), Key: v.Key, VersionId: v.VersionId,
+				})
+			}
+			for _, m := range listOut.DeleteMarkers {
+				c.DeleteObject(context.Background(), &s3.DeleteObjectInput{
+					Bucket: aws.String(name), Key: m.Key, VersionId: m.VersionId,
+				})
+			}
+		}
+
+		// Catch any non-versioned objects ListObjectVersions missed.
+		objs, _ := c.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{Bucket: aws.String(name)})
+		if objs != nil {
+			for _, o := range objs.Contents {
+				c.DeleteObject(context.Background(), &s3.DeleteObjectInput{Bucket: aws.String(name), Key: o.Key})
+			}
+		}
+
+		// Abort any in-flight multipart uploads — the AbortMPU lifecycle
+		// test leaves these intentionally; without an explicit Abort the
+		// bucket DELETE refuses with NotEmpty.
+		mpus, _ := c.ListMultipartUploads(context.Background(), &s3.ListMultipartUploadsInput{Bucket: aws.String(name)})
+		if mpus != nil {
+			for _, u := range mpus.Uploads {
+				c.AbortMultipartUpload(context.Background(), &s3.AbortMultipartUploadInput{
+					Bucket: aws.String(name), Key: u.Key, UploadId: u.UploadId,
+				})
+			}
+		}
+
+		c.DeleteBucket(context.Background(), &s3.DeleteBucketInput{Bucket: aws.String(name)})
+	})
+}
+
+func putExpirationLifecycle(t *testing.T, c *s3.Client, bucket, prefix string, days int32) {
+	t.Helper()
+	_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
+		Bucket: aws.String(bucket),
+		LifecycleConfiguration: &types.BucketLifecycleConfiguration{
+			Rules: []types.LifecycleRule{
+				{
+					ID:     aws.String("expire-prefix"),
+					Status: types.ExpirationStatusEnabled,
+					Filter: &types.LifecycleRuleFilter{Prefix: aws.String(prefix)},
+					Expiration: &types.LifecycleExpiration{
+						Days: aws.Int32(days),
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err)
+}
+
+func putObject(t *testing.T, c *s3.Client, bucket, key, body string) {
+	t.Helper()
+	_, err := c.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+		Body:   strings.NewReader(body),
+	})
+	require.NoError(t, err)
+}
+
+// backdateMtime rewrites the object's filer entry attributes so its Mtime
+// is daysOld days in the past. This sidesteps the AWS-spec minimum of one
+// day for Expiration.Days, letting the lifecycle dispatcher fire on demand.
+func backdateMtime(t *testing.T, fc filer_pb.SeaweedFilerClient, bucket, key string, daysOld int) {
+	t.Helper()
+	dir, name := splitBucketKey(bucket, key)
+	resp, err := fc.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
+		Directory: dir,
+		Name:      name,
+	})
+	require.NoError(t, err, "lookup %s/%s", dir, name)
+	require.NotNil(t, resp.Entry)
+	require.NotNil(t, resp.Entry.Attributes)
+
+	resp.Entry.Attributes.Mtime = time.Now().Add(-time.Duration(daysOld) * 24 * time.Hour).Unix()
+	resp.Entry.Attributes.MtimeNs = 0
+	_, err = fc.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
+		Directory: dir,
+		Entry:     resp.Entry,
+	})
+	require.NoError(t, err, "update %s/%s", dir, name)
+}
+
+func splitBucketKey(bucket, key string) (dir, name string) {
+	full := bucketsPath + "/" + bucket + "/" + key
+	if i := strings.LastIndex(full, "/"); i >= 0 {
+		return full[:i], full[i+1:]
+	}
+	return full, ""
+}
+
+// runShellCommand invokes `weed shell` with a one-shot s3.lifecycle.run-shard
+// command piped via stdin and returns the combined stdout+stderr.
+func runShellCommand(t *testing.T, command string) string {
+	t.Helper()
+	binary := envOr("WEED_BINARY", "")
+	require.NotEmpty(t, binary, "WEED_BINARY must be set")
+	masterEndpoint := envOr("MASTER_ENDPOINT", defaultMasterEndpt)
+	master := strings.TrimPrefix(masterEndpoint, "http://")
+	if u, err := url.Parse(masterEndpoint); err == nil && u.Host != "" {
+		master = u.Host
+	}
+	cmd := exec.Command(binary, "shell", "-master="+master)
+	cmd.Stdin = strings.NewReader(command + "\nexit\n")
+	var out bytes.Buffer
+	cmd.Stdout, cmd.Stderr = &out, &out
+	err := cmd.Run()
+	output := out.String()
+	if err != nil {
+		t.Logf("shell command output:\n%s", output)
+		t.Fatalf("shell exec failed: %v", err)
+	}
+	return output
+}
+
+// TestLifecycleExpirationFiresOnBackdatedObject is the end-to-end
+// validation: a 1-day expiration rule plus an object whose mtime has been
+// backdated to 30 days ago must result in deletion when the shell command
+// runs the matching shard.
+func TestLifecycleExpirationFiresOnBackdatedObject(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("expire")
+	mustCreateBucket(t, c, bucket)
+	putExpirationLifecycle(t, c, bucket, "expire/", 1)
+
+	const oldKey = "expire/old.txt"
+	const freshKey = "keep/fresh.txt"
+	putObject(t, c, bucket, oldKey, "old")
+	putObject(t, c, bucket, freshKey, "fresh")
+	backdateMtime(t, fc, bucket, oldKey, 30)
+
+	// One subscription handles every shard via -shards 0-15; this stays
+	// independent of which (bucket, key) hash lands the target on.
+	// -runtime caps the run by wall-clock so the subprocess exits even
+	// when the in-shard event count never reaches -events.
+	out := runShellCommand(t, fmt.Sprintf(
+		"s3.lifecycle.run-shard -shards 0-15 -s3 %s -events 0 -dispatch 200ms -checkpoint 5s -runtime 10s",
+		envOr("S3_GRPC_ENDPOINT", defaultS3GrpcEndpoint),
+	))
+	t.Logf("shell command output:\n%s", out)
+	require.NotContains(t, out, "FATAL", "shell output:\n%s", out)
+
+	// Allow the dispatched delete to land in the filer + propagate to S3.
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(oldKey),
+		})
+		return err != nil
+	}, 30*time.Second, 500*time.Millisecond, "expected %s/%s to be deleted", bucket, oldKey)
+
+	// The fresh, in-prefix-but-recent object must remain (1d rule, mtime
+	// is now). Best-evidence check after waiting for the same poll window.
+	_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(freshKey),
+	})
+	require.NoError(t, err, "%s should still exist", freshKey)
+}

test/s3/lifecycle/s3_lifecycle_versioning_test.go

@@ -0,0 +1,456 @@
+// Versioning + delete-marker + noncurrent integration scenarios for the
+// event-driven lifecycle worker. Same backdating trick as the base test:
+// AWS rejects Days < 1, so the test ages the object via the filer's
+// UpdateEntry RPC and runs the shell command once with -runtime 10s.
+package lifecycle
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/stretchr/testify/require"
+)
+
+// putVersioningEnabled flips Status=Enabled on the bucket so subsequent
+// PUTs produce versioned siblings rather than overwriting.
+func putVersioningEnabled(t *testing.T, c *s3.Client, bucket string) {
+	t.Helper()
+	_, err := c.PutBucketVersioning(context.Background(), &s3.PutBucketVersioningInput{
+		Bucket:                  aws.String(bucket),
+		VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusEnabled},
+	})
+	require.NoError(t, err)
+}
+
+// putNoncurrentExpirationLifecycle sets a NoncurrentVersionExpiration rule
+// targeting a prefix. Days=1 because AWS rejects 0; the test backdates
+// the noncurrent version.
+func putNoncurrentExpirationLifecycle(t *testing.T, c *s3.Client, bucket, prefix string, days int32) {
+	t.Helper()
+	_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
+		Bucket: aws.String(bucket),
+		LifecycleConfiguration: &types.BucketLifecycleConfiguration{
+			Rules: []types.LifecycleRule{
+				{
+					ID:     aws.String("expire-noncurrent"),
+					Status: types.ExpirationStatusEnabled,
+					Filter: &types.LifecycleRuleFilter{Prefix: aws.String(prefix)},
+					NoncurrentVersionExpiration: &types.NoncurrentVersionExpiration{
+						NoncurrentDays: aws.Int32(days),
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err)
+}
+
+// backdateVersionedMtime ages a specific .versions/v_<id> entry. The
+// version files live under <buckets>/<bucket>/<key>.versions/v_<versionId>.
+//
+// Also clears the ExtNoncurrentSinceNsKey stamp if present. The stamp
+// records when this version was demoted by its successor's PUT; tests
+// that backdate a version's own mtime aren't expressing a coherent claim
+// about the demotion moment, so leaving a real-now stamp would let it
+// dominate the backdated mtime and stop the rule from firing. Clearing
+// makes the lifecycle engine fall back to the sibling-mtime derivation,
+// which is what these tests were originally written against. Production
+// PUTs always write a stamp; this is a test-only escape hatch.
+func backdateVersionedMtime(t *testing.T, fc filer_pb.SeaweedFilerClient, bucket, key, versionID string, daysOld int) {
+	t.Helper()
+	dir := bucketsPath + "/" + bucket + "/" + key + ".versions"
+	name := "v_" + versionID
+	resp, err := fc.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
+		Directory: dir, Name: name,
+	})
+	require.NoError(t, err, "lookup version %s/%s", dir, name)
+	require.NotNil(t, resp.Entry)
+	require.NotNil(t, resp.Entry.Attributes)
+
+	resp.Entry.Attributes.Mtime = time.Now().Add(-time.Duration(daysOld) * 24 * time.Hour).Unix()
+	resp.Entry.Attributes.MtimeNs = 0
+	delete(resp.Entry.Extended, s3_constants.ExtNoncurrentSinceNsKey)
+	_, err = fc.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
+		Directory: dir, Entry: resp.Entry,
+	})
+	require.NoError(t, err)
+}
+
+func runLifecycleShard(t *testing.T) string {
+	t.Helper()
+	// -refresh 1s drives an inter-pass loop inside the 10s runtime so
+	// rules that need two passes to settle (e.g.
+	// TestLifecycleExpiredDeleteMarkerCleanup: pass 1 removes v1, pass 2
+	// removes the now-orphaned marker) get the second pass.
+	out := runShellCommand(t, fmt.Sprintf(
+		"s3.lifecycle.run-shard -shards 0-15 -s3 %s -events 0 -refresh 1s -runtime 10s",
+		envOr("S3_GRPC_ENDPOINT", defaultS3GrpcEndpoint),
+	))
+	require.NotContains(t, out, "FATAL", "shell output:\n%s", out)
+	return out
+}
+
+// TestLifecycleVersionedBucketCreatesDeleteMarker: an Expiration rule on a
+// versioned bucket must produce a delete marker (latest version after
+// expiration) rather than removing the version itself. Pre-fix this would
+// have unconditionally deleted the live version.
+func TestLifecycleVersionedBucketCreatesDeleteMarker(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("vexp")
+	mustCreateBucket(t, c, bucket)
+	putVersioningEnabled(t, c, bucket)
+	putExpirationLifecycle(t, c, bucket, "expire/", 1)
+
+	const key = "expire/old.txt"
+	putObject(t, c, bucket, key, "v1")
+
+	// Look up the version we just created so we can backdate it. The bare
+	// filer entry under <bucket>/<key> is the latest-version pointer in
+	// SeaweedFS's versioned layout.
+	headOut, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key),
+	})
+	require.NoError(t, err)
+	require.NotNil(t, headOut.VersionId)
+	versionID := aws.ToString(headOut.VersionId)
+	require.NotEmpty(t, versionID)
+	backdateVersionedMtime(t, fc, bucket, key, versionID, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// HEAD without versionId returns the latest version. After the worker
+	// runs, the latest must be a delete marker (NoSuchKey on plain HEAD).
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(key),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "expected delete marker to become latest for %s/%s", bucket, key)
+
+	// The original version must still be addressable directly — Expiration
+	// on a versioned bucket creates a marker, it doesn't drop the version.
+	directHead, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(versionID),
+	})
+	require.NoError(t, err, "original version must still exist")
+	require.NotNil(t, directHead)
+
+	// ListObjectVersions should now show a delete marker dominating the
+	// version. The IsLatest=true marker proves the worker created a fresh
+	// marker rather than aging the existing version.
+	listOut, err := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucket), Prefix: aws.String(key),
+	})
+	require.NoError(t, err)
+	var sawMarker bool
+	for _, m := range listOut.DeleteMarkers {
+		if aws.ToString(m.Key) == key && aws.ToBool(m.IsLatest) {
+			sawMarker = true
+			break
+		}
+	}
+	require.True(t, sawMarker, "ListObjectVersions must show a delete marker as latest for %s", key)
+}
+
+// TestLifecycleNoncurrentVersionExpiration: NoncurrentVersionExpiration
+// only fires on noncurrent versions. PUT v1, PUT v2 (so v1 → noncurrent),
+// backdate v1, run worker. v1 must be removed; v2 stays current.
+func TestLifecycleNoncurrentVersionExpiration(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("noncurrent")
+	mustCreateBucket(t, c, bucket)
+	putVersioningEnabled(t, c, bucket)
+	putNoncurrentExpirationLifecycle(t, c, bucket, "v/", 1)
+
+	const key = "v/obj.txt"
+	// First PUT.
+	put1, err := c.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key), Body: strings.NewReader("v1"),
+	})
+	require.NoError(t, err)
+	v1 := aws.ToString(put1.VersionId)
+
+	// Second PUT promotes v1 to noncurrent.
+	put2, err := c.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key), Body: strings.NewReader("v2"),
+	})
+	require.NoError(t, err)
+	v2 := aws.ToString(put2.VersionId)
+	require.NotEqual(t, v1, v2)
+
+	// NoncurrentDays is clocked from the SUCCESSOR's mtime — i.e. when
+	// the version became noncurrent — not from the displaced version's
+	// own mtime. Backdating only v1 leaves the noncurrent clock (v2's
+	// mtime) at "now" and the rule never fires. Age both with v1 older
+	// than v2 so the ordering is realistic.
+	backdateVersionedMtime(t, fc, bucket, key, v1, 31)
+	backdateVersionedMtime(t, fc, bucket, key, v2, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// v1 must be gone.
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(v1),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "noncurrent v1 must be expired")
+
+	// v2 must still be addressable BY VERSION ID — pinning that the
+	// noncurrent dispatch didn't accidentally remove the current
+	// version's file.
+	directHead, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(v2),
+	})
+	require.NoError(t, err, "current version v2 (versionId=%s) must still be addressable directly", v2)
+
+	// And HEAD without versionId (i.e., the latest) must hit v2 — pinning
+	// that the bare-key pointer wasn't removed and a delete marker
+	// wasn't fabricated as a side effect of the noncurrent delete. On
+	// failure we dump ListObjectVersions so the failure log shows
+	// exactly what's there (versions vs delete markers vs bare).
+	currentHead, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key),
+	})
+	if err != nil {
+		listOut, listErr := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{
+			Bucket: aws.String(bucket), Prefix: aws.String(key),
+		})
+		if listErr != nil || listOut == nil {
+			t.Logf("HeadObject(latest) failed: %v\nListObjectVersions err=%v", err, listErr)
+		} else {
+			t.Logf("HeadObject(latest) failed: %v\nListObjectVersions err=%v versions=%d markers=%d", err, listErr, len(listOut.Versions), len(listOut.DeleteMarkers))
+			for _, v := range listOut.Versions {
+				t.Logf("  version key=%s id=%s isLatest=%v", aws.ToString(v.Key), aws.ToString(v.VersionId), aws.ToBool(v.IsLatest))
+			}
+			for _, m := range listOut.DeleteMarkers {
+				t.Logf("  marker  key=%s id=%s isLatest=%v", aws.ToString(m.Key), aws.ToString(m.VersionId), aws.ToBool(m.IsLatest))
+			}
+		}
+		require.NoError(t, err, "HEAD(latest) on %s/%s must succeed; v2 (%s) must remain current", bucket, key, v2)
+	}
+	require.Equal(t, v2, aws.ToString(currentHead.VersionId), "latest pointer must still resolve to v2")
+	_ = directHead
+}
+
+// TestLifecycleExpiredDeleteMarkerCleanup: a sole-survivor delete marker
+// must be removed by an ExpiredObjectDeleteMarker=true rule. Setup: PUT v1,
+// DELETE (creates a marker that becomes latest, with v1 as noncurrent),
+// expire v1 via NoncurrentVersionExpiration so the marker is alone, then
+// the marker rule fires.
+//
+// We exercise this in two passes: pass 1 (Noncurrent rule on a sibling
+// prefix) cleans v1; pass 2 (ExpiredObjectDeleteMarker rule, also on the
+// same prefix) removes the now-orphaned marker. Since the lifecycle XML
+// only carries one set of rules per bucket, we wire both.
+func TestLifecycleExpiredDeleteMarkerCleanup(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("marker")
+	mustCreateBucket(t, c, bucket)
+	putVersioningEnabled(t, c, bucket)
+
+	// Combined rule: noncurrent expiration + delete-marker cleanup, both
+	// on the same prefix.
+	_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
+		Bucket: aws.String(bucket),
+		LifecycleConfiguration: &types.BucketLifecycleConfiguration{
+			Rules: []types.LifecycleRule{
+				{
+					ID:     aws.String("expire-noncurrent"),
+					Status: types.ExpirationStatusEnabled,
+					Filter: &types.LifecycleRuleFilter{Prefix: aws.String("m/")},
+					NoncurrentVersionExpiration: &types.NoncurrentVersionExpiration{
+						NoncurrentDays: aws.Int32(1),
+					},
+				},
+				{
+					ID:     aws.String("expire-marker"),
+					Status: types.ExpirationStatusEnabled,
+					Filter: &types.LifecycleRuleFilter{Prefix: aws.String("m/")},
+					Expiration: &types.LifecycleExpiration{
+						ExpiredObjectDeleteMarker: aws.Bool(true),
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err)
+
+	const key = "m/obj.txt"
+	put1, err := c.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key), Body: strings.NewReader("v1"),
+	})
+	require.NoError(t, err)
+	v1 := aws.ToString(put1.VersionId)
+
+	// Plain DELETE with versioning on creates a marker; the marker becomes
+	// latest, v1 demotes to noncurrent.
+	_, err = c.DeleteObject(context.Background(), &s3.DeleteObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key),
+	})
+	require.NoError(t, err)
+
+	// Backdate v1 so the noncurrent rule expires it.
+	backdateVersionedMtime(t, fc, bucket, key, v1, 30)
+
+	// Backdate the delete marker too. ListObjectVersions tells us the
+	// marker's versionId.
+	listOut, err := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucket), Prefix: aws.String(key),
+	})
+	require.NoError(t, err)
+	var markerID string
+	for _, m := range listOut.DeleteMarkers {
+		if aws.ToString(m.Key) == key {
+			markerID = aws.ToString(m.VersionId)
+			break
+		}
+	}
+	require.NotEmpty(t, markerID, "expected to find the delete marker we just created")
+	backdateVersionedMtime(t, fc, bucket, key, markerID, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// Both v1 and the marker must be gone — the .versions/<key>/ folder
+	// should hold nothing for this key.
+	require.Eventuallyf(t, func() bool {
+		listOut, err := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{
+			Bucket: aws.String(bucket), Prefix: aws.String(key),
+		})
+		if err != nil {
+			return false
+		}
+		for _, v := range listOut.Versions {
+			if aws.ToString(v.Key) == key {
+				return false
+			}
+		}
+		for _, m := range listOut.DeleteMarkers {
+			if aws.ToString(m.Key) == key {
+				return false
+			}
+		}
+		return true
+	}, 30*time.Second, 500*time.Millisecond, "every version and marker for %s must be gone", key)
+}
+
+// TestLifecycleDisabledRuleSkipsObject: a rule with Status=Disabled must
+// not produce dispatches, even on a backdated object that would otherwise
+// match. Negative test for the engine's enabled-status gate.
+func TestLifecycleDisabledRuleSkipsObject(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("disabled")
+	mustCreateBucket(t, c, bucket)
+	_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
+		Bucket: aws.String(bucket),
+		LifecycleConfiguration: &types.BucketLifecycleConfiguration{
+			Rules: []types.LifecycleRule{{
+				ID:         aws.String("disabled-rule"),
+				Status:     types.ExpirationStatusDisabled,
+				Filter:     &types.LifecycleRuleFilter{Prefix: aws.String("d/")},
+				Expiration: &types.LifecycleExpiration{Days: aws.Int32(1)},
+			}},
+		},
+	})
+	require.NoError(t, err)
+
+	const key = "d/obj.txt"
+	putObject(t, c, bucket, key, "still-here")
+	backdateMtime(t, fc, bucket, key, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// Wait the same window; the object MUST still exist after.
+	time.Sleep(2 * time.Second) // give the worker a chance to (incorrectly) act
+	_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key),
+	})
+	require.NoError(t, err, "Disabled rule must not delete the object")
+}
+
+// TestLifecycleTagFilter: a rule with a tag filter must only match objects
+// carrying that tag. Two backdated objects, one tagged, one not — only the
+// tagged one is removed.
+func TestLifecycleTagFilter(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("tagfilter")
+	mustCreateBucket(t, c, bucket)
+	_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
+		Bucket: aws.String(bucket),
+		LifecycleConfiguration: &types.BucketLifecycleConfiguration{
+			Rules: []types.LifecycleRule{{
+				ID:     aws.String("tag-only"),
+				Status: types.ExpirationStatusEnabled,
+				Filter: &types.LifecycleRuleFilter{
+					And: &types.LifecycleRuleAndOperator{
+						Prefix: aws.String("t/"),
+						Tags: []types.Tag{
+							{Key: aws.String("env"), Value: aws.String("temp")},
+						},
+					},
+				},
+				Expiration: &types.LifecycleExpiration{Days: aws.Int32(1)},
+			}},
+		},
+	})
+	require.NoError(t, err)
+
+	const taggedKey = "t/tagged.txt"
+	const untaggedKey = "t/untagged.txt"
+	putObject(t, c, bucket, taggedKey, "tagged")
+	putObject(t, c, bucket, untaggedKey, "untagged")
+
+	// Stamp the tag on one of them.
+	_, err = c.PutObjectTagging(context.Background(), &s3.PutObjectTaggingInput{
+		Bucket: aws.String(bucket), Key: aws.String(taggedKey),
+		Tagging: &types.Tagging{TagSet: []types.Tag{
+			{Key: aws.String("env"), Value: aws.String("temp")},
+		}},
+	})
+	require.NoError(t, err)
+
+	backdateMtime(t, fc, bucket, taggedKey, 30)
+	backdateMtime(t, fc, bucket, untaggedKey, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(taggedKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "tagged object must be expired")
+
+	_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(untaggedKey),
+	})
+	require.NoError(t, err, "untagged object must remain")
+}