snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,33 @@
package bootstrap
import (
"testing"
"github.com/stretchr/testify/assert"
)
// HasPrefix is a thin wrapper around strings.HasPrefix exported by the
// bootstrap package so call sites can avoid pulling strings just to do
// a prefix check. Trivial but at 0% coverage — pin it.
func TestHasPrefix(t *testing.T) {
cases := []struct {
name string
path string
prefix string
want bool
}{
{"matching prefix", "logs/2026/01/01", "logs/", true},
{"exact match", "logs/", "logs/", true},
{"non-matching prefix", "metrics/x", "logs/", false},
{"shorter than prefix", "logs", "logs/", false},
{"empty prefix matches all", "anything", "", true},
{"both empty", "", "", true},
{"empty input non-empty prefix", "", "x", false},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
assert.Equal(t, c.want, HasPrefix(c.path, c.prefix))
})
}
}

View File

@@ -0,0 +1,197 @@
// Package bootstrap is the bucket-level lifecycle walker. The walker
// iterates every entry under a bucket, evaluates every active ActionKey
// against it, and dispatches inline-delete RPCs for currently-due actions;
// not-yet-due entries are left for the meta-log reader to pick up later.
//
// Callback-driven so the listing source and the LifecycleDelete dispatcher
// can be supplied separately (real client or test fake).
package bootstrap
import (
"context"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
"github.com/seaweedfs/seaweedfs/weed/stats"
)
// Entry is the routing-relevant slice of a filer entry. SuccessorModTime
// and NoncurrentIndex are populated only on versioned-bucket walks; the
// retention path bails out conservatively when they're zero / nil.
//
// MPU init directories at .uploads/<id> populate DestKey with the
// destination object key (from entry.Extended[ExtMultipartObjectKey]) so
// rule-prefix matching works on the user's intended path while Path
// stays as the upload directory the dispatcher must rm.
type Entry struct {
Path string
DestKey string
ModTime time.Time
Size int64
IsDirectory bool
IsLatest bool
IsDeleteMarker bool
IsMPUInit bool
NumVersions int
Tags map[string]string
SuccessorModTime time.Time
NoncurrentIndex *int
// VersionID is the S3 version id of this entry, empty for
// non-versioned buckets. Populated by the ListFunc adapter when
// the entry came from a `.versions/` directory; the walker itself
// doesn't use it, but the Dispatcher needs it to address the
// right version on LifecycleDelete.
VersionID string
}
// ListFunc must skip entries with Path <= start so kill-resume picks up
// where the previous run stopped.
type ListFunc func(ctx context.Context, bucket, start string, cb func(*Entry) error) error
// Dispatcher executes one (action, entry) verdict. An error halts the walk;
// the caller decides whether to retry from the recorded last_scanned_path.
type Dispatcher interface {
Delete(ctx context.Context, action *engine.CompiledAction, entry *Entry) error
}
// Checkpoint is the resume state. Caller persists it under
// /etc/s3/lifecycle/<bucket>/_bootstrap.
type Checkpoint struct {
LastScannedPath string
Completed bool
}
type WalkOptions struct {
Resume string
Now time.Time
}
// Walk iterates entries via list, evaluates each active ActionKey via
// MatchPath + EvaluateAction, and calls Dispatcher.Delete for currently-due
// actions. SCAN_AT_DATE actions are skipped (their bootstrap is scheduled
// separately).
func Walk(ctx context.Context, snap *engine.Snapshot, bucket string, list ListFunc, dispatch Dispatcher, opts WalkOptions) (Checkpoint, error) {
now := opts.Now
if now.IsZero() {
now = time.Now().UTC()
}
cp := Checkpoint{LastScannedPath: opts.Resume}
// Reuse one ObjectInfo across the walk; EvaluateAction reads it
// synchronously without retaining.
var info s3lifecycle.ObjectInfo
err := list(ctx, bucket, opts.Resume, func(entry *Entry) error {
if entry == nil || entry.Path == "" {
return nil
}
// MPU init directories at .uploads/<id> are the one directory
// shape lifecycle cares about; everything else stays out.
if entry.IsDirectory && !entry.IsMPUInit {
cp.LastScannedPath = entry.Path
return nil
}
if err := walkEntry(ctx, snap, bucket, entry, dispatch, now, &info); err != nil {
return err
}
cp.LastScannedPath = entry.Path
return nil
})
if err != nil {
return cp, err
}
cp.Completed = true
return cp, nil
}
func walkEntry(ctx context.Context, snap *engine.Snapshot, bucket string, entry *Entry, dispatch Dispatcher, now time.Time, info *s3lifecycle.ObjectInfo) error {
// MPU init: rule-prefix matching uses the destination key, not the
// .uploads/<id> directory path. A bare directory with no DestKey is
// either a stray dir or an init mid-write before metadata landed —
// skip rather than guess.
matchKey := entry.Path
if entry.IsMPUInit {
if entry.DestKey == "" {
return nil
}
matchKey = entry.DestKey
}
keys := snap.MatchPath(bucket, matchKey, nil)
if len(keys) == 0 {
return nil
}
*info = s3lifecycle.ObjectInfo{
Key: matchKey,
ModTime: entry.ModTime,
Size: entry.Size,
IsLatest: entry.IsLatest,
IsDeleteMarker: entry.IsDeleteMarker,
IsMPUInit: entry.IsMPUInit,
NumVersions: entry.NumVersions,
SuccessorModTime: entry.SuccessorModTime,
NoncurrentIndex: entry.NoncurrentIndex,
Tags: entry.Tags,
}
for _, key := range keys {
action := snap.Action(key)
if action == nil {
continue
}
// DISABLED can be flipped at runtime independent of XML Status,
// so skip it even though EvaluateAction would also reject.
// SCAN_AT_DATE actions are processed here too — the date check
// in EvaluateAction (now.Before(rule.ExpirationDate)) gates the
// dispatch, so pre-date walks are no-ops and post-date walks
// expire eligible objects. The earlier "scan-at-date runs its
// own bootstrap" plan was never wired; until that lands, the
// regular bootstrap walk is the only path that fires
// ExpirationDate rules.
if action.Mode == engine.ModeDisabled {
continue
}
// (kind, info) shape gate: ABORT_MPU only on MPU init records,
// every other kind only on regular objects/versions. Mismatched
// pairs would either dispatch a noncurrent action with empty
// version_id (server BLOCKs, cursor freezes) or dispatch
// ABORT_MPU against a regular object path.
if entry.IsMPUInit && key.ActionKind != s3lifecycle.ActionKindAbortMPU {
continue
}
if !entry.IsMPUInit && key.ActionKind == s3lifecycle.ActionKindAbortMPU {
continue
}
res := s3lifecycle.EvaluateAction(action.Rule, key.ActionKind, info, now)
if res.Action == s3lifecycle.ActionNone {
continue
}
if err := dispatch.Delete(ctx, action, entry); err != nil {
glog.Warningf("lifecycle bootstrap: dispatch %s/%s kind=%s: %v",
bucket, entry.Path, key.ActionKind, err)
return err
}
stats.S3LifecycleBootstrapDispatchCounter.WithLabelValues(bucket, key.ActionKind.String()).Inc()
}
return nil
}
// EntryCallback wraps an in-memory slice as a ListFunc; useful for tests.
func EntryCallback(entries []*Entry) ListFunc {
return func(ctx context.Context, bucket, start string, cb func(*Entry) error) error {
for _, e := range entries {
if start != "" && e.Path <= start {
continue
}
if err := cb(e); err != nil {
return err
}
}
return nil
}
}
func HasPrefix(path, prefix string) bool { return strings.HasPrefix(path, prefix) }

View File

@@ -0,0 +1,431 @@
package bootstrap
import (
"context"
"errors"
"reflect"
"sort"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
)
// recorder captures dispatched (action, entry) pairs for assertion.
type recorder struct {
calls []dispatchCall
err error // when set, every Delete returns this error
}
type dispatchCall struct {
kind s3lifecycle.ActionKind
path string
}
func (r *recorder) Delete(ctx context.Context, action *engine.CompiledAction, entry *Entry) error {
if r.err != nil {
return r.err
}
r.calls = append(r.calls, dispatchCall{kind: action.Key.ActionKind, path: entry.Path})
return nil
}
func mustTime(t *testing.T, s string) time.Time {
t.Helper()
tm, err := time.Parse(time.RFC3339, s)
if err != nil {
t.Fatalf("parse %s: %v", s, err)
}
return tm
}
func compileEvDriven(t *testing.T, bucket string, rules ...*s3lifecycle.Rule) *engine.Snapshot {
t.Helper()
prior := map[s3lifecycle.ActionKey]engine.PriorState{}
for _, r := range rules {
rh := s3lifecycle.RuleHash(r)
for _, k := range s3lifecycle.RuleActionKinds(r) {
prior[s3lifecycle.ActionKey{Bucket: bucket, RuleHash: rh, ActionKind: k}] = engine.PriorState{BootstrapComplete: true}
}
}
e := engine.New()
return e.Compile([]engine.CompileInput{{Bucket: bucket, Rules: rules}}, engine.CompileOptions{PriorStates: prior})
}
func TestWalk_DispatchesDueActions(t *testing.T) {
rule := &s3lifecycle.Rule{
ID: "r",
Status: s3lifecycle.StatusEnabled,
ExpirationDays: 30,
Prefix: "logs/",
}
snap := compileEvDriven(t, "bk", rule)
mod := mustTime(t, "2024-01-01T00:00:00Z")
now := mod.AddDate(0, 0, 60) // past the 30d threshold
entries := []*Entry{
{Path: "data/x", IsLatest: true, ModTime: mod}, // wrong prefix
{Path: "logs/a", IsLatest: true, ModTime: mod}, // due
{Path: "logs/b", IsLatest: true, ModTime: now}, // not yet due (mod=now)
}
rec := &recorder{}
cp, err := Walk(context.Background(), snap, "bk", EntryCallback(entries), rec, WalkOptions{Now: now})
if err != nil {
t.Fatalf("Walk: %v", err)
}
if !cp.Completed {
t.Fatalf("walk should complete")
}
if cp.LastScannedPath != "logs/b" {
t.Fatalf("checkpoint last scanned want logs/b, got %q", cp.LastScannedPath)
}
if len(rec.calls) != 1 || rec.calls[0].path != "logs/a" {
t.Fatalf("dispatched calls want [logs/a], got %v", rec.calls)
}
}
func TestWalk_MultiActionRule_AllDueDispatched(t *testing.T) {
// One rule with three actions; all currently-due for the entry. The
// walker must dispatch one Delete per action — this is the
// regression that the per-action keying fixes.
rule := &s3lifecycle.Rule{
ID: "multi",
Status: s3lifecycle.StatusEnabled,
ExpirationDays: 30,
NoncurrentVersionExpirationDays: 7,
AbortMPUDaysAfterInitiation: 5,
}
snap := compileEvDriven(t, "bk", rule)
mod := mustTime(t, "2024-01-01T00:00:00Z")
now := mod.AddDate(0, 0, 100) // past every threshold
// Use a single entry that satisfies all three action shapes is
// unrealistic; in practice each shape is a different entry. Cover
// each shape independently.
entries := []*Entry{
// Current version under ExpirationDays.
{Path: "obj/a", IsLatest: true, ModTime: mod},
// Non-current version under NoncurrentDays.
{Path: "obj/a/.versions/v1", IsLatest: false, ModTime: mod, SuccessorModTime: mod},
// MPU init under AbortMPU. Real init is a directory; DestKey
// carries the eventual object key for prefix matching.
{Path: ".uploads/u1", IsDirectory: true, IsMPUInit: true, DestKey: "obj/a", ModTime: mod},
}
rec := &recorder{}
if _, err := Walk(context.Background(), snap, "bk", EntryCallback(entries), rec, WalkOptions{Now: now}); err != nil {
t.Fatalf("Walk: %v", err)
}
// Exact-shape assertion: each entry dispatches exactly one action,
// and ABORT_MPU only fires on the .uploads/<id> entry. A weaker
// "kinds-as-set" check would have missed the (kind, info) gating
// regression where an MPU init also fired NONCURRENT_DAYS.
want := []dispatchCall{
{kind: s3lifecycle.ActionKindAbortMPU, path: ".uploads/u1"},
{kind: s3lifecycle.ActionKindExpirationDays, path: "obj/a"},
{kind: s3lifecycle.ActionKindNoncurrentDays, path: "obj/a/.versions/v1"},
}
got := append([]dispatchCall(nil), rec.calls...)
sort.Slice(got, func(i, j int) bool {
if got[i].path != got[j].path {
return got[i].path < got[j].path
}
return got[i].kind < got[j].kind
})
if !reflect.DeepEqual(got, want) {
t.Fatalf("dispatch calls mismatch:\n got %+v\nwant %+v", got, want)
}
}
func TestWalk_NotYetDueSkipped(t *testing.T) {
// The reader (Phase 3) is responsible for not-yet-due entries; the
// walker dispatches only currently-due ones, so the meta-log path
// stays the steady-state route.
rule := &s3lifecycle.Rule{
ID: "r",
Status: s3lifecycle.StatusEnabled,
ExpirationDays: 30,
}
snap := compileEvDriven(t, "bk", rule)
mod := mustTime(t, "2024-01-01T00:00:00Z")
now := mod.Add(s3lifecycle.DaysToDuration(10)) // before the 30d threshold
rec := &recorder{}
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
{Path: "x/a", IsLatest: true, ModTime: mod},
}), rec, WalkOptions{Now: now}); err != nil {
t.Fatalf("Walk: %v", err)
}
if len(rec.calls) != 0 {
t.Fatalf("not-yet-due entry should not dispatch, got %v", rec.calls)
}
}
func TestWalk_DateActionFiresAfterDate(t *testing.T) {
// The dedicated SCAN_AT_DATE bootstrap was never wired; until it
// lands, the regular bootstrap walker is the only path that fires
// ExpirationDate rules. Entries past the rule's date must dispatch.
date := mustTime(t, "2025-06-15T00:00:00Z")
rule := &s3lifecycle.Rule{
ID: "d",
Status: s3lifecycle.StatusEnabled,
ExpirationDate: date,
}
snap := compileEvDriven(t, "bk", rule)
rec := &recorder{}
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
{Path: "x/a", IsLatest: true, ModTime: mustTime(t, "2024-01-01T00:00:00Z")},
}), rec, WalkOptions{Now: date.AddDate(0, 1, 0)}); err != nil {
t.Fatalf("Walk: %v", err)
}
if len(rec.calls) != 1 || rec.calls[0].path != "x/a" {
t.Fatalf("date kind past the rule date must dispatch, got %v", rec.calls)
}
}
func TestWalk_DateActionSkippedBeforeDate(t *testing.T) {
// Pre-date walks are no-ops: EvaluateAction returns ActionNone when
// now is before rule.ExpirationDate.
date := mustTime(t, "2025-06-15T00:00:00Z")
rule := &s3lifecycle.Rule{
ID: "d",
Status: s3lifecycle.StatusEnabled,
ExpirationDate: date,
}
snap := compileEvDriven(t, "bk", rule)
rec := &recorder{}
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
{Path: "x/a", IsLatest: true, ModTime: mustTime(t, "2024-01-01T00:00:00Z")},
}), rec, WalkOptions{Now: date.AddDate(0, -1, 0)}); err != nil {
t.Fatalf("Walk: %v", err)
}
if len(rec.calls) != 0 {
t.Fatalf("pre-date walk must not dispatch, got %v", rec.calls)
}
}
func TestWalk_DirectoryEntriesSkipped(t *testing.T) {
// SeaweedFS directory entries can co-exist in the listing; the walker
// must not dispatch deletes against them even when their path matches.
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
snap := compileEvDriven(t, "bk", rule)
mod := mustTime(t, "2024-01-01T00:00:00Z")
now := mod.AddDate(0, 0, 10)
rec := &recorder{}
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
{Path: "x", IsDirectory: true, ModTime: mod}, // directory; must skip
{Path: "x/file", IsLatest: true, ModTime: mod},
}), rec, WalkOptions{Now: now}); err != nil {
t.Fatalf("Walk: %v", err)
}
if len(rec.calls) != 1 || rec.calls[0].path != "x/file" {
t.Fatalf("only the file should dispatch, got %v", rec.calls)
}
}
func TestWalk_DisabledModeSkipped(t *testing.T) {
// An operator-flipped ModeDisabled must short-circuit the walker even
// when the XML rule status is "Enabled" and EvaluateAction would
// otherwise fire.
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
rh := s3lifecycle.RuleHash(rule)
prior := map[s3lifecycle.ActionKey]engine.PriorState{
{Bucket: "bk", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}: {
BootstrapComplete: true, Mode: engine.ModeDisabled,
},
}
e := engine.New()
snap := e.Compile([]engine.CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}}, engine.CompileOptions{PriorStates: prior})
mod := mustTime(t, "2024-01-01T00:00:00Z")
now := mod.AddDate(0, 0, 10)
rec := &recorder{}
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
{Path: "x/a", IsLatest: true, ModTime: mod},
}), rec, WalkOptions{Now: now}); err != nil {
t.Fatalf("Walk: %v", err)
}
if len(rec.calls) != 0 {
t.Fatalf("disabled action must not dispatch, got %v", rec.calls)
}
}
func TestWalk_PendingBootstrapNotDispatched(t *testing.T) {
// Without bootstrap_complete=true in PriorStates, the engine compiles
// the action as inactive. MatchPath filters on IsActive, so the
// walker won't dispatch.
rule := &s3lifecycle.Rule{
ID: "r",
Status: s3lifecycle.StatusEnabled,
ExpirationDays: 1,
}
e := engine.New()
snap := e.Compile([]engine.CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}}, engine.CompileOptions{})
mod := mustTime(t, "2024-01-01T00:00:00Z")
now := mod.AddDate(0, 0, 10)
rec := &recorder{}
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
{Path: "x/a", IsLatest: true, ModTime: mod},
}), rec, WalkOptions{Now: now}); err != nil {
t.Fatalf("Walk: %v", err)
}
if len(rec.calls) != 0 {
t.Fatalf("inactive action should not dispatch, got %v", rec.calls)
}
}
func TestWalk_DispatchErrorHaltsAtCheckpoint(t *testing.T) {
rule := &s3lifecycle.Rule{
ID: "r",
Status: s3lifecycle.StatusEnabled,
ExpirationDays: 1,
}
snap := compileEvDriven(t, "bk", rule)
mod := mustTime(t, "2024-01-01T00:00:00Z")
now := mod.AddDate(0, 0, 10)
entries := []*Entry{
{Path: "a", IsLatest: true, ModTime: mod},
{Path: "b", IsLatest: true, ModTime: mod},
{Path: "c", IsLatest: true, ModTime: mod},
}
wantErr := errors.New("dispatch boom")
rec := &recorder{err: wantErr}
cp, err := Walk(context.Background(), snap, "bk", EntryCallback(entries), rec, WalkOptions{Now: now})
if !errors.Is(err, wantErr) {
t.Fatalf("want dispatch error, got %v", err)
}
if cp.Completed {
t.Fatalf("walk should not be Completed on dispatch failure")
}
// Walker stops on first failure; checkpoint stays at whatever was
// recorded BEFORE the failed entry. Path "a" is the failing entry,
// so LastScannedPath stays at the resume point (empty here).
if cp.LastScannedPath != "" {
t.Fatalf("checkpoint should not advance past failing entry, got %q", cp.LastScannedPath)
}
}
func TestWalk_ResumeFromCheckpoint(t *testing.T) {
rule := &s3lifecycle.Rule{
ID: "r",
Status: s3lifecycle.StatusEnabled,
ExpirationDays: 1,
}
snap := compileEvDriven(t, "bk", rule)
mod := mustTime(t, "2024-01-01T00:00:00Z")
now := mod.AddDate(0, 0, 10)
entries := []*Entry{
{Path: "a", IsLatest: true, ModTime: mod},
{Path: "b", IsLatest: true, ModTime: mod},
{Path: "c", IsLatest: true, ModTime: mod},
}
rec := &recorder{}
cp, err := Walk(context.Background(), snap, "bk", EntryCallback(entries), rec, WalkOptions{Now: now, Resume: "b"})
if err != nil {
t.Fatalf("Walk: %v", err)
}
if !cp.Completed {
t.Fatalf("walk should complete")
}
// Only "c" is processed (entries with Path <= "b" are skipped).
if len(rec.calls) != 1 || rec.calls[0].path != "c" {
t.Fatalf("Resume should only process c, got %v", rec.calls)
}
if cp.LastScannedPath != "c" {
t.Fatalf("checkpoint want c, got %q", cp.LastScannedPath)
}
}
func TestWalk_MPUInitDirMatchesByDestKey(t *testing.T) {
// Existing in-flight MPUs predate the meta-log subscription, so they
// only get cleaned up via the bootstrap walk. The init record is a
// directory whose path is .uploads/<id>; the rule's Filter.Prefix
// applies to the destination object key, not the upload directory.
rule := &s3lifecycle.Rule{
ID: "r-mpu",
Status: s3lifecycle.StatusEnabled,
Prefix: "logs/",
AbortMPUDaysAfterInitiation: 7,
}
snap := compileEvDriven(t, "bk", rule)
mod := mustTime(t, "2024-01-01T00:00:00Z")
now := mod.AddDate(0, 0, 8) // past the 7d threshold
entries := []*Entry{
// Matches: dest key under logs/.
{Path: ".uploads/u-match", IsDirectory: true, IsMPUInit: true, DestKey: "logs/foo.txt", ModTime: mod},
// Filtered out: dest key under data/.
{Path: ".uploads/u-skip", IsDirectory: true, IsMPUInit: true, DestKey: "data/foo.txt", ModTime: mod},
// No DestKey: malformed init mid-write; skip rather than guess.
{Path: ".uploads/u-bare", IsDirectory: true, IsMPUInit: true, ModTime: mod},
}
rec := &recorder{}
if _, err := Walk(context.Background(), snap, "bk", EntryCallback(entries), rec, WalkOptions{Now: now}); err != nil {
t.Fatalf("Walk: %v", err)
}
if len(rec.calls) != 1 {
t.Fatalf("expected 1 dispatch (u-match only), got %v", rec.calls)
}
if rec.calls[0].path != ".uploads/u-match" {
t.Fatalf("dispatch path=%q, want .uploads/u-match (the rm target)", rec.calls[0].path)
}
if rec.calls[0].kind != s3lifecycle.ActionKindAbortMPU {
t.Fatalf("dispatch kind=%v, want AbortMPU", rec.calls[0].kind)
}
}
func TestWalk_NonMPUDirectorySkipped(t *testing.T) {
// Non-MPU directories must still be skipped — the relaxed
// IsDirectory check is gated on IsMPUInit.
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
snap := compileEvDriven(t, "bk", rule)
mod := mustTime(t, "2024-01-01T00:00:00Z")
now := mod.AddDate(0, 0, 100)
rec := &recorder{}
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
{Path: "a/", IsDirectory: true, IsLatest: true, ModTime: mod},
}), rec, WalkOptions{Now: now}); err != nil {
t.Fatalf("Walk: %v", err)
}
if len(rec.calls) != 0 {
t.Fatalf("plain directory should not dispatch, got %v", rec.calls)
}
}
func TestWalk_MPUInitDoesNotFireNoncurrent(t *testing.T) {
// Same rule covers both AbortMPU and NoncurrentVersionExpiration; the
// MPU init record must dispatch only the AbortMPU action. Without the
// engine guard, NONCURRENT_DAYS would fire (IsLatest=false) and the
// server would BLOCK on empty version_id, freezing the cursor.
rule := &s3lifecycle.Rule{
ID: "r",
Status: s3lifecycle.StatusEnabled,
AbortMPUDaysAfterInitiation: 7,
NoncurrentVersionExpirationDays: 7,
}
snap := compileEvDriven(t, "bk", rule)
mod := mustTime(t, "2024-01-01T00:00:00Z")
now := mod.AddDate(0, 0, 30)
rec := &recorder{}
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
{Path: ".uploads/u1", IsDirectory: true, IsMPUInit: true, DestKey: "obj/a", ModTime: mod},
}), rec, WalkOptions{Now: now}); err != nil {
t.Fatalf("Walk: %v", err)
}
if len(rec.calls) != 1 {
t.Fatalf("expected 1 dispatch (AbortMPU only), got %v", rec.calls)
}
if rec.calls[0].kind != s3lifecycle.ActionKindAbortMPU {
t.Fatalf("kind=%v, want AbortMPU", rec.calls[0].kind)
}
}