snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
33
weed/s3api/s3lifecycle/bootstrap/has_prefix_test.go
Normal file
33
weed/s3api/s3lifecycle/bootstrap/has_prefix_test.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// HasPrefix is a thin wrapper around strings.HasPrefix exported by the
|
||||
// bootstrap package so call sites can avoid pulling strings just to do
|
||||
// a prefix check. Trivial but at 0% coverage — pin it.
|
||||
|
||||
func TestHasPrefix(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
path string
|
||||
prefix string
|
||||
want bool
|
||||
}{
|
||||
{"matching prefix", "logs/2026/01/01", "logs/", true},
|
||||
{"exact match", "logs/", "logs/", true},
|
||||
{"non-matching prefix", "metrics/x", "logs/", false},
|
||||
{"shorter than prefix", "logs", "logs/", false},
|
||||
{"empty prefix matches all", "anything", "", true},
|
||||
{"both empty", "", "", true},
|
||||
{"empty input non-empty prefix", "", "x", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
assert.Equal(t, c.want, HasPrefix(c.path, c.prefix))
|
||||
})
|
||||
}
|
||||
}
|
||||
197
weed/s3api/s3lifecycle/bootstrap/walker.go
Normal file
197
weed/s3api/s3lifecycle/bootstrap/walker.go
Normal file
@@ -0,0 +1,197 @@
|
||||
// Package bootstrap is the bucket-level lifecycle walker. The walker
|
||||
// iterates every entry under a bucket, evaluates every active ActionKey
|
||||
// against it, and dispatches inline-delete RPCs for currently-due actions;
|
||||
// not-yet-due entries are left for the meta-log reader to pick up later.
|
||||
//
|
||||
// Callback-driven so the listing source and the LifecycleDelete dispatcher
|
||||
// can be supplied separately (real client or test fake).
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
)
|
||||
|
||||
// Entry is the routing-relevant slice of a filer entry. SuccessorModTime
|
||||
// and NoncurrentIndex are populated only on versioned-bucket walks; the
|
||||
// retention path bails out conservatively when they're zero / nil.
|
||||
//
|
||||
// MPU init directories at .uploads/<id> populate DestKey with the
|
||||
// destination object key (from entry.Extended[ExtMultipartObjectKey]) so
|
||||
// rule-prefix matching works on the user's intended path while Path
|
||||
// stays as the upload directory the dispatcher must rm.
|
||||
type Entry struct {
|
||||
Path string
|
||||
DestKey string
|
||||
ModTime time.Time
|
||||
Size int64
|
||||
IsDirectory bool
|
||||
IsLatest bool
|
||||
IsDeleteMarker bool
|
||||
IsMPUInit bool
|
||||
NumVersions int
|
||||
Tags map[string]string
|
||||
|
||||
SuccessorModTime time.Time
|
||||
NoncurrentIndex *int
|
||||
|
||||
// VersionID is the S3 version id of this entry, empty for
|
||||
// non-versioned buckets. Populated by the ListFunc adapter when
|
||||
// the entry came from a `.versions/` directory; the walker itself
|
||||
// doesn't use it, but the Dispatcher needs it to address the
|
||||
// right version on LifecycleDelete.
|
||||
VersionID string
|
||||
}
|
||||
|
||||
// ListFunc must skip entries with Path <= start so kill-resume picks up
|
||||
// where the previous run stopped.
|
||||
type ListFunc func(ctx context.Context, bucket, start string, cb func(*Entry) error) error
|
||||
|
||||
// Dispatcher executes one (action, entry) verdict. An error halts the walk;
|
||||
// the caller decides whether to retry from the recorded last_scanned_path.
|
||||
type Dispatcher interface {
|
||||
Delete(ctx context.Context, action *engine.CompiledAction, entry *Entry) error
|
||||
}
|
||||
|
||||
// Checkpoint is the resume state. Caller persists it under
|
||||
// /etc/s3/lifecycle/<bucket>/_bootstrap.
|
||||
type Checkpoint struct {
|
||||
LastScannedPath string
|
||||
Completed bool
|
||||
}
|
||||
|
||||
type WalkOptions struct {
|
||||
Resume string
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
// Walk iterates entries via list, evaluates each active ActionKey via
|
||||
// MatchPath + EvaluateAction, and calls Dispatcher.Delete for currently-due
|
||||
// actions. SCAN_AT_DATE actions are skipped (their bootstrap is scheduled
|
||||
// separately).
|
||||
func Walk(ctx context.Context, snap *engine.Snapshot, bucket string, list ListFunc, dispatch Dispatcher, opts WalkOptions) (Checkpoint, error) {
|
||||
now := opts.Now
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
cp := Checkpoint{LastScannedPath: opts.Resume}
|
||||
|
||||
// Reuse one ObjectInfo across the walk; EvaluateAction reads it
|
||||
// synchronously without retaining.
|
||||
var info s3lifecycle.ObjectInfo
|
||||
|
||||
err := list(ctx, bucket, opts.Resume, func(entry *Entry) error {
|
||||
if entry == nil || entry.Path == "" {
|
||||
return nil
|
||||
}
|
||||
// MPU init directories at .uploads/<id> are the one directory
|
||||
// shape lifecycle cares about; everything else stays out.
|
||||
if entry.IsDirectory && !entry.IsMPUInit {
|
||||
cp.LastScannedPath = entry.Path
|
||||
return nil
|
||||
}
|
||||
if err := walkEntry(ctx, snap, bucket, entry, dispatch, now, &info); err != nil {
|
||||
return err
|
||||
}
|
||||
cp.LastScannedPath = entry.Path
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return cp, err
|
||||
}
|
||||
cp.Completed = true
|
||||
return cp, nil
|
||||
}
|
||||
|
||||
func walkEntry(ctx context.Context, snap *engine.Snapshot, bucket string, entry *Entry, dispatch Dispatcher, now time.Time, info *s3lifecycle.ObjectInfo) error {
|
||||
// MPU init: rule-prefix matching uses the destination key, not the
|
||||
// .uploads/<id> directory path. A bare directory with no DestKey is
|
||||
// either a stray dir or an init mid-write before metadata landed —
|
||||
// skip rather than guess.
|
||||
matchKey := entry.Path
|
||||
if entry.IsMPUInit {
|
||||
if entry.DestKey == "" {
|
||||
return nil
|
||||
}
|
||||
matchKey = entry.DestKey
|
||||
}
|
||||
keys := snap.MatchPath(bucket, matchKey, nil)
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
*info = s3lifecycle.ObjectInfo{
|
||||
Key: matchKey,
|
||||
ModTime: entry.ModTime,
|
||||
Size: entry.Size,
|
||||
IsLatest: entry.IsLatest,
|
||||
IsDeleteMarker: entry.IsDeleteMarker,
|
||||
IsMPUInit: entry.IsMPUInit,
|
||||
NumVersions: entry.NumVersions,
|
||||
SuccessorModTime: entry.SuccessorModTime,
|
||||
NoncurrentIndex: entry.NoncurrentIndex,
|
||||
Tags: entry.Tags,
|
||||
}
|
||||
for _, key := range keys {
|
||||
action := snap.Action(key)
|
||||
if action == nil {
|
||||
continue
|
||||
}
|
||||
// DISABLED can be flipped at runtime independent of XML Status,
|
||||
// so skip it even though EvaluateAction would also reject.
|
||||
// SCAN_AT_DATE actions are processed here too — the date check
|
||||
// in EvaluateAction (now.Before(rule.ExpirationDate)) gates the
|
||||
// dispatch, so pre-date walks are no-ops and post-date walks
|
||||
// expire eligible objects. The earlier "scan-at-date runs its
|
||||
// own bootstrap" plan was never wired; until that lands, the
|
||||
// regular bootstrap walk is the only path that fires
|
||||
// ExpirationDate rules.
|
||||
if action.Mode == engine.ModeDisabled {
|
||||
continue
|
||||
}
|
||||
// (kind, info) shape gate: ABORT_MPU only on MPU init records,
|
||||
// every other kind only on regular objects/versions. Mismatched
|
||||
// pairs would either dispatch a noncurrent action with empty
|
||||
// version_id (server BLOCKs, cursor freezes) or dispatch
|
||||
// ABORT_MPU against a regular object path.
|
||||
if entry.IsMPUInit && key.ActionKind != s3lifecycle.ActionKindAbortMPU {
|
||||
continue
|
||||
}
|
||||
if !entry.IsMPUInit && key.ActionKind == s3lifecycle.ActionKindAbortMPU {
|
||||
continue
|
||||
}
|
||||
res := s3lifecycle.EvaluateAction(action.Rule, key.ActionKind, info, now)
|
||||
if res.Action == s3lifecycle.ActionNone {
|
||||
continue
|
||||
}
|
||||
if err := dispatch.Delete(ctx, action, entry); err != nil {
|
||||
glog.Warningf("lifecycle bootstrap: dispatch %s/%s kind=%s: %v",
|
||||
bucket, entry.Path, key.ActionKind, err)
|
||||
return err
|
||||
}
|
||||
stats.S3LifecycleBootstrapDispatchCounter.WithLabelValues(bucket, key.ActionKind.String()).Inc()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EntryCallback wraps an in-memory slice as a ListFunc; useful for tests.
|
||||
func EntryCallback(entries []*Entry) ListFunc {
|
||||
return func(ctx context.Context, bucket, start string, cb func(*Entry) error) error {
|
||||
for _, e := range entries {
|
||||
if start != "" && e.Path <= start {
|
||||
continue
|
||||
}
|
||||
if err := cb(e); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func HasPrefix(path, prefix string) bool { return strings.HasPrefix(path, prefix) }
|
||||
431
weed/s3api/s3lifecycle/bootstrap/walker_test.go
Normal file
431
weed/s3api/s3lifecycle/bootstrap/walker_test.go
Normal file
@@ -0,0 +1,431 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
|
||||
)
|
||||
|
||||
// recorder captures dispatched (action, entry) pairs for assertion.
|
||||
type recorder struct {
|
||||
calls []dispatchCall
|
||||
err error // when set, every Delete returns this error
|
||||
}
|
||||
|
||||
type dispatchCall struct {
|
||||
kind s3lifecycle.ActionKind
|
||||
path string
|
||||
}
|
||||
|
||||
func (r *recorder) Delete(ctx context.Context, action *engine.CompiledAction, entry *Entry) error {
|
||||
if r.err != nil {
|
||||
return r.err
|
||||
}
|
||||
r.calls = append(r.calls, dispatchCall{kind: action.Key.ActionKind, path: entry.Path})
|
||||
return nil
|
||||
}
|
||||
|
||||
func mustTime(t *testing.T, s string) time.Time {
|
||||
t.Helper()
|
||||
tm, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
t.Fatalf("parse %s: %v", s, err)
|
||||
}
|
||||
return tm
|
||||
}
|
||||
|
||||
func compileEvDriven(t *testing.T, bucket string, rules ...*s3lifecycle.Rule) *engine.Snapshot {
|
||||
t.Helper()
|
||||
prior := map[s3lifecycle.ActionKey]engine.PriorState{}
|
||||
for _, r := range rules {
|
||||
rh := s3lifecycle.RuleHash(r)
|
||||
for _, k := range s3lifecycle.RuleActionKinds(r) {
|
||||
prior[s3lifecycle.ActionKey{Bucket: bucket, RuleHash: rh, ActionKind: k}] = engine.PriorState{BootstrapComplete: true}
|
||||
}
|
||||
}
|
||||
e := engine.New()
|
||||
return e.Compile([]engine.CompileInput{{Bucket: bucket, Rules: rules}}, engine.CompileOptions{PriorStates: prior})
|
||||
}
|
||||
|
||||
func TestWalk_DispatchesDueActions(t *testing.T) {
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 30,
|
||||
Prefix: "logs/",
|
||||
}
|
||||
snap := compileEvDriven(t, "bk", rule)
|
||||
|
||||
mod := mustTime(t, "2024-01-01T00:00:00Z")
|
||||
now := mod.AddDate(0, 0, 60) // past the 30d threshold
|
||||
entries := []*Entry{
|
||||
{Path: "data/x", IsLatest: true, ModTime: mod}, // wrong prefix
|
||||
{Path: "logs/a", IsLatest: true, ModTime: mod}, // due
|
||||
{Path: "logs/b", IsLatest: true, ModTime: now}, // not yet due (mod=now)
|
||||
}
|
||||
|
||||
rec := &recorder{}
|
||||
cp, err := Walk(context.Background(), snap, "bk", EntryCallback(entries), rec, WalkOptions{Now: now})
|
||||
if err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
if !cp.Completed {
|
||||
t.Fatalf("walk should complete")
|
||||
}
|
||||
if cp.LastScannedPath != "logs/b" {
|
||||
t.Fatalf("checkpoint last scanned want logs/b, got %q", cp.LastScannedPath)
|
||||
}
|
||||
if len(rec.calls) != 1 || rec.calls[0].path != "logs/a" {
|
||||
t.Fatalf("dispatched calls want [logs/a], got %v", rec.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalk_MultiActionRule_AllDueDispatched(t *testing.T) {
|
||||
// One rule with three actions; all currently-due for the entry. The
|
||||
// walker must dispatch one Delete per action — this is the
|
||||
// regression that the per-action keying fixes.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "multi",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 30,
|
||||
NoncurrentVersionExpirationDays: 7,
|
||||
AbortMPUDaysAfterInitiation: 5,
|
||||
}
|
||||
snap := compileEvDriven(t, "bk", rule)
|
||||
|
||||
mod := mustTime(t, "2024-01-01T00:00:00Z")
|
||||
now := mod.AddDate(0, 0, 100) // past every threshold
|
||||
// Use a single entry that satisfies all three action shapes is
|
||||
// unrealistic; in practice each shape is a different entry. Cover
|
||||
// each shape independently.
|
||||
entries := []*Entry{
|
||||
// Current version under ExpirationDays.
|
||||
{Path: "obj/a", IsLatest: true, ModTime: mod},
|
||||
// Non-current version under NoncurrentDays.
|
||||
{Path: "obj/a/.versions/v1", IsLatest: false, ModTime: mod, SuccessorModTime: mod},
|
||||
// MPU init under AbortMPU. Real init is a directory; DestKey
|
||||
// carries the eventual object key for prefix matching.
|
||||
{Path: ".uploads/u1", IsDirectory: true, IsMPUInit: true, DestKey: "obj/a", ModTime: mod},
|
||||
}
|
||||
|
||||
rec := &recorder{}
|
||||
if _, err := Walk(context.Background(), snap, "bk", EntryCallback(entries), rec, WalkOptions{Now: now}); err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
|
||||
// Exact-shape assertion: each entry dispatches exactly one action,
|
||||
// and ABORT_MPU only fires on the .uploads/<id> entry. A weaker
|
||||
// "kinds-as-set" check would have missed the (kind, info) gating
|
||||
// regression where an MPU init also fired NONCURRENT_DAYS.
|
||||
want := []dispatchCall{
|
||||
{kind: s3lifecycle.ActionKindAbortMPU, path: ".uploads/u1"},
|
||||
{kind: s3lifecycle.ActionKindExpirationDays, path: "obj/a"},
|
||||
{kind: s3lifecycle.ActionKindNoncurrentDays, path: "obj/a/.versions/v1"},
|
||||
}
|
||||
got := append([]dispatchCall(nil), rec.calls...)
|
||||
sort.Slice(got, func(i, j int) bool {
|
||||
if got[i].path != got[j].path {
|
||||
return got[i].path < got[j].path
|
||||
}
|
||||
return got[i].kind < got[j].kind
|
||||
})
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("dispatch calls mismatch:\n got %+v\nwant %+v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalk_NotYetDueSkipped(t *testing.T) {
|
||||
// The reader (Phase 3) is responsible for not-yet-due entries; the
|
||||
// walker dispatches only currently-due ones, so the meta-log path
|
||||
// stays the steady-state route.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 30,
|
||||
}
|
||||
snap := compileEvDriven(t, "bk", rule)
|
||||
mod := mustTime(t, "2024-01-01T00:00:00Z")
|
||||
now := mod.Add(s3lifecycle.DaysToDuration(10)) // before the 30d threshold
|
||||
|
||||
rec := &recorder{}
|
||||
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
|
||||
{Path: "x/a", IsLatest: true, ModTime: mod},
|
||||
}), rec, WalkOptions{Now: now}); err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
if len(rec.calls) != 0 {
|
||||
t.Fatalf("not-yet-due entry should not dispatch, got %v", rec.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalk_DateActionFiresAfterDate(t *testing.T) {
|
||||
// The dedicated SCAN_AT_DATE bootstrap was never wired; until it
|
||||
// lands, the regular bootstrap walker is the only path that fires
|
||||
// ExpirationDate rules. Entries past the rule's date must dispatch.
|
||||
date := mustTime(t, "2025-06-15T00:00:00Z")
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "d",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDate: date,
|
||||
}
|
||||
snap := compileEvDriven(t, "bk", rule)
|
||||
|
||||
rec := &recorder{}
|
||||
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
|
||||
{Path: "x/a", IsLatest: true, ModTime: mustTime(t, "2024-01-01T00:00:00Z")},
|
||||
}), rec, WalkOptions{Now: date.AddDate(0, 1, 0)}); err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
if len(rec.calls) != 1 || rec.calls[0].path != "x/a" {
|
||||
t.Fatalf("date kind past the rule date must dispatch, got %v", rec.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalk_DateActionSkippedBeforeDate(t *testing.T) {
|
||||
// Pre-date walks are no-ops: EvaluateAction returns ActionNone when
|
||||
// now is before rule.ExpirationDate.
|
||||
date := mustTime(t, "2025-06-15T00:00:00Z")
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "d",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDate: date,
|
||||
}
|
||||
snap := compileEvDriven(t, "bk", rule)
|
||||
|
||||
rec := &recorder{}
|
||||
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
|
||||
{Path: "x/a", IsLatest: true, ModTime: mustTime(t, "2024-01-01T00:00:00Z")},
|
||||
}), rec, WalkOptions{Now: date.AddDate(0, -1, 0)}); err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
if len(rec.calls) != 0 {
|
||||
t.Fatalf("pre-date walk must not dispatch, got %v", rec.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalk_DirectoryEntriesSkipped(t *testing.T) {
|
||||
// SeaweedFS directory entries can co-exist in the listing; the walker
|
||||
// must not dispatch deletes against them even when their path matches.
|
||||
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
|
||||
snap := compileEvDriven(t, "bk", rule)
|
||||
mod := mustTime(t, "2024-01-01T00:00:00Z")
|
||||
now := mod.AddDate(0, 0, 10)
|
||||
rec := &recorder{}
|
||||
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
|
||||
{Path: "x", IsDirectory: true, ModTime: mod}, // directory; must skip
|
||||
{Path: "x/file", IsLatest: true, ModTime: mod},
|
||||
}), rec, WalkOptions{Now: now}); err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
if len(rec.calls) != 1 || rec.calls[0].path != "x/file" {
|
||||
t.Fatalf("only the file should dispatch, got %v", rec.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalk_DisabledModeSkipped(t *testing.T) {
|
||||
// An operator-flipped ModeDisabled must short-circuit the walker even
|
||||
// when the XML rule status is "Enabled" and EvaluateAction would
|
||||
// otherwise fire.
|
||||
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
|
||||
rh := s3lifecycle.RuleHash(rule)
|
||||
prior := map[s3lifecycle.ActionKey]engine.PriorState{
|
||||
{Bucket: "bk", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}: {
|
||||
BootstrapComplete: true, Mode: engine.ModeDisabled,
|
||||
},
|
||||
}
|
||||
e := engine.New()
|
||||
snap := e.Compile([]engine.CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}}, engine.CompileOptions{PriorStates: prior})
|
||||
mod := mustTime(t, "2024-01-01T00:00:00Z")
|
||||
now := mod.AddDate(0, 0, 10)
|
||||
rec := &recorder{}
|
||||
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
|
||||
{Path: "x/a", IsLatest: true, ModTime: mod},
|
||||
}), rec, WalkOptions{Now: now}); err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
if len(rec.calls) != 0 {
|
||||
t.Fatalf("disabled action must not dispatch, got %v", rec.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalk_PendingBootstrapNotDispatched(t *testing.T) {
|
||||
// Without bootstrap_complete=true in PriorStates, the engine compiles
|
||||
// the action as inactive. MatchPath filters on IsActive, so the
|
||||
// walker won't dispatch.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 1,
|
||||
}
|
||||
e := engine.New()
|
||||
snap := e.Compile([]engine.CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}}, engine.CompileOptions{})
|
||||
|
||||
mod := mustTime(t, "2024-01-01T00:00:00Z")
|
||||
now := mod.AddDate(0, 0, 10)
|
||||
rec := &recorder{}
|
||||
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
|
||||
{Path: "x/a", IsLatest: true, ModTime: mod},
|
||||
}), rec, WalkOptions{Now: now}); err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
if len(rec.calls) != 0 {
|
||||
t.Fatalf("inactive action should not dispatch, got %v", rec.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalk_DispatchErrorHaltsAtCheckpoint(t *testing.T) {
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 1,
|
||||
}
|
||||
snap := compileEvDriven(t, "bk", rule)
|
||||
mod := mustTime(t, "2024-01-01T00:00:00Z")
|
||||
now := mod.AddDate(0, 0, 10)
|
||||
entries := []*Entry{
|
||||
{Path: "a", IsLatest: true, ModTime: mod},
|
||||
{Path: "b", IsLatest: true, ModTime: mod},
|
||||
{Path: "c", IsLatest: true, ModTime: mod},
|
||||
}
|
||||
|
||||
wantErr := errors.New("dispatch boom")
|
||||
rec := &recorder{err: wantErr}
|
||||
cp, err := Walk(context.Background(), snap, "bk", EntryCallback(entries), rec, WalkOptions{Now: now})
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("want dispatch error, got %v", err)
|
||||
}
|
||||
if cp.Completed {
|
||||
t.Fatalf("walk should not be Completed on dispatch failure")
|
||||
}
|
||||
// Walker stops on first failure; checkpoint stays at whatever was
|
||||
// recorded BEFORE the failed entry. Path "a" is the failing entry,
|
||||
// so LastScannedPath stays at the resume point (empty here).
|
||||
if cp.LastScannedPath != "" {
|
||||
t.Fatalf("checkpoint should not advance past failing entry, got %q", cp.LastScannedPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalk_ResumeFromCheckpoint(t *testing.T) {
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 1,
|
||||
}
|
||||
snap := compileEvDriven(t, "bk", rule)
|
||||
mod := mustTime(t, "2024-01-01T00:00:00Z")
|
||||
now := mod.AddDate(0, 0, 10)
|
||||
entries := []*Entry{
|
||||
{Path: "a", IsLatest: true, ModTime: mod},
|
||||
{Path: "b", IsLatest: true, ModTime: mod},
|
||||
{Path: "c", IsLatest: true, ModTime: mod},
|
||||
}
|
||||
|
||||
rec := &recorder{}
|
||||
cp, err := Walk(context.Background(), snap, "bk", EntryCallback(entries), rec, WalkOptions{Now: now, Resume: "b"})
|
||||
if err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
if !cp.Completed {
|
||||
t.Fatalf("walk should complete")
|
||||
}
|
||||
// Only "c" is processed (entries with Path <= "b" are skipped).
|
||||
if len(rec.calls) != 1 || rec.calls[0].path != "c" {
|
||||
t.Fatalf("Resume should only process c, got %v", rec.calls)
|
||||
}
|
||||
if cp.LastScannedPath != "c" {
|
||||
t.Fatalf("checkpoint want c, got %q", cp.LastScannedPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalk_MPUInitDirMatchesByDestKey(t *testing.T) {
|
||||
// Existing in-flight MPUs predate the meta-log subscription, so they
|
||||
// only get cleaned up via the bootstrap walk. The init record is a
|
||||
// directory whose path is .uploads/<id>; the rule's Filter.Prefix
|
||||
// applies to the destination object key, not the upload directory.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r-mpu",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
Prefix: "logs/",
|
||||
AbortMPUDaysAfterInitiation: 7,
|
||||
}
|
||||
snap := compileEvDriven(t, "bk", rule)
|
||||
mod := mustTime(t, "2024-01-01T00:00:00Z")
|
||||
now := mod.AddDate(0, 0, 8) // past the 7d threshold
|
||||
|
||||
entries := []*Entry{
|
||||
// Matches: dest key under logs/.
|
||||
{Path: ".uploads/u-match", IsDirectory: true, IsMPUInit: true, DestKey: "logs/foo.txt", ModTime: mod},
|
||||
// Filtered out: dest key under data/.
|
||||
{Path: ".uploads/u-skip", IsDirectory: true, IsMPUInit: true, DestKey: "data/foo.txt", ModTime: mod},
|
||||
// No DestKey: malformed init mid-write; skip rather than guess.
|
||||
{Path: ".uploads/u-bare", IsDirectory: true, IsMPUInit: true, ModTime: mod},
|
||||
}
|
||||
|
||||
rec := &recorder{}
|
||||
if _, err := Walk(context.Background(), snap, "bk", EntryCallback(entries), rec, WalkOptions{Now: now}); err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
if len(rec.calls) != 1 {
|
||||
t.Fatalf("expected 1 dispatch (u-match only), got %v", rec.calls)
|
||||
}
|
||||
if rec.calls[0].path != ".uploads/u-match" {
|
||||
t.Fatalf("dispatch path=%q, want .uploads/u-match (the rm target)", rec.calls[0].path)
|
||||
}
|
||||
if rec.calls[0].kind != s3lifecycle.ActionKindAbortMPU {
|
||||
t.Fatalf("dispatch kind=%v, want AbortMPU", rec.calls[0].kind)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalk_NonMPUDirectorySkipped(t *testing.T) {
|
||||
// Non-MPU directories must still be skipped — the relaxed
|
||||
// IsDirectory check is gated on IsMPUInit.
|
||||
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
|
||||
snap := compileEvDriven(t, "bk", rule)
|
||||
mod := mustTime(t, "2024-01-01T00:00:00Z")
|
||||
now := mod.AddDate(0, 0, 100)
|
||||
|
||||
rec := &recorder{}
|
||||
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
|
||||
{Path: "a/", IsDirectory: true, IsLatest: true, ModTime: mod},
|
||||
}), rec, WalkOptions{Now: now}); err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
if len(rec.calls) != 0 {
|
||||
t.Fatalf("plain directory should not dispatch, got %v", rec.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalk_MPUInitDoesNotFireNoncurrent(t *testing.T) {
|
||||
// Same rule covers both AbortMPU and NoncurrentVersionExpiration; the
|
||||
// MPU init record must dispatch only the AbortMPU action. Without the
|
||||
// engine guard, NONCURRENT_DAYS would fire (IsLatest=false) and the
|
||||
// server would BLOCK on empty version_id, freezing the cursor.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
AbortMPUDaysAfterInitiation: 7,
|
||||
NoncurrentVersionExpirationDays: 7,
|
||||
}
|
||||
snap := compileEvDriven(t, "bk", rule)
|
||||
mod := mustTime(t, "2024-01-01T00:00:00Z")
|
||||
now := mod.AddDate(0, 0, 30)
|
||||
|
||||
rec := &recorder{}
|
||||
if _, err := Walk(context.Background(), snap, "bk", EntryCallback([]*Entry{
|
||||
{Path: ".uploads/u1", IsDirectory: true, IsMPUInit: true, DestKey: "obj/a", ModTime: mod},
|
||||
}), rec, WalkOptions{Now: now}); err != nil {
|
||||
t.Fatalf("Walk: %v", err)
|
||||
}
|
||||
if len(rec.calls) != 1 {
|
||||
t.Fatalf("expected 1 dispatch (AbortMPU only), got %v", rec.calls)
|
||||
}
|
||||
if rec.calls[0].kind != s3lifecycle.ActionKindAbortMPU {
|
||||
t.Fatalf("kind=%v, want AbortMPU", rec.calls[0].kind)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user