snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
133
weed/s3api/s3lifecycle/engine/compile.go
Normal file
133
weed/s3api/s3lifecycle/engine/compile.go
Normal file
@@ -0,0 +1,133 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sort"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
)
|
||||
|
||||
var snapshotIDSeq atomic.Uint64
|
||||
|
||||
// CompileInput: at most one entry per Bucket. Duplicates would overwrite
|
||||
// earlier entries' BucketIndex.
|
||||
type CompileInput struct {
|
||||
Bucket string
|
||||
Rules []*s3lifecycle.Rule
|
||||
Versioned bool
|
||||
}
|
||||
|
||||
// PriorState carries durable per-action state into Compile. Missing keys
|
||||
// are treated as bootstrap_complete=false.
|
||||
type PriorState struct {
|
||||
BootstrapComplete bool
|
||||
Mode RuleMode
|
||||
}
|
||||
|
||||
// MetaLogRetention=0 means unbounded; the retention gate doesn't trip.
|
||||
type CompileOptions struct {
|
||||
MetaLogRetention time.Duration
|
||||
BootstrapLookbackMin time.Duration
|
||||
PriorStates map[s3lifecycle.ActionKey]PriorState
|
||||
}
|
||||
|
||||
const defaultBootstrapLookbackMin = 5 * s3lifecycle.SmallDelay
|
||||
|
||||
func (e *Engine) Compile(inputs []CompileInput, opts CompileOptions) *Snapshot {
|
||||
if opts.BootstrapLookbackMin == 0 {
|
||||
opts.BootstrapLookbackMin = defaultBootstrapLookbackMin
|
||||
}
|
||||
|
||||
snap := &Snapshot{
|
||||
id: snapshotIDSeq.Add(1),
|
||||
buckets: make(map[string]*BucketIndex),
|
||||
actions: make(map[s3lifecycle.ActionKey]*CompiledAction),
|
||||
originalDelayGroups: make(map[time.Duration][]s3lifecycle.ActionKey),
|
||||
dateActions: make(map[s3lifecycle.ActionKey]time.Time),
|
||||
}
|
||||
|
||||
for _, in := range inputs {
|
||||
bi := &BucketIndex{bucket: in.Bucket, versioned: in.Versioned}
|
||||
snap.buckets[in.Bucket] = bi
|
||||
|
||||
for _, rule := range in.Rules {
|
||||
ruleHash := s3lifecycle.RuleHash(rule)
|
||||
for _, kind := range s3lifecycle.RuleActionKinds(rule) {
|
||||
key := s3lifecycle.ActionKey{Bucket: in.Bucket, RuleHash: ruleHash, ActionKind: kind}
|
||||
// Durable Mode wins; decideMode is only the seed for new
|
||||
// or legacy-zero actions. Otherwise lag-fallback / operator
|
||||
// SCAN_ONLY would re-promote on every rebuild.
|
||||
prior, hasPrior := opts.PriorStates[key]
|
||||
mode := prior.Mode
|
||||
if !hasPrior || mode == ModeUnspecified {
|
||||
mode = decideMode(rule, kind, opts.MetaLogRetention, opts.BootstrapLookbackMin)
|
||||
}
|
||||
// Active gates routing: MatchPath / MatchOriginalWrite skip
|
||||
// !IsActive actions. ScanAtDate's only dispatch path is the
|
||||
// bootstrap walk's MatchPath call, so the action must be
|
||||
// considered active there or its rule is silently a no-op.
|
||||
// Bootstrap-completion state is per-action and event-driven-
|
||||
// shaped; date-based actions don't need a bootstrap rendezvous.
|
||||
active := mode == ModeScanAtDate ||
|
||||
(prior.BootstrapComplete && mode == ModeEventDriven)
|
||||
|
||||
ca := &CompiledAction{
|
||||
Rule: rule,
|
||||
Bucket: in.Bucket,
|
||||
Key: key,
|
||||
Delay: s3lifecycle.MinTriggerAge(rule, kind),
|
||||
PredicateSensitive: rulePredicateSensitive(rule),
|
||||
Mode: mode,
|
||||
}
|
||||
if active {
|
||||
ca.markActive()
|
||||
}
|
||||
snap.actions[key] = ca
|
||||
bi.actionKeys = append(bi.actionKeys, key)
|
||||
|
||||
// Index every action regardless of `active`; routing
|
||||
// re-filters on IsActive() so MarkActive flips are visible
|
||||
// without a recompile.
|
||||
if mode == ModeScanAtDate {
|
||||
snap.dateActions[key] = rule.ExpirationDate
|
||||
}
|
||||
if mode == ModeEventDriven {
|
||||
snap.originalDelayGroups[ca.Delay] = append(snap.originalDelayGroups[ca.Delay], key)
|
||||
if ca.PredicateSensitive {
|
||||
snap.predicateActions = append(snap.predicateActions, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
snap.allActionsSorted = make([]*CompiledAction, 0, len(snap.actions))
|
||||
for _, a := range snap.actions {
|
||||
snap.allActionsSorted = append(snap.allActionsSorted, a)
|
||||
}
|
||||
sort.Slice(snap.allActionsSorted, func(i, j int) bool {
|
||||
a, b := snap.allActionsSorted[i], snap.allActionsSorted[j]
|
||||
if a.Bucket != b.Bucket {
|
||||
return a.Bucket < b.Bucket
|
||||
}
|
||||
if c := bytes.Compare(a.Key.RuleHash[:], b.Key.RuleHash[:]); c != 0 {
|
||||
return c < 0
|
||||
}
|
||||
return a.Key.ActionKind < b.Key.ActionKind
|
||||
})
|
||||
|
||||
e.current.Store(snap)
|
||||
return snap
|
||||
}
|
||||
|
||||
// rulePredicateSensitive: only tag filters can flip post-PUT. Size is
|
||||
// immutable once written; a size change is a fresh write through the
|
||||
// original-write stream.
|
||||
func rulePredicateSensitive(rule *s3lifecycle.Rule) bool {
|
||||
if rule == nil {
|
||||
return false
|
||||
}
|
||||
return len(rule.FilterTags) > 0
|
||||
}
|
||||
45
weed/s3api/s3lifecycle/engine/compile_helpers_test.go
Normal file
45
weed/s3api/s3lifecycle/engine/compile_helpers_test.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// rulePredicateSensitive's non-nil branches are exercised by the
|
||||
// existing Compile tests; this test pins the defensive nil-rule
|
||||
// branch directly. Compile shouldn't ever pass a nil rule, but a
|
||||
// future caller that does shouldn't panic.
|
||||
|
||||
func TestRulePredicateSensitive_NilRuleReturnsFalse(t *testing.T) {
|
||||
assert.False(t, rulePredicateSensitive(nil))
|
||||
}
|
||||
|
||||
func TestRulePredicateSensitive_NoFilterTagsReturnsFalse(t *testing.T) {
|
||||
// A rule without FilterTags is not predicate-sensitive; the
|
||||
// router's MatchPredicateChange path skips actions whose rule
|
||||
// returns false here.
|
||||
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 7}
|
||||
assert.False(t, rulePredicateSensitive(rule))
|
||||
}
|
||||
|
||||
func TestRulePredicateSensitive_EmptyFilterTagsReturnsFalse(t *testing.T) {
|
||||
// An empty (non-nil) FilterTags map must classify as non-sensitive
|
||||
// — the function checks len, so empty == 0 == false.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
FilterTags: map[string]string{},
|
||||
}
|
||||
assert.False(t, rulePredicateSensitive(rule))
|
||||
}
|
||||
|
||||
func TestRulePredicateSensitive_PopulatedFilterTagsReturnsTrue(t *testing.T) {
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
FilterTags: map[string]string{"env": "prod"},
|
||||
}
|
||||
assert.True(t, rulePredicateSensitive(rule))
|
||||
}
|
||||
132
weed/s3api/s3lifecycle/engine/engine.go
Normal file
132
weed/s3api/s3lifecycle/engine/engine.go
Normal file
@@ -0,0 +1,132 @@
|
||||
// Package engine compiles per-bucket lifecycle rules into a CompiledAction
|
||||
// snapshot. One XML <Rule> expands into N compiled actions (one per populated
|
||||
// action sub-element); each is keyed by ActionKey{bucket, rule_hash,
|
||||
// action_kind} so sibling actions schedule and degrade independently.
|
||||
package engine
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
)
|
||||
|
||||
type Engine struct {
|
||||
current atomic.Pointer[Snapshot]
|
||||
}
|
||||
|
||||
func New() *Engine {
|
||||
e := &Engine{}
|
||||
e.current.Store(&Snapshot{actions: map[s3lifecycle.ActionKey]*CompiledAction{}})
|
||||
return e
|
||||
}
|
||||
|
||||
func (e *Engine) Snapshot() *Snapshot { return e.current.Load() }
|
||||
|
||||
// Snapshot fields are append-only after Compile except CompiledAction.engineState,
|
||||
// which transitions inactive -> active atomically via markActive.
|
||||
type Snapshot struct {
|
||||
id uint64
|
||||
buckets map[string]*BucketIndex
|
||||
actions map[s3lifecycle.ActionKey]*CompiledAction
|
||||
|
||||
allActionsSorted []*CompiledAction
|
||||
|
||||
// Routing indexes hold every ActionKey by mode regardless of activation;
|
||||
// dispatch filters on IsActive().
|
||||
originalDelayGroups map[time.Duration][]s3lifecycle.ActionKey
|
||||
predicateActions []s3lifecycle.ActionKey
|
||||
dateActions map[s3lifecycle.ActionKey]time.Time
|
||||
}
|
||||
|
||||
func (s *Snapshot) SnapshotID() uint64 { return s.id }
|
||||
|
||||
type BucketIndex struct {
|
||||
bucket string
|
||||
versioned bool
|
||||
actionKeys []s3lifecycle.ActionKey
|
||||
}
|
||||
|
||||
type CompiledAction struct {
|
||||
Rule *s3lifecycle.Rule
|
||||
Bucket string
|
||||
Key s3lifecycle.ActionKey
|
||||
Delay time.Duration
|
||||
PredicateSensitive bool
|
||||
Mode RuleMode
|
||||
engineState atomic.Uint32
|
||||
}
|
||||
|
||||
const (
|
||||
engineStateInactive uint32 = 0
|
||||
engineStateActive uint32 = 1
|
||||
)
|
||||
|
||||
func (a *CompiledAction) IsActive() bool {
|
||||
return a.engineState.Load() == engineStateActive
|
||||
}
|
||||
|
||||
func (a *CompiledAction) markActive() {
|
||||
a.engineState.Store(engineStateActive)
|
||||
}
|
||||
|
||||
// MarkActive mirrors the in-memory hint after a durable bootstrap_complete
|
||||
// write. No-op if the key isn't in the snapshot.
|
||||
func (s *Snapshot) MarkActive(key s3lifecycle.ActionKey) {
|
||||
if a, ok := s.actions[key]; ok {
|
||||
a.markActive()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Snapshot) Action(key s3lifecycle.ActionKey) *CompiledAction {
|
||||
return s.actions[key]
|
||||
}
|
||||
|
||||
// AllActions: caller must not mutate.
|
||||
func (s *Snapshot) AllActions() []*CompiledAction {
|
||||
return s.allActionsSorted
|
||||
}
|
||||
|
||||
// OriginalDelayGroups / PredicateActions / DateActions return defensive copies.
|
||||
func (s *Snapshot) OriginalDelayGroups() map[time.Duration][]s3lifecycle.ActionKey {
|
||||
out := make(map[time.Duration][]s3lifecycle.ActionKey, len(s.originalDelayGroups))
|
||||
for d, keys := range s.originalDelayGroups {
|
||||
copied := make([]s3lifecycle.ActionKey, len(keys))
|
||||
copy(copied, keys)
|
||||
out[d] = copied
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Snapshot) PredicateActions() []s3lifecycle.ActionKey {
|
||||
out := make([]s3lifecycle.ActionKey, len(s.predicateActions))
|
||||
copy(out, s.predicateActions)
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Snapshot) DateActions() map[s3lifecycle.ActionKey]time.Time {
|
||||
out := make(map[s3lifecycle.ActionKey]time.Time, len(s.dateActions))
|
||||
for k, v := range s.dateActions {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Snapshot) BucketVersioned(bucket string) bool {
|
||||
if bi, ok := s.buckets[bucket]; ok {
|
||||
return bi.versioned
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// BucketActionKeys returns a defensive copy of the action keys for bucket,
|
||||
// or nil if the bucket has no compiled actions in this snapshot.
|
||||
func (s *Snapshot) BucketActionKeys(bucket string) []s3lifecycle.ActionKey {
|
||||
bi, ok := s.buckets[bucket]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
out := make([]s3lifecycle.ActionKey, len(bi.actionKeys))
|
||||
copy(out, bi.actionKeys)
|
||||
return out
|
||||
}
|
||||
381
weed/s3api/s3lifecycle/engine/engine_test.go
Normal file
381
weed/s3api/s3lifecycle/engine/engine_test.go
Normal file
@@ -0,0 +1,381 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
)
|
||||
|
||||
func ruleExpDays(id, prefix string, days int) *s3lifecycle.Rule {
|
||||
return &s3lifecycle.Rule{
|
||||
ID: id,
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
Prefix: prefix,
|
||||
ExpirationDays: days,
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompile_SingleRuleSingleAction(t *testing.T) {
|
||||
e := New()
|
||||
rule := ruleExpDays("r1", "logs/", 30)
|
||||
snap := e.Compile([]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{
|
||||
PriorStates: map[s3lifecycle.ActionKey]PriorState{
|
||||
{Bucket: "b1", RuleHash: s3lifecycle.RuleHash(rule), ActionKind: s3lifecycle.ActionKindExpirationDays}: {
|
||||
BootstrapComplete: true,
|
||||
},
|
||||
},
|
||||
})
|
||||
if got := len(snap.actions); got != 1 {
|
||||
t.Fatalf("want 1 action, got %d", got)
|
||||
}
|
||||
for _, a := range snap.actions {
|
||||
if !a.IsActive() {
|
||||
t.Fatalf("bootstrap_complete + EVENT_DRIVEN should activate, got mode=%v", a.Mode)
|
||||
}
|
||||
if a.Mode != ModeEventDriven {
|
||||
t.Fatalf("want EVENT_DRIVEN, got %v", a.Mode)
|
||||
}
|
||||
if a.Delay != s3lifecycle.DaysToDuration(30) {
|
||||
t.Fatalf("want 30d delay, got %v", a.Delay)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompile_MultiAction_SiblingsHaveOwnEntries(t *testing.T) {
|
||||
// One XML rule with three actions -> three CompiledActions, three
|
||||
// distinct ActionKeys, three independent delay group memberships.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "multi",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
Prefix: "data/",
|
||||
ExpirationDays: 90,
|
||||
NoncurrentVersionExpirationDays: 30,
|
||||
AbortMPUDaysAfterInitiation: 7,
|
||||
}
|
||||
rh := s3lifecycle.RuleHash(rule)
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{}
|
||||
for _, k := range s3lifecycle.RuleActionKinds(rule) {
|
||||
prior[s3lifecycle.ActionKey{Bucket: "b1", RuleHash: rh, ActionKind: k}] = PriorState{BootstrapComplete: true}
|
||||
}
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{PriorStates: prior})
|
||||
|
||||
if got := len(snap.actions); got != 3 {
|
||||
t.Fatalf("want 3 actions, got %d", got)
|
||||
}
|
||||
wantDelays := map[time.Duration]bool{
|
||||
s3lifecycle.DaysToDuration(90): true, // ExpirationDays
|
||||
s3lifecycle.DaysToDuration(30): true, // NoncurrentDays
|
||||
s3lifecycle.DaysToDuration(7): true, // AbortMPU
|
||||
}
|
||||
for delay, keys := range snap.originalDelayGroups {
|
||||
if !wantDelays[delay] {
|
||||
t.Fatalf("unexpected delay group %v", delay)
|
||||
}
|
||||
if len(keys) != 1 {
|
||||
t.Fatalf("delay %v should hold exactly its own action, got %d", delay, len(keys))
|
||||
}
|
||||
}
|
||||
if len(snap.originalDelayGroups) != 3 {
|
||||
t.Fatalf("want 3 delay groups, got %d", len(snap.originalDelayGroups))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompile_BootstrapPendingIndexedButInactive(t *testing.T) {
|
||||
// Without prior bootstrap_complete=true the action is pending_bootstrap.
|
||||
// It IS indexed in originalDelayGroups (so a later MarkActive flip is
|
||||
// routable without recompile), but IsActive() reads false so the
|
||||
// reader's IsActive-filter skips dispatch.
|
||||
rule := ruleExpDays("r1", "logs/", 30)
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{})
|
||||
|
||||
for _, a := range snap.actions {
|
||||
if a.IsActive() {
|
||||
t.Fatalf("pending_bootstrap action should not be active")
|
||||
}
|
||||
}
|
||||
if len(snap.originalDelayGroups) != 1 {
|
||||
t.Fatalf("EVENT_DRIVEN action should be indexed even when inactive, got %v", snap.originalDelayGroups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompile_RetentionGate(t *testing.T) {
|
||||
// Retention gate compares EventLogHorizon (scales with the day unit)
|
||||
// against MetaLogRetention - BootstrapLookbackMin (where lookback is a
|
||||
// fixed wall-clock minute count). Under the s3tests build tag the day
|
||||
// unit shrinks to 10s, which collapses the margin to zero or negative
|
||||
// and the gate degrades every rule. Skip there; the prod build still
|
||||
// covers the gate semantics this test cares about.
|
||||
if s3lifecycle.DaysToDuration(1) < time.Hour {
|
||||
t.Skip("retention gate semantics require day-unit > lookback margin")
|
||||
}
|
||||
// A 90d ExpirationDays rule with 30d retention should land in scan_only.
|
||||
// A 7d rule (under retention - lookback) stays event_driven.
|
||||
long := ruleExpDays("long", "x/", 90)
|
||||
short := ruleExpDays("short", "y/", 1)
|
||||
rules := []*s3lifecycle.Rule{long, short}
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{}
|
||||
for _, r := range rules {
|
||||
k := s3lifecycle.ActionKey{Bucket: "b1", RuleHash: s3lifecycle.RuleHash(r), ActionKind: s3lifecycle.ActionKindExpirationDays}
|
||||
prior[k] = PriorState{BootstrapComplete: true}
|
||||
}
|
||||
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "b1", Rules: rules}}, CompileOptions{
|
||||
MetaLogRetention: s3lifecycle.DaysToDuration(30),
|
||||
BootstrapLookbackMin: 5 * time.Minute,
|
||||
PriorStates: prior,
|
||||
})
|
||||
|
||||
for _, a := range snap.actions {
|
||||
if a.Rule.ID == "long" && a.Mode != ModeScanOnly {
|
||||
t.Fatalf("90d rule under 30d retention should be scan_only, got %v", a.Mode)
|
||||
}
|
||||
if a.Rule.ID == "short" && a.Mode != ModeEventDriven {
|
||||
t.Fatalf("1d rule should be event_driven, got %v", a.Mode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompile_RetentionUnboundedNeverGates(t *testing.T) {
|
||||
// MetaLogRetention=0 means unbounded (default deployment); even a 100y
|
||||
// rule should stay event_driven.
|
||||
rule := ruleExpDays("decade", "x/", 36500)
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{
|
||||
{RuleHash: s3lifecycle.RuleHash(rule), ActionKind: s3lifecycle.ActionKindExpirationDays}: {BootstrapComplete: true},
|
||||
}
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{PriorStates: prior})
|
||||
for _, a := range snap.actions {
|
||||
if a.Mode != ModeEventDriven {
|
||||
t.Fatalf("unbounded retention should keep event_driven, got %v", a.Mode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompile_SiblingsDegradeIndependently(t *testing.T) {
|
||||
// See TestCompile_RetentionGate for why the s3tests build can't honor
|
||||
// the gate's absolute-time math.
|
||||
if s3lifecycle.DaysToDuration(1) < time.Hour {
|
||||
t.Skip("retention gate semantics require day-unit > lookback margin")
|
||||
}
|
||||
// 90d ExpirationDays + 7d AbortMPU under 30d retention: ExpirationDays
|
||||
// degrades to scan_only, AbortMPU stays event_driven. The whole point
|
||||
// of per-action keying.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "mixed",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
Prefix: "x/",
|
||||
ExpirationDays: 90,
|
||||
AbortMPUDaysAfterInitiation: 7,
|
||||
}
|
||||
rh := s3lifecycle.RuleHash(rule)
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{
|
||||
{RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}: {BootstrapComplete: true},
|
||||
{RuleHash: rh, ActionKind: s3lifecycle.ActionKindAbortMPU}: {BootstrapComplete: true},
|
||||
}
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{
|
||||
MetaLogRetention: s3lifecycle.DaysToDuration(30),
|
||||
BootstrapLookbackMin: 5 * time.Minute,
|
||||
PriorStates: prior,
|
||||
})
|
||||
expDaysKey := s3lifecycle.ActionKey{Bucket: "b1", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}
|
||||
mpuKey := s3lifecycle.ActionKey{Bucket: "b1", RuleHash: rh, ActionKind: s3lifecycle.ActionKindAbortMPU}
|
||||
if snap.actions[expDaysKey].Mode != ModeScanOnly {
|
||||
t.Fatalf("ExpirationDays should be scan_only under 30d retention")
|
||||
}
|
||||
if snap.actions[mpuKey].Mode != ModeEventDriven {
|
||||
t.Fatalf("AbortMPU should stay event_driven (sibling degrades independently)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompile_PriorModePreservedOverDecideMode(t *testing.T) {
|
||||
// A durably-persisted SCAN_ONLY (or DISABLED, or any degraded mode)
|
||||
// must not be re-promoted to EVENT_DRIVEN by decideMode on every
|
||||
// Compile. Otherwise lag-fallback / operator pause / manual scan-only
|
||||
// don't survive an engine rebuild.
|
||||
rule := ruleExpDays("r", "x/", 30)
|
||||
rh := s3lifecycle.RuleHash(rule)
|
||||
key := s3lifecycle.ActionKey{Bucket: "b1", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}
|
||||
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{
|
||||
PriorStates: map[s3lifecycle.ActionKey]PriorState{
|
||||
key: {BootstrapComplete: true, Mode: ModeScanOnly},
|
||||
},
|
||||
})
|
||||
if snap.actions[key].Mode != ModeScanOnly {
|
||||
t.Fatalf("durable Mode=ScanOnly should win over decideMode, got %v", snap.actions[key].Mode)
|
||||
}
|
||||
if snap.actions[key].IsActive() {
|
||||
t.Fatalf("ScanOnly action must not be active")
|
||||
}
|
||||
// And: a missing PriorState falls through to decideMode as before.
|
||||
rule2 := ruleExpDays("fresh", "x/", 30)
|
||||
key2 := s3lifecycle.ActionKey{Bucket: "b1", RuleHash: s3lifecycle.RuleHash(rule2), ActionKind: s3lifecycle.ActionKindExpirationDays}
|
||||
snap2 := e.Compile([]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule2}}}, CompileOptions{})
|
||||
if snap2.actions[key2].Mode != ModeEventDriven {
|
||||
t.Fatalf("missing prior should fall through to decideMode (EventDriven), got %v", snap2.actions[key2].Mode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompile_ExpirationDateScansAtDate(t *testing.T) {
|
||||
date := time.Date(2025, 6, 15, 0, 0, 0, 0, time.UTC)
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "d",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
Prefix: "x/",
|
||||
ExpirationDate: date,
|
||||
}
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{
|
||||
{RuleHash: s3lifecycle.RuleHash(rule), ActionKind: s3lifecycle.ActionKindExpirationDate}: {BootstrapComplete: true},
|
||||
}
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{PriorStates: prior})
|
||||
if len(snap.dateActions) != 1 {
|
||||
t.Fatalf("want 1 date action, got %d", len(snap.dateActions))
|
||||
}
|
||||
for _, d := range snap.dateActions {
|
||||
if !d.Equal(date) {
|
||||
t.Fatalf("date want %v, got %v", date, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompile_DisabledRuleNeverActivates(t *testing.T) {
|
||||
rule := ruleExpDays("d", "x/", 30)
|
||||
rule.Status = s3lifecycle.StatusDisabled
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{
|
||||
{RuleHash: s3lifecycle.RuleHash(rule), ActionKind: s3lifecycle.ActionKindExpirationDays}: {BootstrapComplete: true},
|
||||
}
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{PriorStates: prior})
|
||||
for _, a := range snap.actions {
|
||||
if a.Mode != ModeDisabled || a.IsActive() {
|
||||
t.Fatalf("disabled rule must be ModeDisabled and inactive")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshot_MarkActiveFlipsRoutingFilter(t *testing.T) {
|
||||
rule := ruleExpDays("r", "x/", 30)
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{})
|
||||
key := s3lifecycle.ActionKey{Bucket: "b1", RuleHash: s3lifecycle.RuleHash(rule), ActionKind: s3lifecycle.ActionKindExpirationDays}
|
||||
|
||||
if snap.actions[key].IsActive() {
|
||||
t.Fatalf("should start inactive")
|
||||
}
|
||||
snap.MarkActive(key)
|
||||
if !snap.actions[key].IsActive() {
|
||||
t.Fatalf("MarkActive should flip the bit")
|
||||
}
|
||||
// MarkActive on a missing key is a no-op (stale callback after rebuild).
|
||||
snap.MarkActive(s3lifecycle.ActionKey{})
|
||||
}
|
||||
|
||||
func TestCompile_CrossBucketIdenticalRulesDoNotCollide(t *testing.T) {
|
||||
// Two buckets carry rules whose XML — and therefore RuleHash — is
|
||||
// identical. ActionKey must be bucket-scoped so the second bucket's
|
||||
// CompiledAction does not overwrite the first's in snap.actions.
|
||||
rule := ruleExpDays("shared", "x/", 30)
|
||||
rh := s3lifecycle.RuleHash(rule)
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{
|
||||
{Bucket: "alpha", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}: {BootstrapComplete: true},
|
||||
{Bucket: "beta", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}: {BootstrapComplete: true},
|
||||
}
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{
|
||||
{Bucket: "alpha", Rules: []*s3lifecycle.Rule{rule}},
|
||||
{Bucket: "beta", Rules: []*s3lifecycle.Rule{rule}},
|
||||
}, CompileOptions{PriorStates: prior})
|
||||
|
||||
if got := len(snap.actions); got != 2 {
|
||||
t.Fatalf("want 2 actions (one per bucket), got %d", got)
|
||||
}
|
||||
alphaKey := s3lifecycle.ActionKey{Bucket: "alpha", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}
|
||||
betaKey := s3lifecycle.ActionKey{Bucket: "beta", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}
|
||||
if snap.actions[alphaKey] == nil || snap.actions[alphaKey].Bucket != "alpha" {
|
||||
t.Fatalf("alpha bucket action missing or wrong bucket")
|
||||
}
|
||||
if snap.actions[betaKey] == nil || snap.actions[betaKey].Bucket != "beta" {
|
||||
t.Fatalf("beta bucket action missing or wrong bucket")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngine_SnapshotAtomicSwap(t *testing.T) {
|
||||
e := New()
|
||||
r1 := ruleExpDays("r1", "a/", 1)
|
||||
snap1 := e.Compile([]CompileInput{{Bucket: "b", Rules: []*s3lifecycle.Rule{r1}}}, CompileOptions{})
|
||||
if snap1.SnapshotID() == 0 {
|
||||
t.Fatalf("snapshot id should be > 0")
|
||||
}
|
||||
r2 := ruleExpDays("r2", "b/", 2)
|
||||
snap2 := e.Compile([]CompileInput{{Bucket: "b", Rules: []*s3lifecycle.Rule{r2}}}, CompileOptions{})
|
||||
if snap2.SnapshotID() <= snap1.SnapshotID() {
|
||||
t.Fatalf("snapshot id should be monotonic")
|
||||
}
|
||||
if e.Snapshot() != snap2 {
|
||||
t.Fatalf("Engine.Snapshot should return the latest")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCompile_DelayGroupsDedupeAcrossBuckets pins the scaling property
|
||||
// the dispatcher relies on: originalDelayGroups is keyed by Delay, so N
|
||||
// rules sharing the same Days threshold across M buckets collapse into
|
||||
// one delay group with N*M action keys — not N*M groups. The dispatch
|
||||
// path can then index by delay group rather than per-rule.
|
||||
func TestCompile_DelayGroupsDedupeAcrossBuckets(t *testing.T) {
|
||||
const buckets = 100
|
||||
const rulesPerBucket = 5
|
||||
// Five rules per bucket, but only three distinct day values, so two
|
||||
// pairs of rules per bucket share a delay group.
|
||||
dayValues := []int{1, 1, 7, 7, 30}
|
||||
distinctDays := map[int]bool{}
|
||||
for _, d := range dayValues {
|
||||
distinctDays[d] = true
|
||||
}
|
||||
|
||||
inputs := make([]CompileInput, buckets)
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{}
|
||||
for b := 0; b < buckets; b++ {
|
||||
bucket := "b" + strconv.Itoa(b)
|
||||
rules := make([]*s3lifecycle.Rule, rulesPerBucket)
|
||||
for r, days := range dayValues {
|
||||
rule := ruleExpDays("r"+strconv.Itoa(r), "p"+strconv.Itoa(r)+"/", days)
|
||||
rules[r] = rule
|
||||
prior[s3lifecycle.ActionKey{
|
||||
Bucket: bucket,
|
||||
RuleHash: s3lifecycle.RuleHash(rule),
|
||||
ActionKind: s3lifecycle.ActionKindExpirationDays,
|
||||
}] = PriorState{BootstrapComplete: true}
|
||||
}
|
||||
inputs[b] = CompileInput{Bucket: bucket, Rules: rules}
|
||||
}
|
||||
|
||||
e := New()
|
||||
snap := e.Compile(inputs, CompileOptions{PriorStates: prior})
|
||||
|
||||
if got := len(snap.actions); got != buckets*rulesPerBucket {
|
||||
t.Fatalf("want %d actions (no collapse), got %d", buckets*rulesPerBucket, got)
|
||||
}
|
||||
if got := len(snap.originalDelayGroups); got != len(distinctDays) {
|
||||
t.Fatalf("want %d delay groups (one per distinct Days), got %d",
|
||||
len(distinctDays), got)
|
||||
}
|
||||
// Per-group key count: each distinct day appears in `count` rules per
|
||||
// bucket, so the group holds count*buckets keys.
|
||||
wantPerGroup := map[time.Duration]int{}
|
||||
for _, d := range dayValues {
|
||||
wantPerGroup[s3lifecycle.DaysToDuration(d)] += buckets
|
||||
}
|
||||
for delay, want := range wantPerGroup {
|
||||
if got := len(snap.originalDelayGroups[delay]); got != want {
|
||||
t.Fatalf("delay %v: want %d action keys, got %d", delay, want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
205
weed/s3api/s3lifecycle/engine/hashes.go
Normal file
205
weed/s3api/s3lifecycle/engine/hashes.go
Normal file
@@ -0,0 +1,205 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"hash"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
)
|
||||
|
||||
// hashItem pairs a ruleset member with its parent action so the
|
||||
// sort+hash helpers below can be type-agnostic. Internal type — every
|
||||
// hash callsite collects items into a slice, then hands it off.
|
||||
type hashItem struct {
|
||||
key s3lifecycle.ActionKey
|
||||
action *CompiledAction
|
||||
}
|
||||
|
||||
// sortHashItems orders items by (RuleHash, ActionKind, Bucket). The
|
||||
// composition matches the ActionKey identity model: RuleHash is the
|
||||
// primary content-derived identifier, ActionKind disambiguates siblings
|
||||
// of one rule, Bucket scopes by bucket so the same XML in two buckets
|
||||
// hashes distinctly. Shared between ReplayContentHash and PromotedHash
|
||||
// so the two helpers agree on the on-wire ordering — if one drifted,
|
||||
// the cursor could see "rule changed" on a no-op snapshot rebuild.
|
||||
func sortHashItems(items []hashItem) {
|
||||
sort.Slice(items, func(i, j int) bool {
|
||||
if c := bytes.Compare(items[i].key.RuleHash[:], items[j].key.RuleHash[:]); c != 0 {
|
||||
return c < 0
|
||||
}
|
||||
if items[i].key.ActionKind != items[j].key.ActionKind {
|
||||
return items[i].key.ActionKind < items[j].key.ActionKind
|
||||
}
|
||||
return items[i].key.Bucket < items[j].key.Bucket
|
||||
})
|
||||
}
|
||||
|
||||
// hashWriter is the small varint-tagged writer the hash helpers use.
|
||||
// Each field gets a one-byte tag so a future schema change (new field)
|
||||
// can extend the on-wire format without colliding with existing values.
|
||||
type hashWriter struct {
|
||||
h hash.Hash
|
||||
lenbuf [binary.MaxVarintLen64]byte
|
||||
}
|
||||
|
||||
func newHashWriter() *hashWriter {
|
||||
return &hashWriter{h: sha256.New()}
|
||||
}
|
||||
|
||||
// writeField writes a length-prefixed byte field under tag.
|
||||
func (w *hashWriter) writeField(tag byte, b []byte) {
|
||||
_, _ = w.h.Write([]byte{tag})
|
||||
n := binary.PutUvarint(w.lenbuf[:], uint64(len(b)))
|
||||
_, _ = w.h.Write(w.lenbuf[:n])
|
||||
_, _ = w.h.Write(b)
|
||||
}
|
||||
|
||||
// writeInt writes a varint-encoded signed integer under tag.
|
||||
func (w *hashWriter) writeInt(tag byte, v int64) {
|
||||
_, _ = w.h.Write([]byte{tag})
|
||||
n := binary.PutVarint(w.lenbuf[:], v)
|
||||
_, _ = w.h.Write(w.lenbuf[:n])
|
||||
}
|
||||
|
||||
func (w *hashWriter) sum() [32]byte {
|
||||
var out [32]byte
|
||||
copy(out[:], w.h.Sum(nil))
|
||||
return out
|
||||
}
|
||||
|
||||
// ReplayContentHash hashes the content (action kind, predicate, TTL value)
|
||||
// of every replay-eligible compiled action in the base snapshot, returning
|
||||
// the empty hash when no replay-eligible action exists. The hash is:
|
||||
// - Partition-independent. A retention-driven scan_only promotion does
|
||||
// NOT change this hash; only the dispatch path changes, not the rule
|
||||
// content. (PromotedHash exists to catch partition flips separately.)
|
||||
// - Stable across snapshot reorderings. Actions are pre-sorted by
|
||||
// RuleHash + ActionKind + Bucket so two snapshots with the same rules
|
||||
// compiled in any order hash identically.
|
||||
// - Disabled-rule-aware. ModeDisabled actions are excluded so disabling
|
||||
// a rule changes the hash (it changed the rule set the worker is
|
||||
// scanning under).
|
||||
//
|
||||
// Used as cursor.RuleSetHash. A mismatch between persisted and current
|
||||
// triggers the recovery branch on next daily_run.
|
||||
func ReplayContentHash(s *Snapshot) [32]byte {
|
||||
var empty [32]byte
|
||||
if s == nil {
|
||||
return empty
|
||||
}
|
||||
var items []hashItem
|
||||
for k, a := range s.actions {
|
||||
if a == nil || a.Mode == ModeDisabled {
|
||||
continue
|
||||
}
|
||||
if !isReplayKind(k.ActionKind) {
|
||||
continue
|
||||
}
|
||||
items = append(items, hashItem{key: k, action: a})
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return empty
|
||||
}
|
||||
sortHashItems(items)
|
||||
|
||||
w := newHashWriter()
|
||||
for _, it := range items {
|
||||
w.writeField(0x01, []byte(it.key.Bucket))
|
||||
w.writeField(0x02, it.key.RuleHash[:])
|
||||
w.writeInt(0x03, int64(it.key.ActionKind))
|
||||
// RuleHash already covers the predicate (Prefix + FilterTags + size
|
||||
// filters) and per-kind TTLs, so we don't need to re-canonicalise
|
||||
// the *Rule. But we also include the action's effective TTL
|
||||
// directly so that an "effective TTL of 0" (a malformed rule where
|
||||
// the kind doesn't match the populated field) is distinguishable
|
||||
// from a valid one.
|
||||
w.writeInt(0x04, int64(effectiveTTL(it.action)))
|
||||
}
|
||||
return w.sum()
|
||||
}
|
||||
|
||||
// PromotedHash hashes the set of replay-eligible actions that *would* land
|
||||
// in walk (rather than replay) for the given retentionWindow, due to TTL >
|
||||
// retentionWindow. Empty hash when no rules are promoted. Takes the SAME
|
||||
// retentionWindow value as RulesForShard so the two helpers cannot disagree
|
||||
// about partition membership.
|
||||
//
|
||||
// Detects partition flips in either direction:
|
||||
// - replay → walk (retention dropped): rule appears in this hash but
|
||||
// didn't before.
|
||||
// - walk → replay (retention recovered): rule used to appear here but no
|
||||
// longer does.
|
||||
//
|
||||
// In both cases the persisted hash differs from the freshly computed one,
|
||||
// firing the recovery branch.
|
||||
//
|
||||
// A mismatch with the persisted PromotedHash triggers recovery even when
|
||||
// rule content is unchanged.
|
||||
func PromotedHash(s *Snapshot, retentionWindow time.Duration) [32]byte {
|
||||
var empty [32]byte
|
||||
if s == nil {
|
||||
return empty
|
||||
}
|
||||
var items []hashItem
|
||||
for k, a := range s.actions {
|
||||
if a == nil || a.Mode == ModeDisabled {
|
||||
continue
|
||||
}
|
||||
if !isReplayKind(k.ActionKind) {
|
||||
continue
|
||||
}
|
||||
ttl := effectiveTTL(a)
|
||||
// Mirror RulesForShard's partition predicate exactly: a replay
|
||||
// kind lands in walk when ttl is 0 (malformed) or ttl >
|
||||
// retentionWindow. PromotedHash hashes that walk-bound subset.
|
||||
if ttl > 0 && ttl <= retentionWindow {
|
||||
continue
|
||||
}
|
||||
items = append(items, hashItem{key: k, action: a})
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return empty
|
||||
}
|
||||
sortHashItems(items)
|
||||
|
||||
w := newHashWriter()
|
||||
for _, it := range items {
|
||||
w.writeField(0x01, []byte(it.key.Bucket))
|
||||
w.writeField(0x02, it.key.RuleHash[:])
|
||||
w.writeInt(0x03, int64(it.key.ActionKind))
|
||||
}
|
||||
return w.sum()
|
||||
}
|
||||
|
||||
// MaxEffectiveTTL returns the maximum effective TTL across the *active*
|
||||
// replay-eligible actions in s. Returns 0 for a nil snapshot or one with no
|
||||
// active replay actions; the caller is expected to be in the empty-replay
|
||||
// branch already (per the design's sentinel-cursor logic).
|
||||
//
|
||||
// "Effective TTL" mirrors the partition predicate in views.go: derived from
|
||||
// the rule field that matches the action kind. Walker-only action kinds
|
||||
// (ExpirationDate / ExpiredDeleteMarker / NewerNoncurrent) contribute
|
||||
// nothing — they're either not replay-eligible or, in a walk view, not
|
||||
// active in the replay sense.
|
||||
func MaxEffectiveTTL(s *Snapshot) time.Duration {
|
||||
if s == nil {
|
||||
return 0
|
||||
}
|
||||
var max time.Duration
|
||||
for k, a := range s.actions {
|
||||
if a == nil || !a.IsActive() {
|
||||
continue
|
||||
}
|
||||
if !isReplayKind(k.ActionKind) {
|
||||
continue
|
||||
}
|
||||
if ttl := effectiveTTL(a); ttl > max {
|
||||
max = ttl
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
273
weed/s3api/s3lifecycle/engine/hashes_test.go
Normal file
273
weed/s3api/s3lifecycle/engine/hashes_test.go
Normal file
@@ -0,0 +1,273 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestReplayContentHash_EmptyWhenNoReplayActions(t *testing.T) {
|
||||
var empty [32]byte
|
||||
// Nil snapshot.
|
||||
assert.Equal(t, empty, ReplayContentHash(nil))
|
||||
|
||||
// Walker-only rule: ExpirationDate compiles to a single ScanAtDate
|
||||
// action which is excluded from ReplayContentHash.
|
||||
when := time.Now().Add(24 * time.Hour)
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "d",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDate: when,
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
assert.Equal(t, empty, ReplayContentHash(snap))
|
||||
}
|
||||
|
||||
func TestReplayContentHash_PartitionIndependent(t *testing.T) {
|
||||
// Critical invariant: a retention shift that promotes a rule from
|
||||
// replay to walk (or back) MUST NOT change ReplayContentHash. The hash
|
||||
// is over the rule content as it appears in the base snapshot, not
|
||||
// over the partition it ends up in.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "shift",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 30,
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
hashWithLargeRetention := ReplayContentHash(snap)
|
||||
|
||||
// Whether retention is 1d (promotes to walk) or 365d (stays in
|
||||
// replay), the base snapshot's rule content didn't move — only its
|
||||
// partition did. ReplayContentHash hashes the base, so this is
|
||||
// trivially constant. We re-call here to pin the contract.
|
||||
_, walkSmall := snap.RulesForShard(0, s3lifecycle.DaysToDuration(1))
|
||||
require.NotNil(t, walkSmall, "precondition: 1d retention promotes 30d rule to walk")
|
||||
replayLarge, _ := snap.RulesForShard(0, s3lifecycle.DaysToDuration(365))
|
||||
require.NotNil(t, replayLarge, "precondition: 365d retention keeps 30d rule in replay")
|
||||
|
||||
hashAgain := ReplayContentHash(snap)
|
||||
assert.Equal(t, hashWithLargeRetention, hashAgain,
|
||||
"ReplayContentHash must be partition-independent")
|
||||
}
|
||||
|
||||
func TestReplayContentHash_ChangesOnRuleContentEdit(t *testing.T) {
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 30,
|
||||
}
|
||||
snap1 := buildSnapshotForViews(t, "b1", rule)
|
||||
h1 := ReplayContentHash(snap1)
|
||||
|
||||
// Edit the TTL — RuleHash changes, so the action's key changes, and
|
||||
// the content hash must move.
|
||||
rule2 := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 60,
|
||||
}
|
||||
snap2 := buildSnapshotForViews(t, "b1", rule2)
|
||||
h2 := ReplayContentHash(snap2)
|
||||
|
||||
assert.NotEqual(t, h1, h2, "TTL edit must change ReplayContentHash")
|
||||
}
|
||||
|
||||
func TestReplayContentHash_StableAcrossRuleReorder(t *testing.T) {
|
||||
r1 := &s3lifecycle.Rule{ID: "a", Status: s3lifecycle.StatusEnabled, ExpirationDays: 7, Prefix: "a/"}
|
||||
r2 := &s3lifecycle.Rule{ID: "b", Status: s3lifecycle.StatusEnabled, ExpirationDays: 14, Prefix: "b/"}
|
||||
snapAB := buildSnapshotForViews(t, "b1", r1, r2)
|
||||
snapBA := buildSnapshotForViews(t, "b1", r2, r1)
|
||||
assert.Equal(t, ReplayContentHash(snapAB), ReplayContentHash(snapBA),
|
||||
"reordering rules at compile time must not change ReplayContentHash")
|
||||
}
|
||||
|
||||
func TestReplayContentHash_ExcludesWalkerOnlyAndDisabled(t *testing.T) {
|
||||
// Walker-only kinds and disabled rules don't affect ReplayContentHash;
|
||||
// only replay-eligible action kinds count.
|
||||
replayRule := &s3lifecycle.Rule{
|
||||
ID: "rep",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 7,
|
||||
}
|
||||
snapOnlyReplay := buildSnapshotForViews(t, "b1", replayRule)
|
||||
baseline := ReplayContentHash(snapOnlyReplay)
|
||||
|
||||
walkerRule := &s3lifecycle.Rule{
|
||||
ID: "walker",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpiredObjectDeleteMarker: true,
|
||||
}
|
||||
disabledRule := &s3lifecycle.Rule{
|
||||
ID: "off",
|
||||
Status: s3lifecycle.StatusDisabled,
|
||||
ExpirationDays: 90,
|
||||
}
|
||||
snapMixed := buildSnapshotForViews(t, "b1", replayRule, walkerRule, disabledRule)
|
||||
mixed := ReplayContentHash(snapMixed)
|
||||
|
||||
assert.Equal(t, baseline, mixed,
|
||||
"adding walker-only and disabled rules must not change ReplayContentHash")
|
||||
}
|
||||
|
||||
func TestPromotedHash_EmptyWhenNoneAreInWalk(t *testing.T) {
|
||||
var empty [32]byte
|
||||
assert.Equal(t, empty, PromotedHash(nil, 0))
|
||||
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "fits",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 7,
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
// Large retention → rule stays in replay → no promotion.
|
||||
assert.Equal(t, empty, PromotedHash(snap, s3lifecycle.DaysToDuration(365)))
|
||||
}
|
||||
|
||||
func TestPromotedHash_ChangesOnReplayToWalkPromotion(t *testing.T) {
|
||||
// Retention drops below the rule's TTL → rule promotes from replay to
|
||||
// walk → PromotedHash changes from empty to non-empty.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "long",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 30,
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
|
||||
before := PromotedHash(snap, s3lifecycle.DaysToDuration(365)) // replay
|
||||
after := PromotedHash(snap, s3lifecycle.DaysToDuration(7)) // promoted to walk
|
||||
|
||||
var empty [32]byte
|
||||
assert.Equal(t, empty, before)
|
||||
assert.NotEqual(t, empty, after)
|
||||
assert.NotEqual(t, before, after, "promotion must change PromotedHash")
|
||||
}
|
||||
|
||||
func TestPromotedHash_ChangesOnWalkToReplayDemotion(t *testing.T) {
|
||||
// Retention recovers from < TTL to >= TTL → rule demotes from walk to
|
||||
// replay → PromotedHash changes from non-empty to empty.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "demo",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 30,
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
|
||||
before := PromotedHash(snap, s3lifecycle.DaysToDuration(7)) // walk
|
||||
after := PromotedHash(snap, s3lifecycle.DaysToDuration(365)) // demoted back to replay
|
||||
|
||||
var empty [32]byte
|
||||
assert.NotEqual(t, empty, before)
|
||||
assert.Equal(t, empty, after)
|
||||
assert.NotEqual(t, before, after, "demotion must change PromotedHash")
|
||||
}
|
||||
|
||||
func TestPromotedHash_StableWhenContentUnchangedAndPartitionStays(t *testing.T) {
|
||||
// Same snapshot + same retentionWindow → same PromotedHash.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "stable",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 30,
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
rw := s3lifecycle.DaysToDuration(7) // promoted
|
||||
assert.Equal(t, PromotedHash(snap, rw), PromotedHash(snap, rw))
|
||||
}
|
||||
|
||||
func TestPromotedHash_MatchesRulesForShardWalkMembership(t *testing.T) {
|
||||
// PromotedHash must agree with RulesForShard about which replay-
|
||||
// eligible actions are in walk for the same retentionWindow.
|
||||
rules := []*s3lifecycle.Rule{
|
||||
{ID: "short", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1},
|
||||
{ID: "long", Status: s3lifecycle.StatusEnabled, ExpirationDays: 90},
|
||||
{ID: "noncurrent", Status: s3lifecycle.StatusEnabled, NoncurrentVersionExpirationDays: 200},
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rules...)
|
||||
rw := s3lifecycle.DaysToDuration(30)
|
||||
|
||||
replay, walk := snap.RulesForShard(0, rw)
|
||||
require.NotNil(t, replay)
|
||||
require.NotNil(t, walk)
|
||||
|
||||
// Expected walk members (from replay-eligible kinds only): long, noncurrent.
|
||||
wantPromoted := map[s3lifecycle.ActionKey]struct{}{}
|
||||
for k := range walk.actions {
|
||||
if isReplayKind(k.ActionKind) {
|
||||
wantPromoted[k] = struct{}{}
|
||||
}
|
||||
}
|
||||
assert.Len(t, wantPromoted, 2, "two of three replay-eligible rules should promote")
|
||||
|
||||
hash := PromotedHash(snap, rw)
|
||||
var empty [32]byte
|
||||
assert.NotEqual(t, empty, hash)
|
||||
}
|
||||
|
||||
func TestMaxEffectiveTTL_NilAndEmpty(t *testing.T) {
|
||||
assert.Equal(t, time.Duration(0), MaxEffectiveTTL(nil))
|
||||
|
||||
// Snapshot with only walker-only kinds → no active replay action →
|
||||
// MaxEffectiveTTL returns 0.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "walk",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpiredObjectDeleteMarker: true,
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
assert.Equal(t, time.Duration(0), MaxEffectiveTTL(snap))
|
||||
}
|
||||
|
||||
func TestMaxEffectiveTTL_ReturnsMaxAcrossActiveReplay(t *testing.T) {
|
||||
// Three replay-eligible actions with mixed TTLs; MaxEffectiveTTL picks
|
||||
// the largest.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "mix",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 10,
|
||||
NoncurrentVersionExpirationDays: 60,
|
||||
AbortMPUDaysAfterInitiation: 3,
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
got := MaxEffectiveTTL(snap)
|
||||
assert.Equal(t, s3lifecycle.DaysToDuration(60), got)
|
||||
}
|
||||
|
||||
func TestMaxEffectiveTTL_OperatesOnReplayView(t *testing.T) {
|
||||
// The caller passes the *replay* snapshot here in production. Verify
|
||||
// that on a replay view the answer is the max over the view's
|
||||
// (active) replay actions and excludes anything that was routed away
|
||||
// to walk via scan_only promotion.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "mix",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 10,
|
||||
NoncurrentVersionExpirationDays: 200, // would promote at small retention
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
// Retention 30d: ExpirationDays(10) stays in replay; NoncurrentDays(200)
|
||||
// promotes to walk. The replay view's MaxEffectiveTTL is therefore 10d.
|
||||
replay, _ := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
|
||||
require.NotNil(t, replay)
|
||||
assert.Equal(t, s3lifecycle.DaysToDuration(10), MaxEffectiveTTL(replay))
|
||||
}
|
||||
|
||||
func TestMaxEffectiveTTL_IgnoresInactiveActions(t *testing.T) {
|
||||
// Pre-BootstrapComplete event-driven action is in the snapshot but
|
||||
// inactive. MaxEffectiveTTL only counts active actions.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "pending",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 30,
|
||||
}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{}, // no PriorState → inactive
|
||||
)
|
||||
for _, a := range snap.actions {
|
||||
require.False(t, a.IsActive(), "precondition")
|
||||
}
|
||||
assert.Equal(t, time.Duration(0), MaxEffectiveTTL(snap),
|
||||
"inactive replay actions must not contribute to MaxEffectiveTTL")
|
||||
}
|
||||
134
weed/s3api/s3lifecycle/engine/match.go
Normal file
134
weed/s3api/s3lifecycle/engine/match.go
Normal file
@@ -0,0 +1,134 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
)
|
||||
|
||||
type EventShape int
|
||||
|
||||
const (
|
||||
EventShapeUnknown EventShape = iota
|
||||
// EventShapeOriginalWrite: any event resetting AWS LastModified
|
||||
// (fresh PUT, overwrite, MPU init, version flip).
|
||||
EventShapeOriginalWrite
|
||||
// EventShapePredicateChange: tags / Extended changed without an mtime
|
||||
// reset.
|
||||
EventShapePredicateChange
|
||||
)
|
||||
|
||||
// Event is the routing-relevant slice of a meta-log event. Kept minimal so
|
||||
// the engine doesn't depend on filer_pb.
|
||||
type Event struct {
|
||||
Shape EventShape
|
||||
Bucket string
|
||||
Path string
|
||||
Tags map[string]string
|
||||
Size int64
|
||||
IsLatest bool
|
||||
IsDeleteMarker bool
|
||||
IsMPUInit bool
|
||||
EventTime time.Time
|
||||
}
|
||||
|
||||
// MatchOriginalWrite returns active ActionKeys in the given delay group
|
||||
// whose filter matches the event.
|
||||
func (s *Snapshot) MatchOriginalWrite(ev *Event, delay time.Duration) []s3lifecycle.ActionKey {
|
||||
if ev == nil || ev.Shape != EventShapeOriginalWrite {
|
||||
return nil
|
||||
}
|
||||
keys := s.originalDelayGroups[delay]
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
return s.filterMatching(keys, ev)
|
||||
}
|
||||
|
||||
// MatchPredicateChange returns active predicate-sensitive ActionKeys whose
|
||||
// filter matches the event.
|
||||
func (s *Snapshot) MatchPredicateChange(ev *Event) []s3lifecycle.ActionKey {
|
||||
if ev == nil || ev.Shape != EventShapePredicateChange {
|
||||
return nil
|
||||
}
|
||||
if len(s.predicateActions) == 0 {
|
||||
return nil
|
||||
}
|
||||
return s.filterMatching(s.predicateActions, ev)
|
||||
}
|
||||
|
||||
// MatchPath returns active ActionKeys for a specific bucket+path. ev=nil
|
||||
// applies prefix matching only; pass a non-nil Event to also gate on tags
|
||||
// and size.
|
||||
func (s *Snapshot) MatchPath(bucket, path string, ev *Event) []s3lifecycle.ActionKey {
|
||||
bi := s.buckets[bucket]
|
||||
if bi == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]s3lifecycle.ActionKey, 0, len(bi.actionKeys))
|
||||
for _, k := range bi.actionKeys {
|
||||
a := s.actions[k]
|
||||
if a == nil || !a.IsActive() {
|
||||
continue
|
||||
}
|
||||
if !prefixMatches(a.Rule.Prefix, path) {
|
||||
continue
|
||||
}
|
||||
if ev != nil && !filterAllows(a.Rule, ev) {
|
||||
continue
|
||||
}
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Snapshot) filterMatching(keys []s3lifecycle.ActionKey, ev *Event) []s3lifecycle.ActionKey {
|
||||
out := make([]s3lifecycle.ActionKey, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
a := s.actions[k]
|
||||
if a == nil || !a.IsActive() || a.Bucket != ev.Bucket {
|
||||
continue
|
||||
}
|
||||
if !prefixMatches(a.Rule.Prefix, ev.Path) {
|
||||
continue
|
||||
}
|
||||
if !filterAllows(a.Rule, ev) {
|
||||
continue
|
||||
}
|
||||
switch k.ActionKind {
|
||||
case s3lifecycle.ActionKindAbortMPU:
|
||||
if !ev.IsMPUInit {
|
||||
continue
|
||||
}
|
||||
case s3lifecycle.ActionKindExpiredDeleteMarker:
|
||||
if !ev.IsDeleteMarker || !ev.IsLatest {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func prefixMatches(rulePrefix, path string) bool {
|
||||
if rulePrefix == "" {
|
||||
return true
|
||||
}
|
||||
return strings.HasPrefix(path, rulePrefix)
|
||||
}
|
||||
|
||||
func filterAllows(rule *s3lifecycle.Rule, ev *Event) bool {
|
||||
if rule.FilterSizeGreaterThan > 0 && ev.Size <= rule.FilterSizeGreaterThan {
|
||||
return false
|
||||
}
|
||||
if rule.FilterSizeLessThan > 0 && ev.Size >= rule.FilterSizeLessThan {
|
||||
return false
|
||||
}
|
||||
for k, v := range rule.FilterTags {
|
||||
if got, ok := ev.Tags[k]; !ok || got != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
321
weed/s3api/s3lifecycle/engine/match_helpers_test.go
Normal file
321
weed/s3api/s3lifecycle/engine/match_helpers_test.go
Normal file
@@ -0,0 +1,321 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// prefixMatches and filterAllows are pure helpers exercised indirectly
|
||||
// through MatchOriginalWrite / MatchPath; pinning them directly catches
|
||||
// a regression at the helper level so a Match-test failure isn't the
|
||||
// first signal of a broken filter.
|
||||
|
||||
func TestPrefixMatches_EmptyPrefixAllowsEverything(t *testing.T) {
|
||||
// An empty prefix on a rule means "match every key in the bucket"
|
||||
// — the no-op fast path that skips the HasPrefix call.
|
||||
assert.True(t, prefixMatches("", "logs/2026/01/01"))
|
||||
assert.True(t, prefixMatches("", ""))
|
||||
assert.True(t, prefixMatches("", "/"))
|
||||
}
|
||||
|
||||
func TestPrefixMatches_HonorsExactPrefix(t *testing.T) {
|
||||
assert.True(t, prefixMatches("logs/", "logs/2026/01/01"))
|
||||
assert.True(t, prefixMatches("logs/", "logs/"))
|
||||
}
|
||||
|
||||
func TestPrefixMatches_RejectsNonMatchingPath(t *testing.T) {
|
||||
assert.False(t, prefixMatches("logs/", "metrics/2026/01/01"))
|
||||
assert.False(t, prefixMatches("logs/", "log/2026/01/01")) // no trailing slash
|
||||
}
|
||||
|
||||
func TestPrefixMatches_PathShorterThanPrefixIsReject(t *testing.T) {
|
||||
// Path shorter than the rule prefix can't satisfy HasPrefix; the
|
||||
// helper must say no rather than treat it as a partial match.
|
||||
assert.False(t, prefixMatches("logs/2026/", "logs/"))
|
||||
}
|
||||
|
||||
func TestFilterAllows_NoFiltersAcceptsAnything(t *testing.T) {
|
||||
// A rule without any filter knobs allows every event regardless of
|
||||
// size or tag map. Pins the documented "no filter" fast path.
|
||||
rule := &s3lifecycle.Rule{}
|
||||
assert.True(t, filterAllows(rule, &Event{Size: 0}))
|
||||
assert.True(t, filterAllows(rule, &Event{Size: 1024}))
|
||||
assert.True(t, filterAllows(rule, &Event{Tags: map[string]string{"a": "b"}}))
|
||||
}
|
||||
|
||||
func TestFilterAllows_SizeGreaterThanGate(t *testing.T) {
|
||||
// FilterSizeGreaterThan is "strictly greater"; events at or below
|
||||
// the threshold are rejected.
|
||||
rule := &s3lifecycle.Rule{FilterSizeGreaterThan: 1024}
|
||||
assert.False(t, filterAllows(rule, &Event{Size: 1024}), "boundary value rejected (gate is >, not >=)")
|
||||
assert.False(t, filterAllows(rule, &Event{Size: 100}))
|
||||
assert.True(t, filterAllows(rule, &Event{Size: 1025}))
|
||||
}
|
||||
|
||||
func TestFilterAllows_SizeLessThanGate(t *testing.T) {
|
||||
// FilterSizeLessThan is "strictly less"; events at or above the
|
||||
// threshold are rejected.
|
||||
rule := &s3lifecycle.Rule{FilterSizeLessThan: 1024}
|
||||
assert.False(t, filterAllows(rule, &Event{Size: 1024}), "boundary value rejected (gate is <, not <=)")
|
||||
assert.False(t, filterAllows(rule, &Event{Size: 4096}))
|
||||
assert.True(t, filterAllows(rule, &Event{Size: 1023}))
|
||||
}
|
||||
|
||||
func TestFilterAllows_ZeroSizeFilterIsDisabled(t *testing.T) {
|
||||
// FilterSizeGreaterThan/LessThan = 0 means "not set"; the filter
|
||||
// must let any size through. A regression treating 0 as a real
|
||||
// threshold would reject every event at or above 0 (i.e., all of
|
||||
// them).
|
||||
greater := &s3lifecycle.Rule{FilterSizeGreaterThan: 0}
|
||||
less := &s3lifecycle.Rule{FilterSizeLessThan: 0}
|
||||
for _, size := range []int64{0, 1, 1024, 1 << 30} {
|
||||
assert.True(t, filterAllows(greater, &Event{Size: size}), "Gt 0 must allow size %d", size)
|
||||
assert.True(t, filterAllows(less, &Event{Size: size}), "Lt 0 must allow size %d", size)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterAllows_RequiredTagPresent(t *testing.T) {
|
||||
// A FilterTags entry is "must equal"; the event's Tags map must
|
||||
// have the same key with the same value.
|
||||
rule := &s3lifecycle.Rule{FilterTags: map[string]string{"env": "prod"}}
|
||||
assert.True(t, filterAllows(rule, &Event{Tags: map[string]string{"env": "prod"}}))
|
||||
assert.True(t, filterAllows(rule, &Event{Tags: map[string]string{"env": "prod", "team": "a"}}))
|
||||
}
|
||||
|
||||
func TestFilterAllows_RequiredTagMissingOrWrong(t *testing.T) {
|
||||
rule := &s3lifecycle.Rule{FilterTags: map[string]string{"env": "prod"}}
|
||||
assert.False(t, filterAllows(rule, &Event{}), "nil tags map rejected")
|
||||
assert.False(t, filterAllows(rule, &Event{Tags: map[string]string{}}), "empty tags rejected")
|
||||
assert.False(t, filterAllows(rule, &Event{Tags: map[string]string{"team": "a"}}), "different key rejected")
|
||||
assert.False(t, filterAllows(rule, &Event{Tags: map[string]string{"env": "stage"}}), "wrong value rejected")
|
||||
}
|
||||
|
||||
func TestFilterAllows_AllRequiredTagsMustMatch(t *testing.T) {
|
||||
// Multiple FilterTags are AND'd; missing any one rejects.
|
||||
rule := &s3lifecycle.Rule{FilterTags: map[string]string{"env": "prod", "team": "data"}}
|
||||
assert.True(t, filterAllows(rule, &Event{Tags: map[string]string{"env": "prod", "team": "data"}}))
|
||||
assert.False(t, filterAllows(rule, &Event{Tags: map[string]string{"env": "prod"}}), "missing one tag rejected")
|
||||
assert.False(t, filterAllows(rule, &Event{Tags: map[string]string{"team": "data"}}), "missing other tag rejected")
|
||||
assert.False(t, filterAllows(rule, &Event{Tags: map[string]string{"env": "prod", "team": "ops"}}), "wrong value on second tag rejected")
|
||||
}
|
||||
|
||||
func TestFilterAllows_SizeAndTagAreANDed(t *testing.T) {
|
||||
// A rule with both size and tag filters requires BOTH to pass;
|
||||
// either one failing rejects the event.
|
||||
rule := &s3lifecycle.Rule{
|
||||
FilterSizeGreaterThan: 100,
|
||||
FilterTags: map[string]string{"env": "prod"},
|
||||
}
|
||||
// Both pass.
|
||||
assert.True(t, filterAllows(rule, &Event{Size: 200, Tags: map[string]string{"env": "prod"}}))
|
||||
// Size fails.
|
||||
assert.False(t, filterAllows(rule, &Event{Size: 50, Tags: map[string]string{"env": "prod"}}))
|
||||
// Tag fails.
|
||||
assert.False(t, filterAllows(rule, &Event{Size: 200, Tags: map[string]string{"env": "stage"}}))
|
||||
}
|
||||
|
||||
// ---------- MatchOriginalWrite / MatchPredicateChange / MatchPath edge cases ----------
|
||||
|
||||
// helperSnapshot constructs a Snapshot via the engine's Compile path so
|
||||
// the action map and bucket index are populated correctly. Tests below
|
||||
// poke at the public Match functions rather than the internal maps.
|
||||
func helperSnapshot(t *testing.T) (*Snapshot, *s3lifecycle.Rule, s3lifecycle.ActionKey) {
|
||||
t.Helper()
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
Prefix: "logs/",
|
||||
ExpirationDays: 7,
|
||||
}
|
||||
hash := s3lifecycle.RuleHash(rule)
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{}
|
||||
for _, k := range s3lifecycle.RuleActionKinds(rule) {
|
||||
prior[s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: k}] = PriorState{
|
||||
BootstrapComplete: true,
|
||||
Mode: ModeEventDriven,
|
||||
}
|
||||
}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{PriorStates: prior},
|
||||
)
|
||||
key := s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: s3lifecycle.ActionKindExpirationDays}
|
||||
return snap, rule, key
|
||||
}
|
||||
|
||||
func TestMatchOriginalWrite_NilEventReturnsNil(t *testing.T) {
|
||||
// A nil event is treated as "no match"; the routing path should
|
||||
// not panic on a defensive caller.
|
||||
snap, _, _ := helperSnapshot(t)
|
||||
assert.Nil(t, snap.MatchOriginalWrite(nil, 7*24*time.Hour))
|
||||
}
|
||||
|
||||
func TestMatchOriginalWrite_WrongShapeReturnsNil(t *testing.T) {
|
||||
// Only EventShapeOriginalWrite goes through this path; a
|
||||
// PredicateChange event must not produce delay-group matches.
|
||||
snap, _, _ := helperSnapshot(t)
|
||||
got := snap.MatchOriginalWrite(&Event{Shape: EventShapePredicateChange, Bucket: "bk"}, 7*24*time.Hour)
|
||||
assert.Nil(t, got)
|
||||
}
|
||||
|
||||
func TestMatchOriginalWrite_DelayMismatchReturnsNil(t *testing.T) {
|
||||
// The action is in the 7-day delay group; asking for a different
|
||||
// delay must produce no matches.
|
||||
snap, _, _ := helperSnapshot(t)
|
||||
got := snap.MatchOriginalWrite(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "logs/x"}, 30*24*time.Hour)
|
||||
assert.Empty(t, got)
|
||||
}
|
||||
|
||||
func TestMatchPredicateChange_NilEventReturnsNil(t *testing.T) {
|
||||
snap, _, _ := helperSnapshot(t)
|
||||
assert.Nil(t, snap.MatchPredicateChange(nil))
|
||||
}
|
||||
|
||||
func TestMatchPredicateChange_WrongShapeReturnsNil(t *testing.T) {
|
||||
// EventShapeOriginalWrite events go through MatchOriginalWrite; a
|
||||
// caller passing one to MatchPredicateChange gets nothing back.
|
||||
snap, _, _ := helperSnapshot(t)
|
||||
got := snap.MatchPredicateChange(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk"})
|
||||
assert.Nil(t, got)
|
||||
}
|
||||
|
||||
func TestMatchPredicateChange_NoTagSensitiveActionsReturnsNil(t *testing.T) {
|
||||
// The default helperSnapshot rule has no FilterTags, so it's not
|
||||
// tag-sensitive — predicate changes must yield no matches.
|
||||
snap, _, _ := helperSnapshot(t)
|
||||
got := snap.MatchPredicateChange(&Event{Shape: EventShapePredicateChange, Bucket: "bk", Path: "logs/x"})
|
||||
assert.Nil(t, got, "non-tag-sensitive snapshot returns nil from predicate change")
|
||||
}
|
||||
|
||||
func TestMatchPath_UnknownBucketReturnsNil(t *testing.T) {
|
||||
snap, _, _ := helperSnapshot(t)
|
||||
got := snap.MatchPath("other-bucket", "logs/x", nil)
|
||||
assert.Nil(t, got)
|
||||
}
|
||||
|
||||
func TestMatchPath_NilEventStillAppliesPrefix(t *testing.T) {
|
||||
// MatchPath with ev=nil applies prefix matching only; tags/size
|
||||
// don't gate. Pinned because the bootstrap walker calls it that
|
||||
// way.
|
||||
snap, _, key := helperSnapshot(t)
|
||||
matches := snap.MatchPath("bk", "logs/2026/01/01", nil)
|
||||
assert.Contains(t, matches, key)
|
||||
assert.Empty(t, snap.MatchPath("bk", "metrics/x", nil), "prefix mismatch must reject even with nil event")
|
||||
}
|
||||
|
||||
func TestMatchPath_FilterRespectedWhenEventProvided(t *testing.T) {
|
||||
// When the caller supplies an Event, both prefix and filter gates
|
||||
// fire. A size-filter rule rejects an event below the threshold.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 7,
|
||||
FilterSizeGreaterThan: 1000,
|
||||
}
|
||||
hash := s3lifecycle.RuleHash(rule)
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{}
|
||||
for _, k := range s3lifecycle.RuleActionKinds(rule) {
|
||||
prior[s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: k}] = PriorState{
|
||||
BootstrapComplete: true,
|
||||
Mode: ModeEventDriven,
|
||||
}
|
||||
}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{PriorStates: prior},
|
||||
)
|
||||
belowThreshold := &Event{Size: 500}
|
||||
aboveThreshold := &Event{Size: 2000}
|
||||
assert.Empty(t, snap.MatchPath("bk", "x", belowThreshold), "size below threshold must reject")
|
||||
assert.NotEmpty(t, snap.MatchPath("bk", "x", aboveThreshold), "size above threshold must allow")
|
||||
}
|
||||
|
||||
// ---------- filterMatching direct coverage ----------
|
||||
|
||||
func TestFilterMatching_BucketScoping(t *testing.T) {
|
||||
// Two buckets sharing identical rules: filterMatching must reject
|
||||
// keys whose action.Bucket doesn't match ev.Bucket so a match in
|
||||
// "alpha" doesn't bleed into "beta".
|
||||
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 7}
|
||||
hash := s3lifecycle.RuleHash(rule)
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{}
|
||||
for _, b := range []string{"alpha", "beta"} {
|
||||
for _, k := range s3lifecycle.RuleActionKinds(rule) {
|
||||
prior[s3lifecycle.ActionKey{Bucket: b, RuleHash: hash, ActionKind: k}] = PriorState{
|
||||
BootstrapComplete: true,
|
||||
Mode: ModeEventDriven,
|
||||
}
|
||||
}
|
||||
}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{
|
||||
{Bucket: "alpha", Rules: []*s3lifecycle.Rule{rule}},
|
||||
{Bucket: "beta", Rules: []*s3lifecycle.Rule{rule}},
|
||||
},
|
||||
CompileOptions{PriorStates: prior},
|
||||
)
|
||||
got := snap.MatchOriginalWrite(&Event{
|
||||
Shape: EventShapeOriginalWrite, Bucket: "alpha", Path: "x", Size: 1,
|
||||
}, 7*24*time.Hour)
|
||||
require.NotEmpty(t, got)
|
||||
for _, k := range got {
|
||||
assert.Equal(t, "alpha", k.Bucket, "match leaked across bucket")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterMatching_AbortMPURequiresMPUInit(t *testing.T) {
|
||||
// AbortMPU actions only fire on MPU-init events; a normal write to
|
||||
// the same prefix must not trigger them.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
AbortMPUDaysAfterInitiation: 7,
|
||||
}
|
||||
hash := s3lifecycle.RuleHash(rule)
|
||||
abortKey := s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: s3lifecycle.ActionKindAbortMPU}
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{
|
||||
abortKey: {BootstrapComplete: true, Mode: ModeEventDriven},
|
||||
}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{PriorStates: prior},
|
||||
)
|
||||
regular := &Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "x", IsMPUInit: false}
|
||||
mpu := &Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "x", IsMPUInit: true}
|
||||
assert.Empty(t, snap.MatchOriginalWrite(regular, 7*24*time.Hour), "regular write must not trigger AbortMPU")
|
||||
assert.Contains(t, snap.MatchOriginalWrite(mpu, 7*24*time.Hour), abortKey, "MPU init must trigger AbortMPU")
|
||||
}
|
||||
|
||||
func TestFilterMatching_ExpiredDeleteMarkerRequiresMarkerLatest(t *testing.T) {
|
||||
// EXPIRED_DELETE_MARKER actions only fire when the event is both a
|
||||
// delete marker AND the current latest version. A non-delete event
|
||||
// or a noncurrent delete marker must not trigger.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpiredObjectDeleteMarker: true,
|
||||
}
|
||||
hash := s3lifecycle.RuleHash(rule)
|
||||
markerKey := s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: s3lifecycle.ActionKindExpiredDeleteMarker}
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{
|
||||
markerKey: {BootstrapComplete: true, Mode: ModeEventDriven},
|
||||
}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{PriorStates: prior},
|
||||
)
|
||||
// ExpiredDeleteMarker has MinTriggerAge=0; the action lives in the
|
||||
// zero-duration delay group, so the dispatcher polls it on every
|
||||
// tick.
|
||||
delay := time.Duration(0)
|
||||
notMarker := &Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "x", IsDeleteMarker: false, IsLatest: true}
|
||||
notLatest := &Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "x", IsDeleteMarker: true, IsLatest: false}
|
||||
bothTrue := &Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "x", IsDeleteMarker: true, IsLatest: true}
|
||||
assert.Empty(t, snap.MatchOriginalWrite(notMarker, delay), "non-marker must not trigger")
|
||||
assert.Empty(t, snap.MatchOriginalWrite(notLatest, delay), "noncurrent marker must not trigger")
|
||||
assert.Contains(t, snap.MatchOriginalWrite(bothTrue, delay), markerKey, "marker that is also latest must trigger")
|
||||
}
|
||||
194
weed/s3api/s3lifecycle/engine/match_test.go
Normal file
194
weed/s3api/s3lifecycle/engine/match_test.go
Normal file
@@ -0,0 +1,194 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
)
|
||||
|
||||
// activeAll returns a PriorStates map that activates every action of the
|
||||
// given canonical rules under bucket.
|
||||
func activeAll(bucket string, rules []*s3lifecycle.Rule) map[s3lifecycle.ActionKey]PriorState {
|
||||
out := map[s3lifecycle.ActionKey]PriorState{}
|
||||
for _, r := range rules {
|
||||
rh := s3lifecycle.RuleHash(r)
|
||||
for _, k := range s3lifecycle.RuleActionKinds(r) {
|
||||
out[s3lifecycle.ActionKey{Bucket: bucket, RuleHash: rh, ActionKind: k}] = PriorState{BootstrapComplete: true}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sortedKeys(keys []s3lifecycle.ActionKey) []s3lifecycle.ActionKey {
|
||||
out := append([]s3lifecycle.ActionKey(nil), keys...)
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
return out[i].ActionKind < out[j].ActionKind
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func TestMatchOriginalWrite_DelayGroupRoutes(t *testing.T) {
|
||||
r30 := ruleExpDays("a", "x/", 30)
|
||||
r60 := ruleExpDays("b", "x/", 60)
|
||||
rules := []*s3lifecycle.Rule{r30, r60}
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "bk", Rules: rules}}, CompileOptions{PriorStates: activeAll("bk", rules)})
|
||||
|
||||
ev := &Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "x/o"}
|
||||
got30 := snap.MatchOriginalWrite(ev, s3lifecycle.DaysToDuration(30))
|
||||
if len(got30) != 1 || got30[0].ActionKind != s3lifecycle.ActionKindExpirationDays {
|
||||
t.Fatalf("30d sweep should match r30, got %v", got30)
|
||||
}
|
||||
got60 := snap.MatchOriginalWrite(ev, s3lifecycle.DaysToDuration(60))
|
||||
if len(got60) != 1 {
|
||||
t.Fatalf("60d sweep should match r60, got %v", got60)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchOriginalWrite_PrefixFilter(t *testing.T) {
|
||||
r := ruleExpDays("a", "logs/", 30)
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{r}}}, CompileOptions{PriorStates: activeAll("bk", []*s3lifecycle.Rule{r})})
|
||||
|
||||
if got := snap.MatchOriginalWrite(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "data/x"}, s3lifecycle.DaysToDuration(30)); len(got) != 0 {
|
||||
t.Fatalf("non-matching prefix should reject, got %v", got)
|
||||
}
|
||||
if got := snap.MatchOriginalWrite(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "logs/x"}, s3lifecycle.DaysToDuration(30)); len(got) != 1 {
|
||||
t.Fatalf("matching prefix should fire, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchOriginalWrite_MarkActiveBecomesRoutable(t *testing.T) {
|
||||
// pending_bootstrap actions are indexed but inactive: MatchOriginalWrite
|
||||
// returns nothing. After MarkActive flips engineState, the same key is
|
||||
// routable in subsequent matches without a recompile.
|
||||
r := ruleExpDays("a", "x/", 30)
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{r}}}, CompileOptions{})
|
||||
ev := &Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "x/o"}
|
||||
if got := snap.MatchOriginalWrite(ev, s3lifecycle.DaysToDuration(30)); len(got) != 0 {
|
||||
t.Fatalf("inactive action should not match, got %v", got)
|
||||
}
|
||||
snap.MarkActive(s3lifecycle.ActionKey{Bucket: "bk", RuleHash: s3lifecycle.RuleHash(r), ActionKind: s3lifecycle.ActionKindExpirationDays})
|
||||
if got := snap.MatchOriginalWrite(ev, s3lifecycle.DaysToDuration(30)); len(got) != 1 {
|
||||
t.Fatalf("post-markActive should be routable, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchOriginalWrite_AbortMPUOnlyOnMPUInit(t *testing.T) {
|
||||
r := &s3lifecycle.Rule{
|
||||
ID: "mpu",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
Prefix: ".uploads/",
|
||||
AbortMPUDaysAfterInitiation: 7,
|
||||
}
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{r}}}, CompileOptions{PriorStates: activeAll("bk", []*s3lifecycle.Rule{r})})
|
||||
|
||||
// Non-MPU event under .uploads/ prefix: filtered out by shape gating.
|
||||
got := snap.MatchOriginalWrite(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: ".uploads/u1/", IsMPUInit: false}, s3lifecycle.DaysToDuration(7))
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("non-MPU event should be filtered, got %v", got)
|
||||
}
|
||||
got = snap.MatchOriginalWrite(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: ".uploads/u1/", IsMPUInit: true}, s3lifecycle.DaysToDuration(7))
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("MPU init should fire, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchPredicateChange_OnlyTagSensitiveActions(t *testing.T) {
|
||||
rTag := &s3lifecycle.Rule{
|
||||
ID: "tag",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
Prefix: "x/",
|
||||
ExpirationDays: 30,
|
||||
FilterTags: map[string]string{"env": "prod"},
|
||||
}
|
||||
rPlain := ruleExpDays("plain", "x/", 30)
|
||||
rules := []*s3lifecycle.Rule{rTag, rPlain}
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "bk", Rules: rules}}, CompileOptions{PriorStates: activeAll("bk", rules)})
|
||||
|
||||
// Predicate-change event: should match only the predicate-sensitive rule.
|
||||
ev := &Event{Shape: EventShapePredicateChange, Bucket: "bk", Path: "x/o", Tags: map[string]string{"env": "prod"}}
|
||||
got := snap.MatchPredicateChange(ev)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("only the tag-sensitive rule should match, got %v", got)
|
||||
}
|
||||
rh := s3lifecycle.RuleHash(rTag)
|
||||
if got[0].RuleHash != rh {
|
||||
t.Fatalf("expected match on tag-rule")
|
||||
}
|
||||
|
||||
// Wrong shape: never matches.
|
||||
if got := snap.MatchPredicateChange(&Event{Shape: EventShapeOriginalWrite, Bucket: "bk", Path: "x/o"}); len(got) != 0 {
|
||||
t.Fatalf("wrong shape should be empty, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchPath_BootstrapWalkerSeesAllActiveActions(t *testing.T) {
|
||||
// One rule with three actions (90d Expiration, 30d NoncurrentDays,
|
||||
// 7d AbortMPU). MatchPath returns every active action whose filter
|
||||
// matches the path; the bootstrap walker iterates these and calls
|
||||
// EvaluateAction per kind for the entry.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "multi",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
Prefix: "x/",
|
||||
ExpirationDays: 90,
|
||||
NoncurrentVersionExpirationDays: 30,
|
||||
AbortMPUDaysAfterInitiation: 7,
|
||||
}
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{PriorStates: activeAll("bk", []*s3lifecycle.Rule{rule})})
|
||||
|
||||
got := snap.MatchPath("bk", "x/obj", nil)
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("want all 3 active actions, got %v", got)
|
||||
}
|
||||
wantKinds := []s3lifecycle.ActionKind{
|
||||
s3lifecycle.ActionKindExpirationDays,
|
||||
s3lifecycle.ActionKindNoncurrentDays,
|
||||
s3lifecycle.ActionKindAbortMPU,
|
||||
}
|
||||
gotKinds := []s3lifecycle.ActionKind{}
|
||||
for _, k := range sortedKeys(got) {
|
||||
gotKinds = append(gotKinds, k.ActionKind)
|
||||
}
|
||||
sort.Slice(wantKinds, func(i, j int) bool { return wantKinds[i] < wantKinds[j] })
|
||||
if !reflect.DeepEqual(gotKinds, wantKinds) {
|
||||
t.Fatalf("want kinds %v, got %v", wantKinds, gotKinds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchPath_PrefixMismatchExcludes(t *testing.T) {
|
||||
rule := ruleExpDays("a", "logs/", 30)
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{PriorStates: activeAll("bk", []*s3lifecycle.Rule{rule})})
|
||||
if got := snap.MatchPath("bk", "data/x", nil); len(got) != 0 {
|
||||
t.Fatalf("prefix mismatch should reject, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchPath_FilterRejectsByTag(t *testing.T) {
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "tag",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
Prefix: "x/",
|
||||
ExpirationDays: 30,
|
||||
FilterTags: map[string]string{"env": "prod"},
|
||||
}
|
||||
e := New()
|
||||
snap := e.Compile([]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{PriorStates: activeAll("bk", []*s3lifecycle.Rule{rule})})
|
||||
// With ev passed, tag mismatch rejects.
|
||||
ev := &Event{Bucket: "bk", Path: "x/obj", Tags: map[string]string{"env": "dev"}}
|
||||
if got := snap.MatchPath("bk", "x/obj", ev); len(got) != 0 {
|
||||
t.Fatalf("tag mismatch should reject, got %v", got)
|
||||
}
|
||||
// With ev nil (caller will fetch live state): prefix-only, returns the action.
|
||||
if got := snap.MatchPath("bk", "x/obj", nil); len(got) != 1 {
|
||||
t.Fatalf("ev=nil prefix-only path should return action, got %v", got)
|
||||
}
|
||||
}
|
||||
57
weed/s3api/s3lifecycle/engine/mode.go
Normal file
57
weed/s3api/s3lifecycle/engine/mode.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
)
|
||||
|
||||
// RuleMode mirrors the durable s3_lifecycle_pb.LifecycleState.RuleMode enum;
|
||||
// the worker maps between them when it reads/writes durable state.
|
||||
type RuleMode int
|
||||
|
||||
const (
|
||||
ModeUnspecified RuleMode = iota
|
||||
ModeEventDriven
|
||||
ModeScanAtDate
|
||||
ModeScanOnly
|
||||
ModeDisabled
|
||||
ModePendingBootstrap
|
||||
)
|
||||
|
||||
func (m RuleMode) String() string {
|
||||
switch m {
|
||||
case ModeEventDriven:
|
||||
return "event_driven"
|
||||
case ModeScanAtDate:
|
||||
return "scan_at_date"
|
||||
case ModeScanOnly:
|
||||
return "scan_only"
|
||||
case ModeDisabled:
|
||||
return "disabled"
|
||||
case ModePendingBootstrap:
|
||||
return "pending_bootstrap"
|
||||
default:
|
||||
return "unspecified"
|
||||
}
|
||||
}
|
||||
|
||||
// decideMode: disabled rule -> DISABLED; EXPIRATION_DATE -> SCAN_AT_DATE;
|
||||
// reader-driven kind whose horizon exceeds retention -> SCAN_ONLY; else
|
||||
// EVENT_DRIVEN. metaLogRetention=0 means unbounded (default), gate doesn't
|
||||
// trip.
|
||||
func decideMode(rule *s3lifecycle.Rule, kind s3lifecycle.ActionKind, metaLogRetention, bootstrapLookbackMin time.Duration) RuleMode {
|
||||
if rule == nil || rule.Status != s3lifecycle.StatusEnabled {
|
||||
return ModeDisabled
|
||||
}
|
||||
if kind == s3lifecycle.ActionKindExpirationDate {
|
||||
return ModeScanAtDate
|
||||
}
|
||||
if metaLogRetention > 0 {
|
||||
horizon := s3lifecycle.EventLogHorizon(rule, kind)
|
||||
if horizon > 0 && metaLogRetention < horizon+bootstrapLookbackMin {
|
||||
return ModeScanOnly
|
||||
}
|
||||
}
|
||||
return ModeEventDriven
|
||||
}
|
||||
126
weed/s3api/s3lifecycle/engine/mode_test.go
Normal file
126
weed/s3api/s3lifecycle/engine/mode_test.go
Normal file
@@ -0,0 +1,126 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// decideMode is the per-action mode classifier the scheduler uses when
|
||||
// no durable Mode exists yet. Compile tests cover most paths
|
||||
// indirectly; these direct tests pin every branch so a regression in
|
||||
// decideMode itself can't slip through behind a more elaborate Compile
|
||||
// test failure.
|
||||
|
||||
func TestDecideMode_NilRuleIsDisabled(t *testing.T) {
|
||||
got := decideMode(nil, s3lifecycle.ActionKindExpirationDays, 0, 0)
|
||||
assert.Equal(t, ModeDisabled, got)
|
||||
}
|
||||
|
||||
func TestDecideMode_DisabledStatusIsDisabled(t *testing.T) {
|
||||
rule := &s3lifecycle.Rule{Status: s3lifecycle.StatusDisabled, ExpirationDays: 30}
|
||||
got := decideMode(rule, s3lifecycle.ActionKindExpirationDays, 0, 0)
|
||||
assert.Equal(t, ModeDisabled, got)
|
||||
}
|
||||
|
||||
func TestDecideMode_ExpirationDateIsScanAtDate(t *testing.T) {
|
||||
// ExpirationDate is a one-shot wall-clock trigger; no event log
|
||||
// involvement, so retention gates don't apply.
|
||||
rule := &s3lifecycle.Rule{
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDate: time.Now().Add(24 * time.Hour),
|
||||
}
|
||||
got := decideMode(rule, s3lifecycle.ActionKindExpirationDate, 0, 0)
|
||||
assert.Equal(t, ModeScanAtDate, got)
|
||||
}
|
||||
|
||||
func TestDecideMode_UnboundedRetentionIsEventDriven(t *testing.T) {
|
||||
// metaLogRetention=0 means unbounded; the retention gate doesn't
|
||||
// trip, so any horizon falls back to event-driven.
|
||||
rule := &s3lifecycle.Rule{
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 365,
|
||||
}
|
||||
got := decideMode(rule, s3lifecycle.ActionKindExpirationDays, 0, time.Hour)
|
||||
assert.Equal(t, ModeEventDriven, got)
|
||||
}
|
||||
|
||||
func TestDecideMode_HorizonWithinRetentionIsEventDriven(t *testing.T) {
|
||||
// retention=30d, horizon=7d, lookback=1m → 7d+1m < 30d → no gate.
|
||||
rule := &s3lifecycle.Rule{
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 7,
|
||||
}
|
||||
got := decideMode(rule, s3lifecycle.ActionKindExpirationDays, 30*24*time.Hour, time.Minute)
|
||||
assert.Equal(t, ModeEventDriven, got)
|
||||
}
|
||||
|
||||
func TestDecideMode_HorizonExceedsRetentionIsScanOnly(t *testing.T) {
|
||||
// retention=7d, horizon=30d → can't drive from event log alone, so
|
||||
// the action is promoted to scan_only and only fires from the
|
||||
// bootstrap walk.
|
||||
rule := &s3lifecycle.Rule{
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 30,
|
||||
}
|
||||
got := decideMode(rule, s3lifecycle.ActionKindExpirationDays, 7*24*time.Hour, time.Minute)
|
||||
assert.Equal(t, ModeScanOnly, got)
|
||||
}
|
||||
|
||||
func TestDecideMode_LookbackPushesAcrossThreshold(t *testing.T) {
|
||||
// retention=7d, horizon=7d - the bare horizon fits, but adding the
|
||||
// bootstrapLookbackMin must push it past retention so the gate
|
||||
// trips. Pins that lookback is added to horizon (not retention).
|
||||
rule := &s3lifecycle.Rule{
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 7,
|
||||
}
|
||||
retention := 7 * 24 * time.Hour
|
||||
// horizon already ~= retention; even a tiny lookback should trip.
|
||||
got := decideMode(rule, s3lifecycle.ActionKindExpirationDays, retention, 5*time.Minute)
|
||||
assert.Equal(t, ModeScanOnly, got)
|
||||
}
|
||||
|
||||
func TestDecideMode_ZeroHorizonNeverGates(t *testing.T) {
|
||||
// EventLogHorizon returns 0 for kinds whose rule fields aren't set
|
||||
// (e.g. an EXPIRATION_DAYS kind on a rule with ExpirationDays=0).
|
||||
// A zero horizon must skip the gate so the action falls through to
|
||||
// event-driven rather than incorrectly promote to scan_only.
|
||||
rule := &s3lifecycle.Rule{
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
// ExpirationDays unset -> EventLogHorizon returns 0
|
||||
}
|
||||
got := decideMode(rule, s3lifecycle.ActionKindExpirationDays, time.Hour, time.Minute)
|
||||
assert.Equal(t, ModeEventDriven, got)
|
||||
}
|
||||
|
||||
func TestRuleMode_StringRendersDocumentedNames(t *testing.T) {
|
||||
// Operators read these via metrics labels and admin UI; pin the
|
||||
// exact strings so a rename can't break dashboards.
|
||||
cases := []struct {
|
||||
mode RuleMode
|
||||
want string
|
||||
}{
|
||||
{ModeUnspecified, "unspecified"},
|
||||
{ModeEventDriven, "event_driven"},
|
||||
{ModeScanAtDate, "scan_at_date"},
|
||||
{ModeScanOnly, "scan_only"},
|
||||
{ModeDisabled, "disabled"},
|
||||
{ModePendingBootstrap, "pending_bootstrap"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.want, func(t *testing.T) {
|
||||
assert.Equal(t, c.want, c.mode.String())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuleMode_StringFallsToUnspecifiedForUnknown(t *testing.T) {
|
||||
// A bogus enum value (e.g., a future RuleMode added without a
|
||||
// String() update) must render as "unspecified" rather than empty
|
||||
// or panic.
|
||||
bogus := RuleMode(999)
|
||||
assert.Equal(t, "unspecified", bogus.String())
|
||||
}
|
||||
201
weed/s3api/s3lifecycle/engine/snapshot_accessors_test.go
Normal file
201
weed/s3api/s3lifecycle/engine/snapshot_accessors_test.go
Normal file
@@ -0,0 +1,201 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Direct tests for Snapshot.OriginalDelayGroups / PredicateActions /
|
||||
// DateActions accessors and MarkActive/IsActive transitions. The Compile
|
||||
// tests exercise the construction path; these tests pin the read-side
|
||||
// surface the router and dispatcher reach for at runtime.
|
||||
|
||||
func TestSnapshot_OriginalDelayGroupsExposesCompiledGroups(t *testing.T) {
|
||||
// Two action kinds with different delays must land in distinct
|
||||
// delay groups so the dispatcher polls each at the right cadence.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 7,
|
||||
AbortMPUDaysAfterInitiation: 3,
|
||||
}
|
||||
hash := s3lifecycle.RuleHash(rule)
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{}
|
||||
for _, k := range s3lifecycle.RuleActionKinds(rule) {
|
||||
prior[s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: k}] = PriorState{
|
||||
BootstrapComplete: true,
|
||||
Mode: ModeEventDriven,
|
||||
}
|
||||
}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{PriorStates: prior},
|
||||
)
|
||||
groups := snap.OriginalDelayGroups()
|
||||
require.NotNil(t, groups)
|
||||
|
||||
// 7d expiration delay group must contain the specific expiration key.
|
||||
expirationDelay := s3lifecycle.MinTriggerAge(rule, s3lifecycle.ActionKindExpirationDays)
|
||||
expirationKey := s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: s3lifecycle.ActionKindExpirationDays}
|
||||
assert.Contains(t, groups[expirationDelay], expirationKey, "expiration delay group must carry its specific key")
|
||||
|
||||
// 3d abort-mpu delay group is distinct and contains the abort key.
|
||||
abortDelay := s3lifecycle.MinTriggerAge(rule, s3lifecycle.ActionKindAbortMPU)
|
||||
assert.NotEqual(t, expirationDelay, abortDelay)
|
||||
abortKey := s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: s3lifecycle.ActionKindAbortMPU}
|
||||
assert.Contains(t, groups[abortDelay], abortKey, "abort-mpu delay group must carry its specific key")
|
||||
}
|
||||
|
||||
func TestSnapshot_OriginalDelayGroupsScanOnlyExcluded(t *testing.T) {
|
||||
// Scan-only actions don't go through originalDelayGroups (they
|
||||
// fire from the bootstrap walk only), so the dispatcher's
|
||||
// MatchOriginalWrite path won't see them.
|
||||
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 7}
|
||||
hash := s3lifecycle.RuleHash(rule)
|
||||
scanOnlyKey := s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: s3lifecycle.ActionKindExpirationDays}
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{
|
||||
scanOnlyKey: {BootstrapComplete: true, Mode: ModeScanOnly},
|
||||
}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{PriorStates: prior},
|
||||
)
|
||||
groups := snap.OriginalDelayGroups()
|
||||
for delay, keys := range groups {
|
||||
for _, k := range keys {
|
||||
assert.NotEqual(t, scanOnlyKey, k, "scan-only action leaked into delay group %v", delay)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshot_PredicateActionsContainsTagSensitive(t *testing.T) {
|
||||
// A tag-sensitive rule's actions must surface in PredicateActions
|
||||
// so MatchPredicateChange routes them.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 7,
|
||||
FilterTags: map[string]string{"env": "prod"},
|
||||
}
|
||||
hash := s3lifecycle.RuleHash(rule)
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{}
|
||||
for _, k := range s3lifecycle.RuleActionKinds(rule) {
|
||||
prior[s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: k}] = PriorState{
|
||||
BootstrapComplete: true,
|
||||
Mode: ModeEventDriven,
|
||||
}
|
||||
}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{PriorStates: prior},
|
||||
)
|
||||
predicates := snap.PredicateActions()
|
||||
require.NotEmpty(t, predicates, "tag-sensitive rule must populate predicateActions")
|
||||
// Verify the specific key landed (not just non-empty) so a routing
|
||||
// regression that emits a wrong ActionKey is caught.
|
||||
expectedKey := s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: s3lifecycle.ActionKindExpirationDays}
|
||||
assert.Contains(t, predicates, expectedKey, "predicateActions must carry the rule's specific ActionKey")
|
||||
}
|
||||
|
||||
func TestSnapshot_PredicateActionsEmptyForNonTagSensitiveRule(t *testing.T) {
|
||||
// A rule without FilterTags is not predicate-sensitive; the
|
||||
// predicate-action list must stay empty so MatchPredicateChange
|
||||
// never routes irrelevant kinds.
|
||||
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 7}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{},
|
||||
)
|
||||
assert.Empty(t, snap.PredicateActions())
|
||||
}
|
||||
|
||||
func TestSnapshot_DateActionsContainsExpirationDate(t *testing.T) {
|
||||
// EXPIRATION_DATE rules go into dateActions (not delay groups);
|
||||
// the scan-at-date scheduler reads from this map.
|
||||
when := time.Now().Add(24 * time.Hour)
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDate: when,
|
||||
}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{},
|
||||
)
|
||||
dateActions := snap.DateActions()
|
||||
require.NotEmpty(t, dateActions)
|
||||
for _, dt := range dateActions {
|
||||
assert.Equal(t, when, dt, "DateActions must surface the rule's ExpirationDate verbatim")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshot_DateActionsEmptyForNonDateRule(t *testing.T) {
|
||||
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 7}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{},
|
||||
)
|
||||
assert.Empty(t, snap.DateActions())
|
||||
}
|
||||
|
||||
func TestSnapshot_MarkActiveUnknownKeyIsNoOp(t *testing.T) {
|
||||
// MarkActive must silently skip keys that aren't in the snapshot;
|
||||
// the durable bootstrap-complete write can race with a recompile
|
||||
// that drops the rule, and a panic here would crash the worker.
|
||||
snap := New().Compile(nil, CompileOptions{})
|
||||
snap.MarkActive(s3lifecycle.ActionKey{Bucket: "ghost", ActionKind: s3lifecycle.ActionKindExpirationDays})
|
||||
}
|
||||
|
||||
func TestSnapshot_MarkActiveFlipsCompiledActionToActive(t *testing.T) {
|
||||
// Bootstrap-pending actions land inactive; MarkActive transitions
|
||||
// them to active so the routing filter starts emitting matches.
|
||||
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 7}
|
||||
hash := s3lifecycle.RuleHash(rule)
|
||||
key := s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: s3lifecycle.ActionKindExpirationDays}
|
||||
// No prior state -> the action lands inactive (BootstrapComplete=false).
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{},
|
||||
)
|
||||
a := snap.Action(key)
|
||||
require.NotNil(t, a)
|
||||
assert.False(t, a.IsActive(), "fresh action without prior state must be inactive")
|
||||
snap.MarkActive(key)
|
||||
assert.True(t, a.IsActive(), "MarkActive must transition the action to active")
|
||||
}
|
||||
|
||||
func TestSnapshot_BucketIndexedActionKeysCoverAllKinds(t *testing.T) {
|
||||
// Cross-check that BucketActionKeys lists every kind compiled from
|
||||
// the rule, regardless of mode. The router iterates these for
|
||||
// MatchPath, so a missed kind silently disables prefix-only routing
|
||||
// for that action.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 7,
|
||||
AbortMPUDaysAfterInitiation: 3,
|
||||
ExpiredObjectDeleteMarker: true,
|
||||
}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{},
|
||||
)
|
||||
keys := snap.BucketActionKeys("bk")
|
||||
wantKinds := s3lifecycle.RuleActionKinds(rule)
|
||||
require.Len(t, keys, len(wantKinds))
|
||||
|
||||
// Verify the contents, not just the count: every key must carry
|
||||
// the right bucket scope, and the kinds must match RuleActionKinds
|
||||
// element-for-element (in any order). Catches an indexing
|
||||
// regression that produces N keys but with wrong kinds.
|
||||
gotKinds := make([]s3lifecycle.ActionKind, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
assert.Equal(t, "bk", k.Bucket, "key leaked across bucket")
|
||||
gotKinds = append(gotKinds, k.ActionKind)
|
||||
}
|
||||
assert.ElementsMatch(t, wantKinds, gotKinds, "kinds must match RuleActionKinds")
|
||||
}
|
||||
234
weed/s3api/s3lifecycle/engine/views.go
Normal file
234
weed/s3api/s3lifecycle/engine/views.go
Normal file
@@ -0,0 +1,234 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sort"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
)
|
||||
|
||||
// currentEngine is the package-level pointer the wiring will eventually
|
||||
// populate so daily_run can fetch the live snapshot through CurrentSnapshot.
|
||||
// Today the worker/scheduler owns its own *Engine and passes the snapshot
|
||||
// explicitly to its consumers; nothing calls SetCurrentEngine yet. Phase 4's
|
||||
// daily-replay caller will register the worker's engine here on startup so
|
||||
// the package-level helper resolves without threading the *Engine through
|
||||
// every layer.
|
||||
//
|
||||
// TODO(phase4-wiring): hook scheduler/worker startup to call SetCurrentEngine.
|
||||
var currentEngine atomic.Pointer[Engine]
|
||||
|
||||
// SetCurrentEngine installs e as the package-level engine returned by
|
||||
// CurrentSnapshot. Intended for the worker bootstrap path; tests may also
|
||||
// call it with a locally-constructed engine. Passing nil clears the pointer.
|
||||
func SetCurrentEngine(e *Engine) { currentEngine.Store(e) }
|
||||
|
||||
// CurrentSnapshot returns the latest *Snapshot from the package-level engine,
|
||||
// or nil if no engine has been registered yet. The caller is responsible for
|
||||
// nil-checking the return — a missing engine is treated as "no rules" so the
|
||||
// daily-run loop short-circuits cleanly instead of panicking during early
|
||||
// startup.
|
||||
//
|
||||
// TODO(phase4-wiring): SetCurrentEngine must be invoked at worker startup
|
||||
// for production callers; until then this returns nil.
|
||||
func CurrentSnapshot() *Snapshot {
|
||||
e := currentEngine.Load()
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return e.Snapshot()
|
||||
}
|
||||
|
||||
// cloneAction makes an independent *CompiledAction for a per-view snapshot.
|
||||
// The clone shares Rule (immutable) and Key (value) with the base, but its
|
||||
// engineState (the active bit) is its own atomic — so flipping active on the
|
||||
// clone never reaches the base or any sibling view. Mode is set explicitly by
|
||||
// the caller to encode the per-view dispatch contract (replay forces
|
||||
// ModeEventDriven so router.Route's gate passes; walk and recovery preserve
|
||||
// the base Mode so the walker's "any non-Disabled" gate works as-is).
|
||||
func cloneAction(src *CompiledAction, mode RuleMode, active bool) *CompiledAction {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
dst := &CompiledAction{
|
||||
Rule: src.Rule,
|
||||
Bucket: src.Bucket,
|
||||
Key: src.Key,
|
||||
Delay: src.Delay,
|
||||
PredicateSensitive: src.PredicateSensitive,
|
||||
Mode: mode,
|
||||
}
|
||||
if active {
|
||||
dst.markActive()
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// newView constructs a *Snapshot that holds the supplied per-view action
|
||||
// clones but otherwise shares the base's immutable rule / index data by
|
||||
// pointer (see DESIGN.md "router.Route integration" — only `active` and
|
||||
// `Mode` differ per view; everything else is share-by-pointer).
|
||||
//
|
||||
// Returns nil if actions is empty: callers consume nil as "this partition
|
||||
// has no actions, skip the corresponding dispatch path."
|
||||
func newView(base *Snapshot, actions map[s3lifecycle.ActionKey]*CompiledAction) *Snapshot {
|
||||
if len(actions) == 0 {
|
||||
return nil
|
||||
}
|
||||
// allActionsSorted mirrors the base order: same comparator (Bucket,
|
||||
// RuleHash, ActionKind) so downstream iteration order is stable per view.
|
||||
sorted := make([]*CompiledAction, 0, len(actions))
|
||||
for _, a := range actions {
|
||||
sorted = append(sorted, a)
|
||||
}
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
a, b := sorted[i], sorted[j]
|
||||
if a.Bucket != b.Bucket {
|
||||
return a.Bucket < b.Bucket
|
||||
}
|
||||
if c := bytes.Compare(a.Key.RuleHash[:], b.Key.RuleHash[:]); c != 0 {
|
||||
return c < 0
|
||||
}
|
||||
return a.Key.ActionKind < b.Key.ActionKind
|
||||
})
|
||||
return &Snapshot{
|
||||
id: base.id,
|
||||
buckets: base.buckets,
|
||||
actions: actions,
|
||||
allActionsSorted: sorted,
|
||||
originalDelayGroups: base.originalDelayGroups,
|
||||
predicateActions: base.predicateActions,
|
||||
dateActions: base.dateActions,
|
||||
}
|
||||
}
|
||||
|
||||
// isReplayKind reports whether ActionKind is a candidate for the replay
|
||||
// partition: its trigger derives from a single event's TsNs (or a stamped
|
||||
// noncurrent_since), so the meta-log scan can drive dispatch directly.
|
||||
// ExpirationDate / ExpiredDeleteMarker / NewerNoncurrent are walker-only
|
||||
// because their due time depends on current bucket state, not event age.
|
||||
func isReplayKind(k s3lifecycle.ActionKind) bool {
|
||||
switch k {
|
||||
case s3lifecycle.ActionKindExpirationDays,
|
||||
s3lifecycle.ActionKindNoncurrentDays,
|
||||
s3lifecycle.ActionKindAbortMPU:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// effectiveTTL is the duration used both for partition membership
|
||||
// (TTL ≤ retentionWindow → replay) and MaxEffectiveTTL math (driving the
|
||||
// cursor's sliding scan window). Walker-only kinds return 0 so they never
|
||||
// contribute to MaxEffectiveTTL and never satisfy the partition gate when
|
||||
// retentionWindow is finite.
|
||||
func effectiveTTL(a *CompiledAction) time.Duration {
|
||||
if a == nil || a.Rule == nil {
|
||||
return 0
|
||||
}
|
||||
switch a.Key.ActionKind {
|
||||
case s3lifecycle.ActionKindExpirationDays:
|
||||
if a.Rule.ExpirationDays > 0 {
|
||||
return s3lifecycle.DaysToDuration(a.Rule.ExpirationDays)
|
||||
}
|
||||
case s3lifecycle.ActionKindNoncurrentDays:
|
||||
if a.Rule.NoncurrentVersionExpirationDays > 0 {
|
||||
return s3lifecycle.DaysToDuration(a.Rule.NoncurrentVersionExpirationDays)
|
||||
}
|
||||
case s3lifecycle.ActionKindAbortMPU:
|
||||
if a.Rule.AbortMPUDaysAfterInitiation > 0 {
|
||||
return s3lifecycle.DaysToDuration(a.Rule.AbortMPUDaysAfterInitiation)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// RulesForShard partitions the base snapshot's compiled actions into a
|
||||
// replay view and a walk view.
|
||||
//
|
||||
// shardID is **reserved for forward-compatibility** and not consumed by
|
||||
// this implementation. Every shard sees every rule today because the
|
||||
// shard filter runs at the entry-iteration site (meta-log subscription
|
||||
// per shard, walker entry-loop ShardID check) rather than at view
|
||||
// construction. Keeping the parameter in the signature preserves the
|
||||
// API surface for a future move to per-shard rule sets (e.g. when
|
||||
// bucket-shard ownership is added) — removing it now would be a
|
||||
// breaking change at that future point. See DESIGN.md "open questions
|
||||
// → per-shard rule snapshots vs. global."
|
||||
//
|
||||
// retentionWindow is the only window input the engine sees; the
|
||||
// daily_run caller computes it as `now - earliest_available` and
|
||||
// passes it in so the partition is stable across one daily_run
|
||||
// invocation.
|
||||
//
|
||||
// Membership:
|
||||
// - replay: clones of ExpirationDays / NoncurrentDays / AbortMPU actions
|
||||
// whose TTL ≤ retentionWindow. active=true, Mode forced to
|
||||
// ModeEventDriven (rehabilitating any prior ModeScanOnly lock-in so
|
||||
// router.Route's gate accepts the clone).
|
||||
// - walk: clones of ExpirationDate / ExpiredDeleteMarker / NewerNoncurrent
|
||||
// actions, plus any replay-eligible action whose TTL exceeds
|
||||
// retentionWindow (the "scan_only promotion" path). active=true, Mode
|
||||
// preserved from the base — the walker only rejects ModeDisabled.
|
||||
//
|
||||
// Either return value may be nil when its partition has no actions.
|
||||
// Disabled actions (Mode == ModeDisabled) are excluded from both views.
|
||||
func (s *Snapshot) RulesForShard(shardID int, retentionWindow time.Duration) (replay, walk *Snapshot) {
|
||||
_ = shardID
|
||||
if s == nil {
|
||||
return nil, nil
|
||||
}
|
||||
replayActions := map[s3lifecycle.ActionKey]*CompiledAction{}
|
||||
walkActions := map[s3lifecycle.ActionKey]*CompiledAction{}
|
||||
for key, a := range s.actions {
|
||||
if a == nil || a.Mode == ModeDisabled {
|
||||
continue
|
||||
}
|
||||
if isReplayKind(key.ActionKind) {
|
||||
ttl := effectiveTTL(a)
|
||||
// retentionWindow == 0 means "no retention info supplied" — the
|
||||
// caller should treat the partition as walk-only to stay safe,
|
||||
// since promoting every rule into replay without retention
|
||||
// proof would defeat the scan_only protection. ttl == 0 is a
|
||||
// malformed rule (kind says replay but no TTL): also routes to
|
||||
// walk so the walker can decide.
|
||||
if ttl > 0 && ttl <= retentionWindow {
|
||||
replayActions[key] = cloneAction(a, ModeEventDriven, true)
|
||||
} else {
|
||||
walkActions[key] = cloneAction(a, a.Mode, true)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Walker-only kinds: preserve Mode (ModeScanAtDate for
|
||||
// ExpirationDate, ModeEventDriven for the others) so the walker's
|
||||
// per-rule evaluator runs as today.
|
||||
walkActions[key] = cloneAction(a, a.Mode, true)
|
||||
}
|
||||
return newView(s, replayActions), newView(s, walkActions)
|
||||
}
|
||||
|
||||
// RecoveryView returns a *Snapshot that contains a clone of every action in
|
||||
// the base, with active=true regardless of compile-time IsActive/
|
||||
// BootstrapComplete and Mode preserved as-is. The recovery walker uses this
|
||||
// view to catch already-due objects across the full rule set on cold start,
|
||||
// rule edit, retention loss, or partition flip. Disabled actions stay out —
|
||||
// recovery shouldn't resurrect an explicitly disabled rule.
|
||||
//
|
||||
// Unlike RulesForShard, RecoveryView takes no retentionWindow: every
|
||||
// non-disabled action is activated, partition-independent, because recovery
|
||||
// must reach rules whose subjects sit outside the replay scan window.
|
||||
func RecoveryView(s *Snapshot) *Snapshot {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
actions := map[s3lifecycle.ActionKey]*CompiledAction{}
|
||||
for key, a := range s.actions {
|
||||
if a == nil || a.Mode == ModeDisabled {
|
||||
continue
|
||||
}
|
||||
actions[key] = cloneAction(a, a.Mode, true)
|
||||
}
|
||||
return newView(s, actions)
|
||||
}
|
||||
325
weed/s3api/s3lifecycle/engine/views_test.go
Normal file
325
weed/s3api/s3lifecycle/engine/views_test.go
Normal file
@@ -0,0 +1,325 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// buildSnapshotForViews compiles a snapshot with a single bucket containing
|
||||
// one rule per supplied factory, marking every action bootstrap_complete so
|
||||
// the base actions are active. Tests can then call RulesForShard and assert
|
||||
// the per-view clones come out independent.
|
||||
func buildSnapshotForViews(t *testing.T, bucket string, rules ...*s3lifecycle.Rule) *Snapshot {
|
||||
t.Helper()
|
||||
prior := map[s3lifecycle.ActionKey]PriorState{}
|
||||
for _, r := range rules {
|
||||
rh := s3lifecycle.RuleHash(r)
|
||||
for _, k := range s3lifecycle.RuleActionKinds(r) {
|
||||
prior[s3lifecycle.ActionKey{Bucket: bucket, RuleHash: rh, ActionKind: k}] = PriorState{
|
||||
BootstrapComplete: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
return New().Compile(
|
||||
[]CompileInput{{Bucket: bucket, Rules: rules}},
|
||||
CompileOptions{PriorStates: prior},
|
||||
)
|
||||
}
|
||||
|
||||
func TestCurrentSnapshot_NilWhenNoEngineRegistered(t *testing.T) {
|
||||
// Save and restore the package-level pointer so tests stay isolated.
|
||||
prev := currentEngine.Load()
|
||||
t.Cleanup(func() { currentEngine.Store(prev) })
|
||||
|
||||
SetCurrentEngine(nil)
|
||||
assert.Nil(t, CurrentSnapshot())
|
||||
}
|
||||
|
||||
func TestCurrentSnapshot_ReturnsRegisteredEngineSnapshot(t *testing.T) {
|
||||
prev := currentEngine.Load()
|
||||
t.Cleanup(func() { currentEngine.Store(prev) })
|
||||
|
||||
e := New()
|
||||
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 7}
|
||||
snap := e.Compile([]CompileInput{{Bucket: "b", Rules: []*s3lifecycle.Rule{rule}}}, CompileOptions{})
|
||||
SetCurrentEngine(e)
|
||||
assert.Same(t, snap, CurrentSnapshot())
|
||||
}
|
||||
|
||||
func TestRulesForShard_NilSnapshot(t *testing.T) {
|
||||
var s *Snapshot
|
||||
replay, walk := s.RulesForShard(0, 30*24*time.Hour)
|
||||
assert.Nil(t, replay)
|
||||
assert.Nil(t, walk)
|
||||
}
|
||||
|
||||
func TestRulesForShard_ReplayPartitionMembership(t *testing.T) {
|
||||
// ExpirationDays / NoncurrentDays / AbortMPU below retentionWindow all
|
||||
// land in replay; the replay map carries clones with active=true and
|
||||
// Mode=ModeEventDriven.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "all-replay",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 3,
|
||||
NoncurrentVersionExpirationDays: 5,
|
||||
AbortMPUDaysAfterInitiation: 2,
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
replay, walk := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
|
||||
require.NotNil(t, replay, "all three replay-eligible kinds should land in replay")
|
||||
assert.Nil(t, walk, "no walker kinds, no scan_only promotion, walk should be empty")
|
||||
assert.Len(t, replay.actions, 3)
|
||||
for k, a := range replay.actions {
|
||||
assert.True(t, a.IsActive(), "replay clone must be active for kind %v", k.ActionKind)
|
||||
assert.Equal(t, ModeEventDriven, a.Mode, "replay Mode must be ModeEventDriven for kind %v", k.ActionKind)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesForShard_WalkOnlyKindsRouteToWalk(t *testing.T) {
|
||||
// ExpirationDate, ExpiredObjectDeleteMarker, NewerNoncurrent are walker-
|
||||
// only and must land in walk regardless of retentionWindow.
|
||||
when := time.Now().Add(24 * time.Hour)
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "walker-mix",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDate: when,
|
||||
ExpiredObjectDeleteMarker: true,
|
||||
NewerNoncurrentVersions: 2,
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
replay, walk := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
|
||||
assert.Nil(t, replay, "walker-only rule has no replay partition")
|
||||
require.NotNil(t, walk)
|
||||
assert.Len(t, walk.actions, 3, "all three walker kinds should land in walk")
|
||||
for _, a := range walk.actions {
|
||||
assert.True(t, a.IsActive())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesForShard_MultiActionXMLRuleSplits(t *testing.T) {
|
||||
// A single XML rule that compiles to ExpirationDays + NewerNoncurrent
|
||||
// must split: ExpirationDays → replay, NewerNoncurrent → walk.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "split",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 3,
|
||||
NewerNoncurrentVersions: 5,
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
replay, walk := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
|
||||
require.NotNil(t, replay)
|
||||
require.NotNil(t, walk)
|
||||
assert.Len(t, replay.actions, 1)
|
||||
assert.Len(t, walk.actions, 1)
|
||||
for k := range replay.actions {
|
||||
assert.Equal(t, s3lifecycle.ActionKindExpirationDays, k.ActionKind)
|
||||
}
|
||||
for k := range walk.actions {
|
||||
assert.Equal(t, s3lifecycle.ActionKindNewerNoncurrent, k.ActionKind)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesForShard_ScanOnlyPromotionRoutesToWalk(t *testing.T) {
|
||||
// A replay-eligible kind whose TTL exceeds retentionWindow promotes to
|
||||
// walk. The clone's active=true is set; Mode is preserved as-is from
|
||||
// the base (the walker only rejects ModeDisabled).
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "long",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 90,
|
||||
AbortMPUDaysAfterInitiation: 1,
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
// Retention window of 30 days promotes ExpirationDays(90d) to walk;
|
||||
// AbortMPU(1d) stays in replay.
|
||||
replay, walk := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
|
||||
require.NotNil(t, replay)
|
||||
require.NotNil(t, walk)
|
||||
assert.Len(t, replay.actions, 1)
|
||||
assert.Len(t, walk.actions, 1)
|
||||
for k, a := range replay.actions {
|
||||
assert.Equal(t, s3lifecycle.ActionKindAbortMPU, k.ActionKind)
|
||||
assert.Equal(t, ModeEventDriven, a.Mode)
|
||||
}
|
||||
for k := range walk.actions {
|
||||
assert.Equal(t, s3lifecycle.ActionKindExpirationDays, k.ActionKind)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRulesForShard_ReplayRewritesScanOnlyMode(t *testing.T) {
|
||||
// A base action carrying Mode=ModeScanOnly via PriorState must still
|
||||
// land in replay as ModeEventDriven once retention rehabilitates its
|
||||
// TTL. router.Route gates on ModeEventDriven, so this is the
|
||||
// rehabilitation contract.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "rehab",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 5,
|
||||
}
|
||||
rh := s3lifecycle.RuleHash(rule)
|
||||
key := s3lifecycle.ActionKey{Bucket: "b1", RuleHash: rh, ActionKind: s3lifecycle.ActionKindExpirationDays}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{
|
||||
PriorStates: map[s3lifecycle.ActionKey]PriorState{
|
||||
// Persistent ModeScanOnly carried over from a prior compile.
|
||||
key: {BootstrapComplete: true, Mode: ModeScanOnly},
|
||||
},
|
||||
},
|
||||
)
|
||||
// Verify the base really did keep ModeScanOnly (Compile preserves it).
|
||||
require.Equal(t, ModeScanOnly, snap.actions[key].Mode)
|
||||
|
||||
replay, _ := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
|
||||
require.NotNil(t, replay)
|
||||
clone := replay.actions[key]
|
||||
require.NotNil(t, clone)
|
||||
assert.Equal(t, ModeEventDriven, clone.Mode, "replay must force ModeEventDriven on the clone")
|
||||
assert.True(t, clone.IsActive())
|
||||
}
|
||||
|
||||
func TestRulesForShard_DisabledRuleExcludedFromBothViews(t *testing.T) {
|
||||
rule := &s3lifecycle.Rule{ID: "off", Status: s3lifecycle.StatusDisabled, ExpirationDays: 7}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
replay, walk := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
|
||||
assert.Nil(t, replay)
|
||||
assert.Nil(t, walk)
|
||||
}
|
||||
|
||||
func TestRulesForShard_ClonesAreIndependentOfBase(t *testing.T) {
|
||||
// Toggling active on a view clone must never propagate to the base
|
||||
// action — that's the whole point of cloning per view.
|
||||
rule := &s3lifecycle.Rule{ID: "iso", Status: s3lifecycle.StatusEnabled, ExpirationDays: 3}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
key := s3lifecycle.ActionKey{
|
||||
Bucket: "b1",
|
||||
RuleHash: s3lifecycle.RuleHash(rule),
|
||||
ActionKind: s3lifecycle.ActionKindExpirationDays,
|
||||
}
|
||||
base := snap.Action(key)
|
||||
require.NotNil(t, base)
|
||||
baseActiveBefore := base.IsActive()
|
||||
|
||||
replay, _ := snap.RulesForShard(0, s3lifecycle.DaysToDuration(30))
|
||||
require.NotNil(t, replay)
|
||||
clone := replay.actions[key]
|
||||
require.NotNil(t, clone)
|
||||
assert.NotSame(t, base, clone, "clone must be a distinct *CompiledAction")
|
||||
|
||||
// Force the clone inactive (write directly to its atomic so we exercise
|
||||
// the active bit boundary, not just markActive's setter).
|
||||
clone.engineState.Store(engineStateInactive)
|
||||
assert.False(t, clone.IsActive())
|
||||
assert.Equal(t, baseActiveBefore, base.IsActive(),
|
||||
"toggling the clone must not perturb the base's active bit")
|
||||
}
|
||||
|
||||
func TestRulesForShard_ZeroRetentionRoutesAllReplayKindsToWalk(t *testing.T) {
|
||||
// retentionWindow == 0 (no proven retention) must route every replay-
|
||||
// eligible action into walk — the partition is "ttl > 0 && ttl <=
|
||||
// retentionWindow", and 0 fails the upper bound. This is the safe
|
||||
// default before the meta-log horizon is known.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "any",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 1,
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
replay, walk := snap.RulesForShard(0, 0)
|
||||
assert.Nil(t, replay)
|
||||
require.NotNil(t, walk)
|
||||
assert.Len(t, walk.actions, 1)
|
||||
}
|
||||
|
||||
func TestRecoveryView_ActivatesEveryNonDisabledAction(t *testing.T) {
|
||||
// Base snapshot has no PriorState → event-driven actions land inactive
|
||||
// (pre-BootstrapComplete). Recovery view must clone them with active=
|
||||
// true so the recovery walker emits matches for them.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "pending",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 7,
|
||||
}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{}, // no PriorStates → action lands inactive
|
||||
)
|
||||
key := s3lifecycle.ActionKey{
|
||||
Bucket: "b1",
|
||||
RuleHash: s3lifecycle.RuleHash(rule),
|
||||
ActionKind: s3lifecycle.ActionKindExpirationDays,
|
||||
}
|
||||
require.False(t, snap.actions[key].IsActive(), "precondition: base action must be inactive")
|
||||
|
||||
rv := RecoveryView(snap)
|
||||
require.NotNil(t, rv)
|
||||
clone := rv.actions[key]
|
||||
require.NotNil(t, clone)
|
||||
assert.True(t, clone.IsActive(), "recovery view must force-activate even pre-BootstrapComplete actions")
|
||||
assert.Equal(t, ModeEventDriven, clone.Mode, "recovery preserves base Mode")
|
||||
}
|
||||
|
||||
func TestRecoveryView_PreservesBaseModeForScanAtDate(t *testing.T) {
|
||||
// ScanAtDate actions are active at compile time (no BootstrapComplete
|
||||
// requirement). Recovery view must keep their Mode as ModeScanAtDate so
|
||||
// the walker's date evaluator runs as expected — a Mode rewrite here
|
||||
// would re-route them through the event-driven gate by mistake.
|
||||
when := time.Now().Add(48 * time.Hour)
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "d",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDate: when,
|
||||
}
|
||||
snap := New().Compile(
|
||||
[]CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
CompileOptions{},
|
||||
)
|
||||
rv := RecoveryView(snap)
|
||||
require.NotNil(t, rv)
|
||||
require.Len(t, rv.actions, 1)
|
||||
for _, a := range rv.actions {
|
||||
assert.Equal(t, ModeScanAtDate, a.Mode)
|
||||
assert.True(t, a.IsActive())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoveryView_ExcludesDisabledRules(t *testing.T) {
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "off",
|
||||
Status: s3lifecycle.StatusDisabled,
|
||||
ExpirationDays: 7,
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
rv := RecoveryView(snap)
|
||||
assert.Nil(t, rv, "disabled rules must not surface in recovery view")
|
||||
}
|
||||
|
||||
func TestRecoveryView_ClonesAreIndependentOfBase(t *testing.T) {
|
||||
rule := &s3lifecycle.Rule{ID: "iso", Status: s3lifecycle.StatusEnabled, ExpirationDays: 3}
|
||||
snap := buildSnapshotForViews(t, "b1", rule)
|
||||
key := s3lifecycle.ActionKey{
|
||||
Bucket: "b1",
|
||||
RuleHash: s3lifecycle.RuleHash(rule),
|
||||
ActionKind: s3lifecycle.ActionKindExpirationDays,
|
||||
}
|
||||
base := snap.Action(key)
|
||||
baseActiveBefore := base.IsActive()
|
||||
|
||||
rv := RecoveryView(snap)
|
||||
require.NotNil(t, rv)
|
||||
clone := rv.actions[key]
|
||||
require.NotNil(t, clone)
|
||||
assert.NotSame(t, base, clone)
|
||||
|
||||
clone.engineState.Store(engineStateInactive)
|
||||
assert.Equal(t, baseActiveBefore, base.IsActive(),
|
||||
"flipping the recovery clone must not perturb the base")
|
||||
}
|
||||
|
||||
func TestRecoveryView_NilSnapshot(t *testing.T) {
|
||||
assert.Nil(t, RecoveryView(nil))
|
||||
}
|
||||
Reference in New Issue
Block a user