snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
335
weed/s3api/s3lifecycle/router/helpers_test.go
Normal file
335
weed/s3api/s3lifecycle/router/helpers_test.go
Normal file
@@ -0,0 +1,335 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Direct coverage for the pure helpers in router.go that the larger
|
||||
// Route* tests exercise indirectly. Pinning each one separately catches
|
||||
// regressions before they manifest as a broader Route failure that's
|
||||
// harder to diagnose.
|
||||
|
||||
// ---------- successorModTimeFromContainer ----------
|
||||
|
||||
func TestSuccessorModTimeFromContainer_MissingExtReturnsZero(t *testing.T) {
|
||||
assert.True(t, successorModTimeFromContainer(&filer_pb.Entry{}).IsZero())
|
||||
assert.True(t, successorModTimeFromContainer(&filer_pb.Entry{
|
||||
Extended: map[string][]byte{},
|
||||
}).IsZero())
|
||||
}
|
||||
|
||||
func TestSuccessorModTimeFromContainer_EmptyValueReturnsZero(t *testing.T) {
|
||||
got := successorModTimeFromContainer(&filer_pb.Entry{
|
||||
Extended: map[string][]byte{s3_constants.ExtLatestVersionMtimeKey: nil},
|
||||
})
|
||||
assert.True(t, got.IsZero())
|
||||
}
|
||||
|
||||
func TestSuccessorModTimeFromContainer_NonNumericReturnsZero(t *testing.T) {
|
||||
// A malformed value is the writer's bug; the helper must not
|
||||
// propagate a parse error or surface a nonsense time.
|
||||
got := successorModTimeFromContainer(&filer_pb.Entry{
|
||||
Extended: map[string][]byte{s3_constants.ExtLatestVersionMtimeKey: []byte("not-a-number")},
|
||||
})
|
||||
assert.True(t, got.IsZero())
|
||||
}
|
||||
|
||||
func TestSuccessorModTimeFromContainer_NonPositiveReturnsZero(t *testing.T) {
|
||||
// The container ext stores Unix seconds; <=0 means "not set" so
|
||||
// the helper falls back to zero rather than emitting 1970-01-01.
|
||||
for _, raw := range []string{"0", "-1", "-1000"} {
|
||||
t.Run(raw, func(t *testing.T) {
|
||||
got := successorModTimeFromContainer(&filer_pb.Entry{
|
||||
Extended: map[string][]byte{s3_constants.ExtLatestVersionMtimeKey: []byte(raw)},
|
||||
})
|
||||
assert.True(t, got.IsZero(), "value %q must produce zero time", raw)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSuccessorModTimeFromContainer_PositiveSecondsRoundTrip(t *testing.T) {
|
||||
got := successorModTimeFromContainer(&filer_pb.Entry{
|
||||
Extended: map[string][]byte{s3_constants.ExtLatestVersionMtimeKey: []byte("1700000000")},
|
||||
})
|
||||
assert.Equal(t, time.Unix(1700000000, 0).UTC(), got.UTC())
|
||||
}
|
||||
|
||||
// ---------- logicalKeyFromVersionPath ----------
|
||||
|
||||
func TestLogicalKeyFromVersionPath_ExtractsLogicalKey(t *testing.T) {
|
||||
logical, ok := logicalKeyFromVersionPath("a/b/c.versions/v_abc")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "a/b/c", logical)
|
||||
}
|
||||
|
||||
func TestLogicalKeyFromVersionPath_RejectsPathWithoutVersionsParent(t *testing.T) {
|
||||
// The parent of the version file must end with .versions; otherwise
|
||||
// it's not a version path.
|
||||
_, ok := logicalKeyFromVersionPath("a/b/c/v_abc")
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestLogicalKeyFromVersionPath_RejectsRootLevelPath(t *testing.T) {
|
||||
// LastIndex returning 0 means the only "/" is at index 0, which
|
||||
// would yield an empty parent — not a real version path.
|
||||
_, ok := logicalKeyFromVersionPath("/v_abc")
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestLogicalKeyFromVersionPath_RejectsNoSlashes(t *testing.T) {
|
||||
_, ok := logicalKeyFromVersionPath("v_abc")
|
||||
assert.False(t, ok)
|
||||
_, ok = logicalKeyFromVersionPath("")
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestLogicalKeyFromVersionPath_RejectsBareVersionsContainer(t *testing.T) {
|
||||
// A path that's only the .versions container has no version-file
|
||||
// child to extract a logical key from.
|
||||
_, ok := logicalKeyFromVersionPath(s3_constants.VersionsFolder + "/v_x")
|
||||
assert.False(t, ok, "logical key cannot be empty after trim")
|
||||
}
|
||||
|
||||
// ---------- isVersionsContainerKey ----------
|
||||
|
||||
func TestIsVersionsContainerKey(t *testing.T) {
|
||||
cases := []struct {
|
||||
key string
|
||||
want bool
|
||||
}{
|
||||
{"obj" + s3_constants.VersionsFolder, true},
|
||||
{"a/b/obj" + s3_constants.VersionsFolder, true},
|
||||
// Bucket-root .versions is rejected explicitly: it has no
|
||||
// logical object key, so the router can't process it.
|
||||
{s3_constants.VersionsFolder, false},
|
||||
{"obj.versions/v_x", false}, // the version file inside, not the container
|
||||
{"obj", false},
|
||||
{"", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.key, func(t *testing.T) {
|
||||
assert.Equal(t, c.want, isVersionsContainerKey(c.key))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- isVersionFolderPath ----------
|
||||
|
||||
func TestIsVersionFolderPath(t *testing.T) {
|
||||
// Reports whether a key sits inside a .versions/ folder, looking at
|
||||
// the parent segment specifically. The router uses this to skip
|
||||
// version-file events that need sibling state to classify.
|
||||
cases := []struct {
|
||||
key string
|
||||
want bool
|
||||
}{
|
||||
{"obj.versions/v_aaa", true},
|
||||
{"a/b/obj.versions/v_aaa", true},
|
||||
{"obj/v_aaa", false}, // parent isn't .versions
|
||||
{"obj.versions", false}, // the container itself, not a child
|
||||
// "obj.versions/" reads as the trailing-slash form of the path
|
||||
// "obj.versions" → leaf is "obj.versions" which ends with the
|
||||
// suffix → true. The router never sees this shape from the
|
||||
// reader (events carry concrete leaf names), but pin it here so
|
||||
// the contract is documented.
|
||||
{"obj.versions/", true},
|
||||
{"v_aaa", false}, // no slash at all
|
||||
{"", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.key, func(t *testing.T) {
|
||||
assert.Equal(t, c.want, isVersionFolderPath(c.key))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- isDeleteMarkerEntry ----------
|
||||
|
||||
func TestIsDeleteMarkerEntry_NilOrEmptyExtendedReturnsFalse(t *testing.T) {
|
||||
assert.False(t, isDeleteMarkerEntry(nil))
|
||||
assert.False(t, isDeleteMarkerEntry(&filer_pb.Entry{}))
|
||||
assert.False(t, isDeleteMarkerEntry(&filer_pb.Entry{Extended: map[string][]byte{}}))
|
||||
}
|
||||
|
||||
func TestIsDeleteMarkerEntry_TrueOnlyForLiteralTrue(t *testing.T) {
|
||||
// The marker key must be exactly "true"; any other value is not a
|
||||
// marker. Pinning catches a regression that does case-insensitive
|
||||
// matching or treats "1" as truthy.
|
||||
mk := func(val string) *filer_pb.Entry {
|
||||
return &filer_pb.Entry{Extended: map[string][]byte{s3_constants.ExtDeleteMarkerKey: []byte(val)}}
|
||||
}
|
||||
assert.True(t, isDeleteMarkerEntry(mk("true")))
|
||||
assert.False(t, isDeleteMarkerEntry(mk("True")))
|
||||
assert.False(t, isDeleteMarkerEntry(mk("TRUE")))
|
||||
assert.False(t, isDeleteMarkerEntry(mk("1")))
|
||||
assert.False(t, isDeleteMarkerEntry(mk("")))
|
||||
assert.False(t, isDeleteMarkerEntry(mk("false")))
|
||||
}
|
||||
|
||||
// ---------- extractTags ----------
|
||||
|
||||
func TestExtractTags_NilOrEmptyReturnsNil(t *testing.T) {
|
||||
assert.Nil(t, extractTags(nil))
|
||||
assert.Nil(t, extractTags(map[string][]byte{}))
|
||||
}
|
||||
|
||||
func TestExtractTags_OnlyKeysWithObjectTaggingPrefix(t *testing.T) {
|
||||
// extractTags returns ExtVersionIdKey, Mime, etc. → no.
|
||||
// It only picks up keys with the AmzObjectTagging prefix and strips
|
||||
// that prefix to produce the tag map.
|
||||
prefix := s3_constants.AmzObjectTagging + "-"
|
||||
ext := map[string][]byte{
|
||||
prefix + "env": []byte("prod"),
|
||||
prefix + "team": []byte("data"),
|
||||
"X-Amz-Other": []byte("ignored"),
|
||||
"Seaweed-X-Internal": []byte("ignored"),
|
||||
}
|
||||
got := extractTags(ext)
|
||||
require.Len(t, got, 2)
|
||||
assert.Equal(t, "prod", got["env"])
|
||||
assert.Equal(t, "data", got["team"])
|
||||
}
|
||||
|
||||
func TestExtractTags_ReturnsNilWhenNoTaggingPrefixedKeys(t *testing.T) {
|
||||
// Keys with no tagging prefix produce a nil map (not an empty map);
|
||||
// the router treats nil as "no tags" without an extra branch.
|
||||
ext := map[string][]byte{
|
||||
"X-Amz-Other": []byte("ignored"),
|
||||
"Seaweed-X-Internal": []byte("ignored"),
|
||||
}
|
||||
got := extractTags(ext)
|
||||
assert.Nil(t, got)
|
||||
}
|
||||
|
||||
// ---------- hasActiveEventDrivenAction ----------
|
||||
|
||||
func TestHasActiveEventDrivenAction(t *testing.T) {
|
||||
// Build a snapshot with two action kinds and one of them set to
|
||||
// scan-only (inactive in event-driven view). hasActiveEventDrivenAction
|
||||
// must answer true only for the active event-driven kind.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 7,
|
||||
AbortMPUDaysAfterInitiation: 3,
|
||||
}
|
||||
hash := s3lifecycle.RuleHash(rule)
|
||||
expirationKey := s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: s3lifecycle.ActionKindExpirationDays}
|
||||
abortKey := s3lifecycle.ActionKey{Bucket: "bk", RuleHash: hash, ActionKind: s3lifecycle.ActionKindAbortMPU}
|
||||
prior := map[s3lifecycle.ActionKey]engine.PriorState{
|
||||
expirationKey: {BootstrapComplete: true, Mode: engine.ModeEventDriven},
|
||||
abortKey: {BootstrapComplete: true, Mode: engine.ModeScanOnly},
|
||||
}
|
||||
snap := engine.New().Compile(
|
||||
[]engine.CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
engine.CompileOptions{PriorStates: prior},
|
||||
)
|
||||
keys := []s3lifecycle.ActionKey{expirationKey, abortKey}
|
||||
|
||||
// Active event-driven kind matches.
|
||||
assert.True(t, hasActiveEventDrivenAction(snap, keys, s3lifecycle.ActionKindExpirationDays))
|
||||
// Scan-only kind does NOT match — operator promoted it to scan-only
|
||||
// and the router must respect that by skipping the event-driven path.
|
||||
assert.False(t, hasActiveEventDrivenAction(snap, keys, s3lifecycle.ActionKindAbortMPU))
|
||||
// A kind not in the keys list is false.
|
||||
assert.False(t, hasActiveEventDrivenAction(snap, keys, s3lifecycle.ActionKindNoncurrentDays))
|
||||
}
|
||||
|
||||
func TestHasActiveEventDrivenAction_NilActionSkipped(t *testing.T) {
|
||||
// A key the snapshot doesn't know about returns nil from Action;
|
||||
// the helper must skip rather than panic.
|
||||
snap := engine.New().Compile(nil, engine.CompileOptions{})
|
||||
keys := []s3lifecycle.ActionKey{
|
||||
{Bucket: "ghost", ActionKind: s3lifecycle.ActionKindExpirationDays},
|
||||
}
|
||||
assert.False(t, hasActiveEventDrivenAction(snap, keys, s3lifecycle.ActionKindExpirationDays))
|
||||
}
|
||||
|
||||
// ---------- engine.Snapshot accessors ----------
|
||||
|
||||
func TestSnapshot_BucketVersionedReportsCompiledFlag(t *testing.T) {
|
||||
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 7}
|
||||
snap := engine.New().Compile(
|
||||
[]engine.CompileInput{
|
||||
{Bucket: "vbk", Rules: []*s3lifecycle.Rule{rule}, Versioned: true},
|
||||
{Bucket: "ubk", Rules: []*s3lifecycle.Rule{rule}, Versioned: false},
|
||||
},
|
||||
engine.CompileOptions{},
|
||||
)
|
||||
assert.True(t, snap.BucketVersioned("vbk"))
|
||||
assert.False(t, snap.BucketVersioned("ubk"))
|
||||
// Unknown bucket: not versioned, not a panic.
|
||||
assert.False(t, snap.BucketVersioned("missing"))
|
||||
}
|
||||
|
||||
func TestSnapshot_BucketActionKeysReturnsCompiledList(t *testing.T) {
|
||||
// Every CompileInput rule emits one action key per RuleActionKind;
|
||||
// BucketActionKeys must surface them all (regardless of Mode) so
|
||||
// MatchPath can iterate.
|
||||
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 7}
|
||||
snap := engine.New().Compile(
|
||||
[]engine.CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
engine.CompileOptions{},
|
||||
)
|
||||
keys := snap.BucketActionKeys("bk")
|
||||
require.NotEmpty(t, keys)
|
||||
for _, k := range keys {
|
||||
assert.Equal(t, "bk", k.Bucket)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshot_BucketActionKeysUnknownBucketReturnsNil(t *testing.T) {
|
||||
snap := engine.New().Compile(nil, engine.CompileOptions{})
|
||||
assert.Nil(t, snap.BucketActionKeys("missing"))
|
||||
}
|
||||
|
||||
func TestSnapshot_ActionUnknownKeyReturnsNil(t *testing.T) {
|
||||
snap := engine.New().Compile(nil, engine.CompileOptions{})
|
||||
got := snap.Action(s3lifecycle.ActionKey{Bucket: "ghost", ActionKind: s3lifecycle.ActionKindExpirationDays})
|
||||
assert.Nil(t, got)
|
||||
}
|
||||
|
||||
func TestSnapshot_AllActionsCoversEveryCompiledKind(t *testing.T) {
|
||||
// AllActions must enumerate every compiled action regardless of
|
||||
// active state, because the dispatcher uses it for full-snapshot
|
||||
// reporting.
|
||||
rule := &s3lifecycle.Rule{
|
||||
ID: "r",
|
||||
Status: s3lifecycle.StatusEnabled,
|
||||
ExpirationDays: 7,
|
||||
AbortMPUDaysAfterInitiation: 3,
|
||||
}
|
||||
snap := engine.New().Compile(
|
||||
[]engine.CompileInput{{Bucket: "bk", Rules: []*s3lifecycle.Rule{rule}}},
|
||||
engine.CompileOptions{},
|
||||
)
|
||||
all := snap.AllActions()
|
||||
wantKinds := s3lifecycle.RuleActionKinds(rule)
|
||||
require.Len(t, all, len(wantKinds))
|
||||
seen := map[s3lifecycle.ActionKind]bool{}
|
||||
for _, a := range all {
|
||||
seen[a.Key.ActionKind] = true
|
||||
}
|
||||
for _, k := range wantKinds {
|
||||
assert.True(t, seen[k], "missing kind %v in AllActions", k)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshot_SnapshotIDIsMonotonicAcrossRecompiles(t *testing.T) {
|
||||
// Every Compile call advances snapshotIDSeq; pin that successive
|
||||
// snapshots from the same Engine carry strictly increasing IDs so
|
||||
// the dispatcher's stale-snapshot check works.
|
||||
e := engine.New()
|
||||
first := e.Compile(nil, engine.CompileOptions{})
|
||||
second := e.Compile(nil, engine.CompileOptions{})
|
||||
assert.Greater(t, second.SnapshotID(), first.SnapshotID(),
|
||||
"second compile must produce a strictly greater snapshot id")
|
||||
}
|
||||
873
weed/s3api/s3lifecycle/router/router.go
Normal file
873
weed/s3api/s3lifecycle/router/router.go
Normal file
@@ -0,0 +1,873 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader"
|
||||
)
|
||||
|
||||
// SiblingLister inspects the surviving versions of a versioned key.
|
||||
// nil receiver or an error means "unknown" — callers suppress. Four
|
||||
// queries: Survivors paginates the .versions/ container plus the bare
|
||||
// null version (used by sole-survivor and bootstrap); LookupVersion
|
||||
// fetches a single version file by id (used by pointer-transition
|
||||
// routing to read the displaced version's identity and mtime);
|
||||
// ListVersions paginates every version file in the .versions/
|
||||
// container (used to compute NoncurrentIndex when a NewerNoncurrent
|
||||
// rule is active); LookupNullVersion returns the bare-key entry that
|
||||
// represents the null version (used by pointer-transition routing
|
||||
// when oldID is empty and to include the null in expansion ranks).
|
||||
type SiblingLister interface {
|
||||
Survivors(ctx context.Context, bucket, objectKey string) (Survivors, error)
|
||||
LookupVersion(ctx context.Context, bucket, objectKey, versionID string) (*filer_pb.Entry, error)
|
||||
ListVersions(ctx context.Context, bucket, objectKey string) ([]*filer_pb.Entry, error)
|
||||
LookupNullVersion(ctx context.Context, bucket, objectKey string) (entry *filer_pb.Entry, explicit bool, err error)
|
||||
}
|
||||
|
||||
// Survivors describes the state under .versions/<key>/ plus the bare
|
||||
// null-version that exists when versioning was turned on after the
|
||||
// object was first PUT (s3api_object_versioning.go treats <bucket>/<key>
|
||||
// as a regular file, the null version, in that case).
|
||||
type Survivors struct {
|
||||
Count int // entries under .versions/<key>/, capped at 2
|
||||
LoneEntry *filer_pb.Entry // populated when Count == 1
|
||||
HasNullVersion bool // bare <bucket>/<key> exists as a regular file
|
||||
}
|
||||
|
||||
// Match is one (event, action) pair where EvaluateAction fired. The
|
||||
// dispatcher runs `LifecycleDelete` at DueTime; identity-CAS in the RPC
|
||||
// guards against drift between schedule time and dispatch time.
|
||||
//
|
||||
// For NoncurrentDays / NewerNoncurrent on a versioned bucket, ObjectKey
|
||||
// is the seaweedfs storage path (logical-key + ".versions/" + version_id)
|
||||
// so the dispatcher can locate the specific version, and VersionID
|
||||
// carries the AWS-visible version ID separately.
|
||||
type Match struct {
|
||||
Key s3lifecycle.ActionKey
|
||||
Action *engine.CompiledAction
|
||||
Result s3lifecycle.EvalResult
|
||||
EventTs time.Time
|
||||
DueTime time.Time
|
||||
Bucket string
|
||||
ObjectKey string
|
||||
VersionID string
|
||||
Identity *EntryIdentity
|
||||
}
|
||||
|
||||
// EntryIdentity is the schedule-time CAS witness; the dispatcher serializes
|
||||
// it into the LifecycleDelete request. The fields mirror
|
||||
// s3_lifecycle_pb.EntryIdentity but stay in-package so the router doesn't
|
||||
// pull a proto dependency.
|
||||
type EntryIdentity struct {
|
||||
MtimeNs int64
|
||||
Size int64
|
||||
HeadFid string
|
||||
ExtendedHash []byte
|
||||
}
|
||||
|
||||
// Route returns the matches that fire for ev against snap. Only active
|
||||
// event-driven actions are considered; SCAN_AT_DATE and DISABLED bypass
|
||||
// this path.
|
||||
func Route(ctx context.Context, snap *engine.Snapshot, ev *reader.Event, now time.Time, lister SiblingLister) []Match {
|
||||
if snap == nil || ev == nil {
|
||||
return nil
|
||||
}
|
||||
keys := snap.BucketActionKeys(ev.Bucket)
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Bootstrap-expanded version event: sibling state is pre-computed,
|
||||
// info.Key is the LOGICAL key so rule prefixes match. Skip the
|
||||
// meta-log path's version-folder skip.
|
||||
if ev.BootstrapVersion != nil {
|
||||
return routeBootstrapVersion(snap, ev, keys)
|
||||
}
|
||||
|
||||
versioned := snap.BucketVersioned(ev.Bucket)
|
||||
|
||||
// .versions/ directory metadata update: when ExtLatestVersionIdKey
|
||||
// changes, the OLD pointer value names a version that's now
|
||||
// noncurrent. Route NoncurrentDays / NewerNoncurrent for it without
|
||||
// waiting for the next bootstrap.
|
||||
if versioned && ev.NewEntry != nil && ev.OldEntry != nil && ev.NewEntry.IsDirectory && isVersionsContainerKey(ev.Key) {
|
||||
if !hasActiveEventDrivenAction(snap, keys, s3lifecycle.ActionKindNoncurrentDays) &&
|
||||
!hasActiveEventDrivenAction(snap, keys, s3lifecycle.ActionKindNewerNoncurrent) {
|
||||
return nil
|
||||
}
|
||||
return routePointerTransition(ctx, snap, ev, keys, lister)
|
||||
}
|
||||
|
||||
// EXP_DM can fire on two version-folder events: the marker create
|
||||
// (sole survivor immediately) and a noncurrent hard-delete that
|
||||
// leaves only the marker behind. Both reach routeSoleSurvivorMarker.
|
||||
if versioned && isVersionFolderPath(ev.Key) {
|
||||
if !hasActiveEventDrivenAction(snap, keys, s3lifecycle.ActionKindExpiredDeleteMarker) {
|
||||
return nil
|
||||
}
|
||||
return routeSoleSurvivorMarker(ctx, snap, ev, keys, lister)
|
||||
}
|
||||
|
||||
if ev.NewEntry == nil {
|
||||
return nil
|
||||
}
|
||||
info := buildObjectInfo(ev, versioned)
|
||||
if info == nil {
|
||||
return nil
|
||||
}
|
||||
eventTime := time.Unix(0, ev.TsNs)
|
||||
|
||||
var matches []Match
|
||||
for _, key := range keys {
|
||||
action := snap.Action(key)
|
||||
if action == nil || !action.IsActive() {
|
||||
continue
|
||||
}
|
||||
if action.Mode != engine.ModeEventDriven {
|
||||
continue
|
||||
}
|
||||
// (kind, info) shape gate: ABORT_MPU only fires on MPU init events,
|
||||
// and other kinds never do. Without this an MPU init would be
|
||||
// matched against NONCURRENT_DAYS (IsLatest=false reads as a
|
||||
// non-current version) and the dispatcher would BLOCK on empty
|
||||
// version_id.
|
||||
if info.IsMPUInit && key.ActionKind != s3lifecycle.ActionKindAbortMPU {
|
||||
continue
|
||||
}
|
||||
if !info.IsMPUInit && key.ActionKind == s3lifecycle.ActionKindAbortMPU {
|
||||
continue
|
||||
}
|
||||
// Schedule from the per-kind due moment. ExpirationDate is
|
||||
// rule-relative (the date IS the moment); other kinds are
|
||||
// ModTime-relative. Using ModTime+Delay for ExpirationDate
|
||||
// (Delay=0) puts dueTime at the entry's mtime — a backdated
|
||||
// object's mtime is BEFORE the rule's date, so the eligibility
|
||||
// check below would skip it. ComputeDueAt encapsulates both
|
||||
// shapes; the dispatcher's identity-CAS catches drift if the
|
||||
// object changes meanwhile.
|
||||
dueTime := s3lifecycle.ComputeDueAt(action.Rule, key.ActionKind, info)
|
||||
if dueTime.IsZero() {
|
||||
continue
|
||||
}
|
||||
res := s3lifecycle.EvaluateAction(action.Rule, key.ActionKind, info, dueTime)
|
||||
if res.Action == s3lifecycle.ActionNone {
|
||||
continue
|
||||
}
|
||||
matches = append(matches, Match{
|
||||
Key: key,
|
||||
Action: action,
|
||||
Result: res,
|
||||
EventTs: eventTime,
|
||||
DueTime: dueTime,
|
||||
Bucket: ev.Bucket,
|
||||
ObjectKey: ev.Key,
|
||||
Identity: buildIdentity(ev),
|
||||
})
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
// routePointerTransition handles a .versions/ container update where
|
||||
// ExtLatestVersionIdKey changed: the OLD pointer value names a version
|
||||
// that just became noncurrent. Two lookup shapes:
|
||||
//
|
||||
// - Pure NoncurrentVersionExpirationDays without NewerNoncurrentVersions:
|
||||
// a single LookupVersion of oldID is enough — the displaced version
|
||||
// is the only one that newly entered eligibility for this rule.
|
||||
//
|
||||
// - Any active NewerNoncurrentVersions rule: a pointer flip shifts
|
||||
// every prior noncurrent's rank by one, so the version that *just
|
||||
// crossed* the keep-count threshold needs evaluation too. List the
|
||||
// full .versions/ container, rank newest-first, and route every
|
||||
// eligible noncurrent. Identity-CAS handles dedup with earlier
|
||||
// schedules.
|
||||
//
|
||||
// Without this branch the worker has to wait for the next bootstrap to
|
||||
// schedule retention on a freshly-noncurrent version.
|
||||
func routePointerTransition(ctx context.Context, snap *engine.Snapshot, ev *reader.Event, keys []s3lifecycle.ActionKey, lister SiblingLister) []Match {
|
||||
if lister == nil {
|
||||
return nil
|
||||
}
|
||||
logical := strings.TrimSuffix(ev.Key, s3_constants.VersionsFolder)
|
||||
if logical == "" {
|
||||
return nil
|
||||
}
|
||||
oldID := string(ev.OldEntry.Extended[s3_constants.ExtLatestVersionIdKey])
|
||||
newID := string(ev.NewEntry.Extended[s3_constants.ExtLatestVersionIdKey])
|
||||
if oldID == newID {
|
||||
// Same id means the update didn't transition the pointer.
|
||||
return nil
|
||||
}
|
||||
// oldID == "" doesn't mean "nothing displaced": a bare null may
|
||||
// have been the implicit/explicit latest before the pointer
|
||||
// flipped to a real id.
|
||||
// newID == "" means a suspended-versioning write cleared the
|
||||
// pointer and made the bare null current. The cached
|
||||
// ExtLatestVersionMtimeKey may still hold the prior latest's
|
||||
// mtime (stale), so we must NOT use successorModTimeFromContainer
|
||||
// in that case — derive the successor clock from the null entry's
|
||||
// mtime instead. latestIDForExpand carries the same substitution
|
||||
// so the expansion path's latestPos lookup matches the null sibling.
|
||||
var successor time.Time
|
||||
latestIDForExpand := newID
|
||||
if newID == "" {
|
||||
nullEntry, _, err := lister.LookupNullVersion(ctx, ev.Bucket, logical)
|
||||
if err != nil {
|
||||
glog.V(2).Infof("lifecycle router: lookup null %s/%s: %v", ev.Bucket, logical, err)
|
||||
return nil
|
||||
}
|
||||
if nullEntry == nil || nullEntry.Attributes == nil {
|
||||
return nil
|
||||
}
|
||||
successor = time.Unix(nullEntry.Attributes.Mtime, int64(nullEntry.Attributes.MtimeNs))
|
||||
latestIDForExpand = "null"
|
||||
} else {
|
||||
successor = successorModTimeFromContainer(ev.NewEntry)
|
||||
}
|
||||
if successor.IsZero() {
|
||||
return nil
|
||||
}
|
||||
if needsFullExpansion(snap, keys) {
|
||||
return routePointerTransitionExpand(ctx, snap, ev, keys, lister, logical, latestIDForExpand, successor)
|
||||
}
|
||||
return routePointerTransitionDisplaced(ctx, snap, ev, keys, lister, logical, oldID, successor)
|
||||
}
|
||||
|
||||
// needsFullExpansion reports whether any active event-driven rule on
|
||||
// this bucket cares about NoncurrentIndex (NewerNoncurrentVersions > 0
|
||||
// in either NoncurrentDays or pure-count NewerNoncurrent).
|
||||
func needsFullExpansion(snap *engine.Snapshot, keys []s3lifecycle.ActionKey) bool {
|
||||
for _, k := range keys {
|
||||
if k.ActionKind != s3lifecycle.ActionKindNoncurrentDays && k.ActionKind != s3lifecycle.ActionKindNewerNoncurrent {
|
||||
continue
|
||||
}
|
||||
a := snap.Action(k)
|
||||
if a == nil || !a.IsActive() || a.Mode != engine.ModeEventDriven {
|
||||
continue
|
||||
}
|
||||
if a.Rule != nil && a.Rule.NewerNoncurrentVersions > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// successorModTimeFromContainer reads the cached latest-version mtime
|
||||
// from the .versions/ container's Extended map.
|
||||
// updateLatestVersionInDirectory writes it via setCachedListMetadata
|
||||
// alongside ExtLatestVersionIdKey, but the directory's own
|
||||
// Attributes.Mtime is preserved across pointer updates — using it
|
||||
// directly would let a stale dir mtime trigger expiration immediately.
|
||||
// Returns zero time if the cached mtime is missing or unparseable; the
|
||||
// caller suppresses in that case.
|
||||
func successorModTimeFromContainer(entry *filer_pb.Entry) time.Time {
|
||||
raw, ok := entry.Extended[s3_constants.ExtLatestVersionMtimeKey]
|
||||
if !ok || len(raw) == 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
secs, err := strconv.ParseInt(string(raw), 10, 64)
|
||||
if err != nil || secs <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Unix(secs, 0)
|
||||
}
|
||||
|
||||
// routePointerTransitionDisplaced is the single-lookup path: only the
|
||||
// displaced version's noncurrent eligibility could have changed, so
|
||||
// fetching just its file is enough. oldID == "" routes the bare null
|
||||
// version instead — it was the implicit latest before the pointer
|
||||
// flipped to a real id.
|
||||
func routePointerTransitionDisplaced(ctx context.Context, snap *engine.Snapshot, ev *reader.Event, keys []s3lifecycle.ActionKey, lister SiblingLister, logical, oldID string, successor time.Time) []Match {
|
||||
var displaced *filer_pb.Entry
|
||||
displacedID := oldID
|
||||
if oldID == "" {
|
||||
nullEntry, _, err := lister.LookupNullVersion(ctx, ev.Bucket, logical)
|
||||
if err != nil {
|
||||
glog.V(2).Infof("lifecycle router: lookup null version %s/%s: %v", ev.Bucket, logical, err)
|
||||
return nil
|
||||
}
|
||||
if nullEntry == nil {
|
||||
return nil
|
||||
}
|
||||
displaced = nullEntry
|
||||
displacedID = "null"
|
||||
} else {
|
||||
entry, err := lister.LookupVersion(ctx, ev.Bucket, logical, oldID)
|
||||
if err != nil {
|
||||
glog.V(2).Infof("lifecycle router: lookup displaced version %s/%s/%s: %v", ev.Bucket, logical, oldID, err)
|
||||
return nil
|
||||
}
|
||||
displaced = entry
|
||||
}
|
||||
if displaced == nil || displaced.Attributes == nil {
|
||||
return nil
|
||||
}
|
||||
// Prefer the explicit demotion stamp on the displaced entry over the
|
||||
// container-derived successor. Stamp is written by the S3 PUT handler
|
||||
// at the moment the pointer flipped; container value is derived from
|
||||
// the new latest's mtime and may drift across retries.
|
||||
effectiveSuccessor := successor
|
||||
if stamp := s3lifecycle.SuccessorFromEntryStamp(displaced); !stamp.IsZero() {
|
||||
effectiveSuccessor = stamp
|
||||
}
|
||||
idx := 0
|
||||
info := &s3lifecycle.ObjectInfo{
|
||||
Key: logical,
|
||||
ModTime: time.Unix(displaced.Attributes.Mtime, int64(displaced.Attributes.MtimeNs)),
|
||||
Size: int64(displaced.Attributes.FileSize),
|
||||
IsLatest: false,
|
||||
IsDeleteMarker: string(displaced.Extended[s3_constants.ExtDeleteMarkerKey]) == "true",
|
||||
NoncurrentIndex: &idx,
|
||||
SuccessorModTime: effectiveSuccessor,
|
||||
}
|
||||
if tags := extractTags(displaced.Extended); len(tags) > 0 {
|
||||
info.Tags = tags
|
||||
}
|
||||
return emitNoncurrentMatches(snap, ev, keys, info, displaced, displacedID, effectiveSuccessor)
|
||||
}
|
||||
|
||||
// routePointerTransitionExpand routes only the versions that newly
|
||||
// became eligible by the pointer flip:
|
||||
//
|
||||
// - rank 0: the displaced version (newly noncurrent), needed for the
|
||||
// pure-NoncurrentDays clock,
|
||||
// - rank == rule.NewerNoncurrentVersions for each active rule that
|
||||
// gates on count: the version at exactly that rank just crossed
|
||||
// from kept to expired.
|
||||
//
|
||||
// Emitting every eligible noncurrent on every PUT would push
|
||||
// O(versions) heap entries per flip — Schedule.Add doesn't dedup, so
|
||||
// identity-CAS at dispatch only stops the wasted RPC, not the heap
|
||||
// growth. Bootstrap still owns full backfill.
|
||||
func routePointerTransitionExpand(ctx context.Context, snap *engine.Snapshot, ev *reader.Event, keys []s3lifecycle.ActionKey, lister SiblingLister, logical, newID string, successor time.Time) []Match {
|
||||
rawVersions, err := lister.ListVersions(ctx, ev.Bucket, logical)
|
||||
if err != nil {
|
||||
glog.V(2).Infof("lifecycle router: list versions %s/%s: %v", ev.Bucket, logical, err)
|
||||
return nil
|
||||
}
|
||||
// Include the bare null version in the sibling set: count-based
|
||||
// ranks are wrong if a pre-versioning or suspended-null entry
|
||||
// exists outside .versions/. ID is "null" for sort + emit.
|
||||
nullEntry, _, nullErr := lister.LookupNullVersion(ctx, ev.Bucket, logical)
|
||||
if nullErr != nil {
|
||||
glog.V(2).Infof("lifecycle router: lookup null %s/%s: %v", ev.Bucket, logical, nullErr)
|
||||
return nil
|
||||
}
|
||||
type sibling struct {
|
||||
entry *filer_pb.Entry
|
||||
id string
|
||||
}
|
||||
siblings := make([]sibling, 0, len(rawVersions)+1)
|
||||
for _, v := range rawVersions {
|
||||
if v == nil || v.Attributes == nil {
|
||||
continue
|
||||
}
|
||||
id := string(v.Extended[s3_constants.ExtVersionIdKey])
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
siblings = append(siblings, sibling{entry: v, id: id})
|
||||
}
|
||||
if nullEntry != nil && nullEntry.Attributes != nil {
|
||||
siblings = append(siblings, sibling{entry: nullEntry, id: "null"})
|
||||
}
|
||||
if len(siblings) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.SliceStable(siblings, func(i, j int) bool {
|
||||
mi := siblings[i].entry.Attributes.Mtime*int64(1e9) + int64(siblings[i].entry.Attributes.MtimeNs)
|
||||
mj := siblings[j].entry.Attributes.Mtime*int64(1e9) + int64(siblings[j].entry.Attributes.MtimeNs)
|
||||
if mi != mj {
|
||||
return mi > mj
|
||||
}
|
||||
return s3lifecycle.CompareVersionIds(siblings[i].id, siblings[j].id) < 0
|
||||
})
|
||||
// Resolve latestPos by finding newID. Default to -1 so a missing
|
||||
// newID (race with the listing, or torn write) suppresses the
|
||||
// expansion: we'd otherwise call the actual newest sibling "latest"
|
||||
// against the pointer's intent and misrank every noncurrent.
|
||||
// Bootstrap repairs state on the next walk.
|
||||
latestPos := -1
|
||||
if newID != "" {
|
||||
for i, s := range siblings {
|
||||
if s.id == newID {
|
||||
latestPos = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if latestPos < 0 {
|
||||
glog.V(2).Infof("lifecycle router: pointer transition %s/%s: new id %s not found in listing", ev.Bucket, logical, newID)
|
||||
return nil
|
||||
}
|
||||
noncurrentCount := len(siblings) - 1
|
||||
|
||||
// Collect the target noncurrent ranks: 0 (the freshly displaced)
|
||||
// plus N for each active count-gated rule.
|
||||
rankSet := map[int]struct{}{0: {}}
|
||||
for _, k := range keys {
|
||||
if k.ActionKind != s3lifecycle.ActionKindNoncurrentDays && k.ActionKind != s3lifecycle.ActionKindNewerNoncurrent {
|
||||
continue
|
||||
}
|
||||
a := snap.Action(k)
|
||||
if a == nil || !a.IsActive() || a.Mode != engine.ModeEventDriven {
|
||||
continue
|
||||
}
|
||||
if a.Rule != nil && a.Rule.NewerNoncurrentVersions > 0 {
|
||||
rankSet[a.Rule.NewerNoncurrentVersions] = struct{}{}
|
||||
}
|
||||
}
|
||||
ranks := make([]int, 0, len(rankSet))
|
||||
for r := range rankSet {
|
||||
ranks = append(ranks, r)
|
||||
}
|
||||
sort.Ints(ranks)
|
||||
|
||||
var matches []Match
|
||||
for _, rank := range ranks {
|
||||
if rank >= noncurrentCount {
|
||||
continue
|
||||
}
|
||||
// Convert noncurrent rank to position in the sorted slice,
|
||||
// skipping the latest's slot.
|
||||
i := rank
|
||||
if rank >= latestPos {
|
||||
i = rank + 1
|
||||
}
|
||||
s := siblings[i]
|
||||
// Successor mtime: the entry directly newer than this one in
|
||||
// the sorted list. When the next-newer slot is the latest,
|
||||
// use the cached successor (the new latest's mtime); otherwise
|
||||
// the immediate predecessor's mtime.
|
||||
var thisSuccessor time.Time
|
||||
if i > 0 && i-1 != latestPos {
|
||||
thisSuccessor = time.Unix(siblings[i-1].entry.Attributes.Mtime, int64(siblings[i-1].entry.Attributes.MtimeNs))
|
||||
} else {
|
||||
thisSuccessor = successor
|
||||
}
|
||||
// Override with the explicit demotion stamp when present —
|
||||
// PUT-time wall clock beats derived sibling mtime for accuracy
|
||||
// and is immune to mtime edits on the sibling itself.
|
||||
if stamp := s3lifecycle.SuccessorFromEntryStamp(s.entry); !stamp.IsZero() {
|
||||
thisSuccessor = stamp
|
||||
}
|
||||
idx := rank
|
||||
info := &s3lifecycle.ObjectInfo{
|
||||
Key: logical,
|
||||
ModTime: time.Unix(s.entry.Attributes.Mtime, int64(s.entry.Attributes.MtimeNs)),
|
||||
Size: int64(s.entry.Attributes.FileSize),
|
||||
IsLatest: false,
|
||||
IsDeleteMarker: string(s.entry.Extended[s3_constants.ExtDeleteMarkerKey]) == "true",
|
||||
NoncurrentIndex: &idx,
|
||||
SuccessorModTime: thisSuccessor,
|
||||
NumVersions: len(siblings),
|
||||
}
|
||||
if tags := extractTags(s.entry.Extended); len(tags) > 0 {
|
||||
info.Tags = tags
|
||||
}
|
||||
matches = append(matches, emitNoncurrentMatches(snap, ev, keys, info, s.entry, s.id, thisSuccessor)...)
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
// emitNoncurrentMatches walks NoncurrentDays / NewerNoncurrent action
|
||||
// keys and emits Matches for each one that fires. Shared between the
|
||||
// single-lookup and full-expansion paths.
|
||||
func emitNoncurrentMatches(snap *engine.Snapshot, ev *reader.Event, keys []s3lifecycle.ActionKey, info *s3lifecycle.ObjectInfo, entry *filer_pb.Entry, versionID string, successor time.Time) []Match {
|
||||
eventTime := time.Unix(0, ev.TsNs)
|
||||
identity := buildIdentityFromEntry(entry)
|
||||
var matches []Match
|
||||
for _, key := range keys {
|
||||
if key.ActionKind != s3lifecycle.ActionKindNoncurrentDays && key.ActionKind != s3lifecycle.ActionKindNewerNoncurrent {
|
||||
continue
|
||||
}
|
||||
action := snap.Action(key)
|
||||
if action == nil || !action.IsActive() || action.Mode != engine.ModeEventDriven {
|
||||
continue
|
||||
}
|
||||
clock := successor
|
||||
if clock.IsZero() {
|
||||
clock = info.ModTime
|
||||
}
|
||||
dueTime := clock.Add(action.Delay)
|
||||
res := s3lifecycle.EvaluateAction(action.Rule, key.ActionKind, info, dueTime)
|
||||
if res.Action == s3lifecycle.ActionNone {
|
||||
continue
|
||||
}
|
||||
matches = append(matches, Match{
|
||||
Key: key,
|
||||
Action: action,
|
||||
Result: res,
|
||||
EventTs: eventTime,
|
||||
DueTime: dueTime,
|
||||
Bucket: ev.Bucket,
|
||||
ObjectKey: info.Key,
|
||||
VersionID: versionID,
|
||||
Identity: identity,
|
||||
})
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
// routeSoleSurvivorMarker emits an EXP_DM Match against the LOGICAL key
|
||||
// (so the dispatcher can call deleteSpecificObjectVersion) with the
|
||||
// marker's version_id. Handles two events: a marker create (the new
|
||||
// entry IS the marker) and a noncurrent hard-delete that leaves the
|
||||
// marker behind (the listing's lone entry IS the marker). The server
|
||||
// re-checks before deleting.
|
||||
func routeSoleSurvivorMarker(ctx context.Context, snap *engine.Snapshot, ev *reader.Event, keys []s3lifecycle.ActionKey, lister SiblingLister) []Match {
|
||||
if lister == nil {
|
||||
return nil
|
||||
}
|
||||
// Skip the listing RPC for events that can't possibly produce a
|
||||
// sole-survivor marker: a regular non-marker version create/update
|
||||
// always lands at Count >= 2.
|
||||
if ev.NewEntry != nil && !isDeleteMarkerEntry(ev.NewEntry) {
|
||||
return nil
|
||||
}
|
||||
logicalKey, ok := logicalKeyFromVersionPath(ev.Key)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
s, err := lister.Survivors(ctx, ev.Bucket, logicalKey)
|
||||
if err != nil {
|
||||
glog.V(2).Infof("lifecycle router: survivors %s/%s: %v", ev.Bucket, logicalKey, err)
|
||||
return nil
|
||||
}
|
||||
// Pre-versioning bare-key objects (the "null" version) live outside
|
||||
// .versions/. Treating count==1 as sole-survivor while a null
|
||||
// version exists would let lifecycle delete the marker and re-expose
|
||||
// the old object.
|
||||
if s.Count != 1 || s.HasNullVersion || s.LoneEntry == nil {
|
||||
return nil
|
||||
}
|
||||
if !isDeleteMarkerEntry(s.LoneEntry) {
|
||||
return nil
|
||||
}
|
||||
versionID := string(s.LoneEntry.Extended[s3_constants.ExtVersionIdKey])
|
||||
if versionID == "" {
|
||||
// Empty version_id would BLOCK at dispatch and freeze the cursor.
|
||||
return nil
|
||||
}
|
||||
entry := s.LoneEntry
|
||||
info := &s3lifecycle.ObjectInfo{
|
||||
Key: logicalKey,
|
||||
ModTime: time.Unix(entry.Attributes.Mtime, int64(entry.Attributes.MtimeNs)),
|
||||
Size: int64(entry.Attributes.FileSize),
|
||||
IsLatest: true,
|
||||
IsDeleteMarker: true,
|
||||
NumVersions: 1,
|
||||
}
|
||||
if tags := extractTags(entry.Extended); len(tags) > 0 {
|
||||
info.Tags = tags
|
||||
}
|
||||
eventTime := time.Unix(0, ev.TsNs)
|
||||
identity := buildIdentityFromEntry(entry)
|
||||
var matches []Match
|
||||
for _, key := range keys {
|
||||
if key.ActionKind != s3lifecycle.ActionKindExpiredDeleteMarker {
|
||||
continue
|
||||
}
|
||||
action := snap.Action(key)
|
||||
if action == nil || !action.IsActive() || action.Mode != engine.ModeEventDriven {
|
||||
continue
|
||||
}
|
||||
dueTime := info.ModTime.Add(action.Delay)
|
||||
res := s3lifecycle.EvaluateAction(action.Rule, key.ActionKind, info, dueTime)
|
||||
if res.Action == s3lifecycle.ActionNone {
|
||||
continue
|
||||
}
|
||||
matches = append(matches, Match{
|
||||
Key: key,
|
||||
Action: action,
|
||||
Result: res,
|
||||
EventTs: eventTime,
|
||||
DueTime: dueTime,
|
||||
Bucket: ev.Bucket,
|
||||
ObjectKey: logicalKey,
|
||||
VersionID: versionID,
|
||||
Identity: identity,
|
||||
})
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
// routeBootstrapVersion handles a synthesized event from BucketBootstrapper.
|
||||
// The bootstrap walker has already listed .versions/<key>/, sorted siblings
|
||||
// newest-first, and stamped each one's IsLatest / NoncurrentIndex /
|
||||
// SuccessorModTime. The router only needs to assemble ObjectInfo and run
|
||||
// the match loop with the standard kind gates. ev.NewEntry is the version
|
||||
// file itself; ev.Key is the version-folder path; the LOGICAL key from
|
||||
// BootstrapVersion drives prefix matching and the dispatcher.
|
||||
func routeBootstrapVersion(snap *engine.Snapshot, ev *reader.Event, keys []s3lifecycle.ActionKey) []Match {
|
||||
bv := ev.BootstrapVersion
|
||||
entry := ev.NewEntry
|
||||
if entry == nil || entry.Attributes == nil || bv.LogicalKey == "" {
|
||||
return nil
|
||||
}
|
||||
idx := bv.NoncurrentIndex
|
||||
info := &s3lifecycle.ObjectInfo{
|
||||
Key: bv.LogicalKey,
|
||||
ModTime: time.Unix(entry.Attributes.Mtime, int64(entry.Attributes.MtimeNs)),
|
||||
Size: int64(entry.Attributes.FileSize),
|
||||
IsLatest: bv.IsLatest,
|
||||
IsDeleteMarker: bv.IsDeleteMarker,
|
||||
NumVersions: bv.NumVersions,
|
||||
SuccessorModTime: bv.SuccessorModTime,
|
||||
}
|
||||
if !bv.IsLatest {
|
||||
info.NoncurrentIndex = &idx
|
||||
}
|
||||
if tags := extractTags(entry.Extended); len(tags) > 0 {
|
||||
info.Tags = tags
|
||||
}
|
||||
eventTime := time.Unix(0, ev.TsNs)
|
||||
identity := buildIdentityFromEntry(entry)
|
||||
var matches []Match
|
||||
for _, key := range keys {
|
||||
action := snap.Action(key)
|
||||
if action == nil || !action.IsActive() || action.Mode != engine.ModeEventDriven {
|
||||
continue
|
||||
}
|
||||
// ABORT_MPU never applies to a versioned object.
|
||||
if key.ActionKind == s3lifecycle.ActionKindAbortMPU {
|
||||
continue
|
||||
}
|
||||
// Noncurrent rules clock from when this version was replaced
|
||||
// (SuccessorModTime), not from when it was originally written.
|
||||
// Bootstrap populates SuccessorModTime; fall back to ModTime
|
||||
// for the latest version (no successor exists).
|
||||
clock := info.ModTime
|
||||
if !info.IsLatest && !info.SuccessorModTime.IsZero() {
|
||||
clock = info.SuccessorModTime
|
||||
}
|
||||
dueTime := clock.Add(action.Delay)
|
||||
res := s3lifecycle.EvaluateAction(action.Rule, key.ActionKind, info, dueTime)
|
||||
if res.Action == s3lifecycle.ActionNone {
|
||||
continue
|
||||
}
|
||||
// Pin the version_id only for kinds the dispatcher needs to
|
||||
// target by version: noncurrent retention and the marker
|
||||
// itself. EXPIRATION_DAYS / EXPIRATION_DATE on the latest
|
||||
// must NOT carry it — between schedule and dispatch a fresh
|
||||
// PUT can land, and identity-CAS against the original
|
||||
// version's bytes would still pass even though the latest has
|
||||
// moved on. Empty VersionID makes the dispatcher fetch the
|
||||
// current latest, where identity-CAS resolves to STALE_IDENTITY
|
||||
// and bootstrap re-schedules with the new latest's identity.
|
||||
var matchVersionID string
|
||||
switch key.ActionKind {
|
||||
case s3lifecycle.ActionKindNoncurrentDays,
|
||||
s3lifecycle.ActionKindNewerNoncurrent,
|
||||
s3lifecycle.ActionKindExpiredDeleteMarker:
|
||||
matchVersionID = bv.VersionID
|
||||
}
|
||||
matches = append(matches, Match{
|
||||
Key: key,
|
||||
Action: action,
|
||||
Result: res,
|
||||
EventTs: eventTime,
|
||||
DueTime: dueTime,
|
||||
Bucket: ev.Bucket,
|
||||
ObjectKey: bv.LogicalKey,
|
||||
VersionID: matchVersionID,
|
||||
Identity: identity,
|
||||
})
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
// logicalKeyFromVersionPath extracts <logical> from <logical>.versions/<file>.
|
||||
// Returns false for a bucket-root marker (path = ".versions/<file>"), since
|
||||
// AWS S3 has no concept of an object at the bucket key itself.
|
||||
func logicalKeyFromVersionPath(versionPath string) (string, bool) {
|
||||
lastSlash := strings.LastIndex(versionPath, "/")
|
||||
if lastSlash <= 0 {
|
||||
return "", false
|
||||
}
|
||||
parent := versionPath[:lastSlash]
|
||||
if !strings.HasSuffix(parent, s3_constants.VersionsFolder) {
|
||||
return "", false
|
||||
}
|
||||
logical := strings.TrimSuffix(parent, s3_constants.VersionsFolder)
|
||||
if logical == "" {
|
||||
return "", false
|
||||
}
|
||||
return logical, true
|
||||
}
|
||||
|
||||
// buildObjectInfo derives an ObjectInfo from a meta-log event. Returns
|
||||
// nil for shapes the router can't classify safely: missing attributes,
|
||||
// non-MPU directories, version-folder files (those route through
|
||||
// routeSoleSurvivorMarker upstream when EXP_DM applies). On a versioned
|
||||
// bucket the latest pointer lives in the .versions/ directory's
|
||||
// Extended map; without it we leave NumVersions=0 so the bootstrap walk
|
||||
// drives noncurrent retention.
|
||||
func buildObjectInfo(ev *reader.Event, versioned bool) *s3lifecycle.ObjectInfo {
|
||||
entry := ev.NewEntry
|
||||
if entry == nil || entry.Attributes == nil {
|
||||
return nil
|
||||
}
|
||||
if destKey, ok := mpuInitInfo(ev, entry); ok {
|
||||
return &s3lifecycle.ObjectInfo{
|
||||
Key: destKey,
|
||||
ModTime: time.Unix(entry.Attributes.Mtime, int64(entry.Attributes.MtimeNs)),
|
||||
IsMPUInit: true,
|
||||
}
|
||||
}
|
||||
if entry.IsDirectory {
|
||||
return nil
|
||||
}
|
||||
info := &s3lifecycle.ObjectInfo{
|
||||
Key: ev.Key,
|
||||
ModTime: time.Unix(entry.Attributes.Mtime, int64(entry.Attributes.MtimeNs)),
|
||||
Size: int64(entry.Attributes.FileSize),
|
||||
IsLatest: true,
|
||||
NumVersions: 1,
|
||||
}
|
||||
if versioned {
|
||||
if isVersionFolderPath(ev.Key) {
|
||||
return nil
|
||||
}
|
||||
info.NumVersions = 0
|
||||
}
|
||||
if tags := extractTags(entry.Extended); len(tags) > 0 {
|
||||
info.Tags = tags
|
||||
}
|
||||
if isDeleteMarkerEntry(entry) {
|
||||
info.IsDeleteMarker = true
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
// isVersionsContainerKey reports whether the bucket-relative key IS a
|
||||
// .versions/ container itself (e.g. "logs/foo.versions"), as opposed to
|
||||
// a file inside one. Used to recognize directory-level events whose
|
||||
// Extended map carries the latest pointer for an object.
|
||||
func isVersionsContainerKey(key string) bool {
|
||||
if key == s3_constants.VersionsFolder {
|
||||
// Bucket-root .versions: no logical object key.
|
||||
return false
|
||||
}
|
||||
return strings.HasSuffix(key, s3_constants.VersionsFolder)
|
||||
}
|
||||
|
||||
// isVersionFolderPath reports whether the bucket-relative key sits inside a
|
||||
// .versions/ folder — i.e. the path's parent segment ends with the
|
||||
// VersionsFolder suffix. Used by the versioned-bucket gate so the router
|
||||
// skips version-file events that need sibling state to be classified
|
||||
// safely.
|
||||
func isVersionFolderPath(key string) bool {
|
||||
idx := strings.LastIndex(key, "/")
|
||||
if idx <= 0 {
|
||||
return false
|
||||
}
|
||||
parent := key[:idx]
|
||||
parentIdx := strings.LastIndex(parent, "/")
|
||||
leaf := parent[parentIdx+1:]
|
||||
return strings.HasSuffix(leaf, s3_constants.VersionsFolder)
|
||||
}
|
||||
|
||||
// mpuInitInfo recognizes a multipart-upload init: a directory entry at
|
||||
// `.uploads/<upload_id>` carrying the destination key in Extended. Sub-events
|
||||
// for part uploads (deeper paths under the upload directory) are deliberately
|
||||
// rejected — they ride a different mtime and would over-fire ABORT_MPU.
|
||||
func mpuInitInfo(ev *reader.Event, entry *filer_pb.Entry) (destKey string, ok bool) {
|
||||
uploadsPrefix := s3_constants.MultipartUploadsFolder + "/"
|
||||
if !entry.IsDirectory || !strings.HasPrefix(ev.Key, uploadsPrefix) {
|
||||
return "", false
|
||||
}
|
||||
rest := ev.Key[len(uploadsPrefix):]
|
||||
if rest == "" || strings.ContainsRune(rest, '/') {
|
||||
// `.uploads/` itself or `.uploads/<id>/<part>...`; not the init.
|
||||
return "", false
|
||||
}
|
||||
keyBytes, hasKey := entry.Extended[s3_constants.ExtMultipartObjectKey]
|
||||
if !hasKey || len(keyBytes) == 0 {
|
||||
return "", false
|
||||
}
|
||||
return string(keyBytes), true
|
||||
}
|
||||
|
||||
// buildIdentity captures the entry's schedule-time fingerprint for the CAS
|
||||
// witness. Returns nil if the event has no entry to fingerprint (deletes).
|
||||
func buildIdentity(ev *reader.Event) *EntryIdentity {
|
||||
return buildIdentityFromEntry(ev.NewEntry)
|
||||
}
|
||||
|
||||
func buildIdentityFromEntry(entry *filer_pb.Entry) *EntryIdentity {
|
||||
if entry == nil {
|
||||
return nil
|
||||
}
|
||||
id := &EntryIdentity{}
|
||||
if entry.Attributes != nil {
|
||||
// Mirror the server-side computeEntryIdentity encoding: Mtime
|
||||
// (seconds) and MtimeNs (nanosecond component) combine into
|
||||
// EntryIdentity.MtimeNs as nanoseconds-since-epoch.
|
||||
id.MtimeNs = entry.Attributes.Mtime*int64(1e9) + int64(entry.Attributes.MtimeNs)
|
||||
id.Size = int64(entry.Attributes.FileSize)
|
||||
}
|
||||
if len(entry.GetChunks()) > 0 {
|
||||
// Meta-log events arrive with chunk.FileId cleared by
|
||||
// BeforeEntrySerialization; GetFileIdString reconstructs it from
|
||||
// Fid so the worker matches the server-side fingerprint.
|
||||
id.HeadFid = entry.GetChunks()[0].GetFileIdString()
|
||||
}
|
||||
id.ExtendedHash = s3lifecycle.HashExtended(entry.Extended)
|
||||
return id
|
||||
}
|
||||
|
||||
func extractTags(ext map[string][]byte) map[string]string {
|
||||
if len(ext) == 0 {
|
||||
return nil
|
||||
}
|
||||
prefix := s3_constants.AmzObjectTagging + "-"
|
||||
var out map[string]string
|
||||
for k, v := range ext {
|
||||
if !strings.HasPrefix(k, prefix) {
|
||||
continue
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]string{}
|
||||
}
|
||||
out[k[len(prefix):]] = string(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// hasActiveEventDrivenAction gates I/O (e.g. sibling listing) on whether
|
||||
// a match could actually fire. Mirrors the per-key filter in Route so
|
||||
// disabled or scan-only actions don't pay the RPC.
|
||||
func hasActiveEventDrivenAction(snap *engine.Snapshot, keys []s3lifecycle.ActionKey, kind s3lifecycle.ActionKind) bool {
|
||||
for _, k := range keys {
|
||||
if k.ActionKind != kind {
|
||||
continue
|
||||
}
|
||||
a := snap.Action(k)
|
||||
if a == nil {
|
||||
continue
|
||||
}
|
||||
if a.IsActive() && a.Mode == engine.ModeEventDriven {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isDeleteMarkerEntry mirrors every read site for ExtDeleteMarkerKey:
|
||||
// production writes []byte("true").
|
||||
func isDeleteMarkerEntry(entry *filer_pb.Entry) bool {
|
||||
if entry == nil || len(entry.Extended) == 0 {
|
||||
return false
|
||||
}
|
||||
v, ok := entry.Extended[s3_constants.ExtDeleteMarkerKey]
|
||||
return ok && string(v) == "true"
|
||||
}
|
||||
1652
weed/s3api/s3lifecycle/router/router_test.go
Normal file
1652
weed/s3api/s3lifecycle/router/router_test.go
Normal file
File diff suppressed because it is too large
Load Diff
76
weed/s3api/s3lifecycle/router/schedule.go
Normal file
76
weed/s3api/s3lifecycle/router/schedule.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Schedule is the worker's in-memory pending list, ordered by DueTime.
|
||||
// The dispatcher polls Drain on each tick to retrieve due Matches.
|
||||
//
|
||||
// Duplicates are allowed: a re-schedule for the same (ActionKey, ObjectKey)
|
||||
// before the prior dispatch ran results in two heap entries. The
|
||||
// LifecycleDelete RPC's identity-CAS makes the second dispatch a no-op
|
||||
// (NOOP_RESOLVED with STALE_IDENTITY), so dedup at insert time would only
|
||||
// be a micro-optimization at the cost of an extra map.
|
||||
type Schedule struct {
|
||||
mu sync.Mutex
|
||||
h scheduleHeap
|
||||
}
|
||||
|
||||
func NewSchedule() *Schedule {
|
||||
return &Schedule{}
|
||||
}
|
||||
|
||||
// Add enqueues a Match.
|
||||
func (s *Schedule) Add(m Match) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
heap.Push(&s.h, m)
|
||||
}
|
||||
|
||||
// Len returns the number of pending Matches.
|
||||
func (s *Schedule) Len() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.h.Len()
|
||||
}
|
||||
|
||||
// NextDue returns the DueTime of the earliest pending Match. ok=false if
|
||||
// the schedule is empty.
|
||||
func (s *Schedule) NextDue() (time.Time, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.h.Len() == 0 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return s.h[0].DueTime, true
|
||||
}
|
||||
|
||||
// Drain pops and returns all Matches whose DueTime <= now, in DueTime order.
|
||||
// Subsequent calls return only newly-due Matches.
|
||||
func (s *Schedule) Drain(now time.Time) []Match {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []Match
|
||||
for s.h.Len() > 0 && !s.h[0].DueTime.After(now) {
|
||||
out = append(out, heap.Pop(&s.h).(Match))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// scheduleHeap implements heap.Interface ordered by Match.DueTime ascending.
|
||||
type scheduleHeap []Match
|
||||
|
||||
func (h scheduleHeap) Len() int { return len(h) }
|
||||
func (h scheduleHeap) Less(i, j int) bool { return h[i].DueTime.Before(h[j].DueTime) }
|
||||
func (h scheduleHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
|
||||
func (h *scheduleHeap) Push(x interface{}) { *h = append(*h, x.(Match)) }
|
||||
func (h *scheduleHeap) Pop() interface{} {
|
||||
old := *h
|
||||
n := len(old)
|
||||
x := old[n-1]
|
||||
*h = old[:n-1]
|
||||
return x
|
||||
}
|
||||
203
weed/s3api/s3lifecycle/router/schedule_test.go
Normal file
203
weed/s3api/s3lifecycle/router/schedule_test.go
Normal file
@@ -0,0 +1,203 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
)
|
||||
|
||||
func mkMatch(due time.Time, key string) Match {
|
||||
return Match{
|
||||
Key: s3lifecycle.ActionKey{Bucket: key},
|
||||
DueTime: due,
|
||||
ObjectKey: key,
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleEmpty(t *testing.T) {
|
||||
s := NewSchedule()
|
||||
if s.Len() != 0 {
|
||||
t.Fatal("Len != 0")
|
||||
}
|
||||
if _, ok := s.NextDue(); ok {
|
||||
t.Fatal("NextDue ok=true on empty")
|
||||
}
|
||||
if got := s.Drain(time.Now()); got != nil {
|
||||
t.Fatalf("Drain on empty returned %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleOrderedByDueTime(t *testing.T) {
|
||||
s := NewSchedule()
|
||||
t0 := time.Now()
|
||||
s.Add(mkMatch(t0.Add(3*time.Second), "c"))
|
||||
s.Add(mkMatch(t0.Add(1*time.Second), "a"))
|
||||
s.Add(mkMatch(t0.Add(2*time.Second), "b"))
|
||||
|
||||
if s.Len() != 3 {
|
||||
t.Fatalf("Len=%d, want 3", s.Len())
|
||||
}
|
||||
due, ok := s.NextDue()
|
||||
if !ok || !due.Equal(t0.Add(1*time.Second)) {
|
||||
t.Fatalf("NextDue=%v ok=%v", due, ok)
|
||||
}
|
||||
|
||||
got := s.Drain(t0.Add(2 * time.Second))
|
||||
if len(got) != 2 || got[0].ObjectKey != "a" || got[1].ObjectKey != "b" {
|
||||
t.Fatalf("Drain order: %+v", got)
|
||||
}
|
||||
if s.Len() != 1 {
|
||||
t.Fatalf("Len after drain=%d, want 1", s.Len())
|
||||
}
|
||||
|
||||
got = s.Drain(t0.Add(5 * time.Second))
|
||||
if len(got) != 1 || got[0].ObjectKey != "c" {
|
||||
t.Fatalf("Drain rest: %+v", got)
|
||||
}
|
||||
if s.Len() != 0 {
|
||||
t.Fatal("Len != 0 after final drain")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleDrainBoundaryInclusive(t *testing.T) {
|
||||
// DueTime exactly equal to now is drainable (<=).
|
||||
s := NewSchedule()
|
||||
t0 := time.Now()
|
||||
s.Add(mkMatch(t0, "a"))
|
||||
got := s.Drain(t0)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("expected boundary-inclusive drain, got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleAllowsDuplicates(t *testing.T) {
|
||||
s := NewSchedule()
|
||||
t0 := time.Now()
|
||||
s.Add(mkMatch(t0, "a"))
|
||||
s.Add(mkMatch(t0, "a"))
|
||||
if s.Len() != 2 {
|
||||
t.Fatalf("dup count=%d, want 2", s.Len())
|
||||
}
|
||||
got := s.Drain(t0)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("Drain dup count=%d, want 2", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleDrainBeforeAnyDueReturnsNothing(t *testing.T) {
|
||||
// Drain at a time before the earliest DueTime must return an empty
|
||||
// slice and leave the heap intact. Otherwise the dispatcher would
|
||||
// consume future-due matches early.
|
||||
s := NewSchedule()
|
||||
t0 := time.Now()
|
||||
s.Add(mkMatch(t0.Add(5*time.Second), "a"))
|
||||
s.Add(mkMatch(t0.Add(10*time.Second), "b"))
|
||||
|
||||
got := s.Drain(t0)
|
||||
if got != nil {
|
||||
t.Fatalf("Drain before any due should be nil, got %+v", got)
|
||||
}
|
||||
if s.Len() != 2 {
|
||||
t.Fatalf("Len after no-op Drain=%d, want 2", s.Len())
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleNextDueAfterPartialDrain(t *testing.T) {
|
||||
// After draining a prefix, NextDue must point at the earliest
|
||||
// remaining Match — regression catch for a Drain implementation
|
||||
// that forgets to maintain the heap invariant.
|
||||
s := NewSchedule()
|
||||
t0 := time.Now()
|
||||
s.Add(mkMatch(t0.Add(1*time.Second), "a"))
|
||||
s.Add(mkMatch(t0.Add(2*time.Second), "b"))
|
||||
s.Add(mkMatch(t0.Add(3*time.Second), "c"))
|
||||
|
||||
got := s.Drain(t0.Add(1500 * time.Millisecond))
|
||||
if len(got) != 1 || got[0].ObjectKey != "a" {
|
||||
t.Fatalf("Drain prefix=%+v, want [a]", got)
|
||||
}
|
||||
due, ok := s.NextDue()
|
||||
if !ok || !due.Equal(t0.Add(2*time.Second)) {
|
||||
t.Fatalf("NextDue after partial Drain=%v ok=%v, want t+2s", due, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleAddAfterDrainKeepsOrder(t *testing.T) {
|
||||
// Adding to a non-empty schedule mid-stream — after a real Drain
|
||||
// has popped at least one entry, a fresh Match with an earlier
|
||||
// DueTime than the existing minimum must become the next drainable.
|
||||
// Pins that heap.Pop + Push together preserve the heap invariant
|
||||
// across the Drain → Add boundary.
|
||||
s := NewSchedule()
|
||||
t0 := time.Now()
|
||||
s.Add(mkMatch(t0.Add(10*time.Second), "old"))
|
||||
s.Add(mkMatch(t0.Add(6*time.Second), "drain_me"))
|
||||
|
||||
drained := s.Drain(t0.Add(7 * time.Second))
|
||||
if len(drained) != 1 || drained[0].ObjectKey != "drain_me" {
|
||||
t.Fatalf("pre-drain=%+v, want [drain_me]", drained)
|
||||
}
|
||||
|
||||
s.Add(mkMatch(t0.Add(5*time.Second), "new")) // added after Drain, earlier than old
|
||||
|
||||
due, ok := s.NextDue()
|
||||
if !ok || !due.Equal(t0.Add(5*time.Second)) {
|
||||
t.Fatalf("NextDue=%v ok=%v, want t+5s", due, ok)
|
||||
}
|
||||
|
||||
got := s.Drain(t0.Add(6 * time.Second))
|
||||
if len(got) != 1 || got[0].ObjectKey != "new" {
|
||||
t.Fatalf("Drain=%+v, want [new]", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleDrainOrderIsAscendingDueTime(t *testing.T) {
|
||||
// Drain must return Matches in DueTime order regardless of insert
|
||||
// order — explicit pinning of the contract documented on Drain.
|
||||
s := NewSchedule()
|
||||
t0 := time.Now()
|
||||
for _, off := range []time.Duration{
|
||||
5 * time.Second,
|
||||
1 * time.Second,
|
||||
3 * time.Second,
|
||||
2 * time.Second,
|
||||
4 * time.Second,
|
||||
} {
|
||||
s.Add(mkMatch(t0.Add(off), "k"))
|
||||
}
|
||||
|
||||
got := s.Drain(t0.Add(10 * time.Second))
|
||||
if len(got) != 5 {
|
||||
t.Fatalf("Drain count=%d, want 5", len(got))
|
||||
}
|
||||
for i := 1; i < len(got); i++ {
|
||||
if got[i].DueTime.Before(got[i-1].DueTime) {
|
||||
t.Fatalf("Drain[%d]=%v is before Drain[%d]=%v", i, got[i].DueTime, i-1, got[i-1].DueTime)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduleConcurrentAddDrainNoRace(t *testing.T) {
|
||||
// The dispatcher's Add and Drain run on separate goroutines; the
|
||||
// schedule's mutex must serialize them without deadlock. -race
|
||||
// catches a regression that drops the lock on either path.
|
||||
s := NewSchedule()
|
||||
t0 := time.Now()
|
||||
const N = 64
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(N * 2)
|
||||
for i := 0; i < N; i++ {
|
||||
i := i
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
s.Add(mkMatch(t0.Add(time.Duration(i)*time.Millisecond), "k"))
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_ = s.Drain(t0.Add(10 * time.Second))
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
Reference in New Issue
Block a user