snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
33
weed/s3api/s3lifecycle/dispatcher/filer_persister.go
Normal file
33
weed/s3api/s3lifecycle/dispatcher/filer_persister.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package dispatcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// FilerStore is the small subset of filer-client operations the
|
||||
// daily-replay cursor persister needs (dailyrun.FilerCursorPersister).
|
||||
// Tests inject an in-memory fake.
|
||||
type FilerStore interface {
|
||||
Read(ctx context.Context, dir, name string) ([]byte, error)
|
||||
Save(ctx context.Context, dir, name string, content []byte) error
|
||||
}
|
||||
|
||||
// NewFilerStoreClient adapts a SeaweedFilerClient into a FilerStore.
|
||||
func NewFilerStoreClient(client filer_pb.SeaweedFilerClient) FilerStore {
|
||||
return &filerStoreClient{client: client}
|
||||
}
|
||||
|
||||
type filerStoreClient struct {
|
||||
client filer_pb.SeaweedFilerClient
|
||||
}
|
||||
|
||||
func (s *filerStoreClient) Read(ctx context.Context, dir, name string) ([]byte, error) {
|
||||
return filer.ReadInsideFiler(ctx, s.client, dir, name)
|
||||
}
|
||||
|
||||
func (s *filerStoreClient) Save(ctx context.Context, dir, name string, content []byte) error {
|
||||
return filer.SaveInsideFiler(ctx, s.client, dir, name, content)
|
||||
}
|
||||
171
weed/s3api/s3lifecycle/dispatcher/sibling_lister.go
Normal file
171
weed/s3api/s3lifecycle/dispatcher/sibling_lister.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package dispatcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
// filerSiblingLister inspects the .versions/<key>/ folder and the bare
|
||||
// null-version. Listing caps at 2 — callers only distinguish 1 vs >=2.
|
||||
type filerSiblingLister struct {
|
||||
client filer_pb.SeaweedFilerClient
|
||||
bucketsPath string
|
||||
}
|
||||
|
||||
// NewFilerSiblingLister returns a router.SiblingLister backed by the
|
||||
// filer client. Exported so the daily-replay worker (and any future
|
||||
// non-dispatcher caller) can reuse the same .versions/ + null-bare
|
||||
// lookup logic without re-implementing it. Phase 5 deletes the
|
||||
// dispatcher's internal use; this constructor stays.
|
||||
func NewFilerSiblingLister(client filer_pb.SeaweedFilerClient, bucketsPath string) router.SiblingLister {
|
||||
return &filerSiblingLister{client: client, bucketsPath: bucketsPath}
|
||||
}
|
||||
|
||||
func (l *filerSiblingLister) Survivors(ctx context.Context, bucket, objectKey string) (router.Survivors, error) {
|
||||
bucketPath := strings.TrimSuffix(l.bucketsPath, "/") + "/" + bucket
|
||||
versionsDir := bucketPath + "/" + objectKey + s3_constants.VersionsFolder
|
||||
|
||||
var s router.Survivors
|
||||
err := filer_pb.SeaweedList(ctx, l.client, versionsDir, "", func(entry *filer_pb.Entry, _ bool) error {
|
||||
s.Count++
|
||||
if s.Count == 1 && entry != nil {
|
||||
s.LoneEntry = entry
|
||||
} else if s.Count > 1 {
|
||||
s.LoneEntry = nil
|
||||
}
|
||||
return nil
|
||||
}, "", false, 2)
|
||||
if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return router.Survivors{}, err
|
||||
}
|
||||
|
||||
// NewFullPath strips a trailing slash from objectKey so directory-key
|
||||
// objects (foo/) split the same as regular keys. LookupEntry
|
||||
// normalizes the gRPC string-mapped not-found into ErrNotFound; a
|
||||
// raw client.LookupDirectoryEntry would return that as a generic
|
||||
// error and suppress every otherwise-valid match.
|
||||
parent, name := util.NewFullPath(bucketPath, objectKey).DirAndName()
|
||||
resp, err := filer_pb.LookupEntry(ctx, l.client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: parent,
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return s, nil
|
||||
}
|
||||
return router.Survivors{}, err
|
||||
}
|
||||
// Bare regular file or an explicit S3 directory-marker (an empty
|
||||
// directory entry with Mime set) both count as the null version.
|
||||
if resp.Entry != nil && (!resp.Entry.IsDirectory || resp.Entry.IsDirectoryKeyObject()) {
|
||||
s.HasNullVersion = true
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// ListVersions paginates every file under .versions/<key>/. Used by
|
||||
// the pointer-transition expansion path when a NewerNoncurrentVersions
|
||||
// rule needs accurate per-version ranking. Subdirectories and entries
|
||||
// without ExtVersionIdKey are filtered out. NotFound returns
|
||||
// (nil, nil) so a hard-deleted .versions/ container collapses cleanly.
|
||||
func (l *filerSiblingLister) ListVersions(ctx context.Context, bucket, objectKey string) ([]*filer_pb.Entry, error) {
|
||||
bucketPath := strings.TrimSuffix(l.bucketsPath, "/") + "/" + bucket
|
||||
dir := bucketPath + "/" + objectKey + s3_constants.VersionsFolder
|
||||
const pageSize uint32 = 1024
|
||||
startFrom := ""
|
||||
var versions []*filer_pb.Entry
|
||||
for {
|
||||
var pageCount uint32
|
||||
var lastName string
|
||||
err := filer_pb.SeaweedList(ctx, l.client, dir, "", func(e *filer_pb.Entry, _ bool) error {
|
||||
pageCount++
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
lastName = e.Name
|
||||
if e.Attributes == nil || e.IsDirectory {
|
||||
return nil
|
||||
}
|
||||
if id, ok := e.Extended[s3_constants.ExtVersionIdKey]; !ok || len(id) == 0 {
|
||||
return nil
|
||||
}
|
||||
versions = append(versions, e)
|
||||
return nil
|
||||
}, startFrom, false, pageSize)
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if pageCount < pageSize {
|
||||
return versions, nil
|
||||
}
|
||||
startFrom = lastName
|
||||
}
|
||||
}
|
||||
|
||||
// LookupNullVersion returns the bare-key entry that represents the
|
||||
// null version of objectKey, if any. Both regular files and explicit
|
||||
// S3 directory-key markers (an empty directory entry with Mime set)
|
||||
// qualify. explicit reports whether the entry's Extended map carries
|
||||
// ExtVersionIdKey="null" — the marker the suspended-versioning write
|
||||
// path applies. NotFound returns (nil, false, nil).
|
||||
func (l *filerSiblingLister) LookupNullVersion(ctx context.Context, bucket, objectKey string) (*filer_pb.Entry, bool, error) {
|
||||
bucketPath := strings.TrimSuffix(l.bucketsPath, "/") + "/" + bucket
|
||||
parent, name := util.NewFullPath(bucketPath, objectKey).DirAndName()
|
||||
resp, err := filer_pb.LookupEntry(ctx, l.client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: parent,
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return nil, false, nil
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
if resp == nil || resp.Entry == nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
e := resp.Entry
|
||||
if e.IsDirectory && !e.IsDirectoryKeyObject() {
|
||||
return nil, false, nil
|
||||
}
|
||||
explicit := false
|
||||
if id, ok := e.Extended[s3_constants.ExtVersionIdKey]; ok && string(id) == "null" {
|
||||
explicit = true
|
||||
}
|
||||
return e, explicit, nil
|
||||
}
|
||||
|
||||
// LookupVersion fetches <bucketsPath>/<bucket>/<objectKey>.versions/v_<id>
|
||||
// for the pointer-transition router branch. NotFound returns (nil, nil)
|
||||
// — the displaced version may have been hard-deleted between the
|
||||
// pointer update and the lookup.
|
||||
func (l *filerSiblingLister) LookupVersion(ctx context.Context, bucket, objectKey, versionID string) (*filer_pb.Entry, error) {
|
||||
if versionID == "" {
|
||||
return nil, nil
|
||||
}
|
||||
bucketPath := strings.TrimSuffix(l.bucketsPath, "/") + "/" + bucket
|
||||
dir := bucketPath + "/" + objectKey + s3_constants.VersionsFolder
|
||||
resp, err := filer_pb.LookupEntry(ctx, l.client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: "v_" + versionID,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if resp == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return resp.Entry, nil
|
||||
}
|
||||
502
weed/s3api/s3lifecycle/dispatcher/sibling_lister_test.go
Normal file
502
weed/s3api/s3lifecycle/dispatcher/sibling_lister_test.go
Normal file
@@ -0,0 +1,502 @@
|
||||
package dispatcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/metadata"
|
||||
)
|
||||
|
||||
// Tests for filerSiblingLister cover the four routing-critical surfaces
|
||||
// the dispatcher pipeline depends on: Survivors (count cap + null
|
||||
// detection), ListVersions (pagination + filter), LookupNullVersion
|
||||
// (regular vs directory-key marker, explicit-null flag), and
|
||||
// LookupVersion (v_ prefix + NotFound collapse).
|
||||
|
||||
type fsListStream struct {
|
||||
responses []*filer_pb.ListEntriesResponse
|
||||
index int
|
||||
// recvErr is returned from the FIRST Recv() call before any
|
||||
// responses are delivered; lets tests inject the gRPC NotFound the
|
||||
// real filer surfaces when the parent directory doesn't exist.
|
||||
recvErr error
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
func (s *fsListStream) Recv() (*filer_pb.ListEntriesResponse, error) {
|
||||
if s.ctx != nil {
|
||||
if err := s.ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if s.recvErr != nil {
|
||||
err := s.recvErr
|
||||
s.recvErr = nil
|
||||
return nil, err
|
||||
}
|
||||
if s.index >= len(s.responses) {
|
||||
return nil, io.EOF
|
||||
}
|
||||
resp := s.responses[s.index]
|
||||
s.index++
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *fsListStream) Header() (metadata.MD, error) { return metadata.MD{}, nil }
|
||||
func (s *fsListStream) Trailer() metadata.MD { return metadata.MD{} }
|
||||
func (s *fsListStream) CloseSend() error { return nil }
|
||||
func (s *fsListStream) Context() context.Context {
|
||||
if s.ctx != nil {
|
||||
return s.ctx
|
||||
}
|
||||
return context.Background()
|
||||
}
|
||||
func (s *fsListStream) SendMsg(any) error { return nil }
|
||||
func (s *fsListStream) RecvMsg(any) error { return nil }
|
||||
|
||||
// fsFakeFiler implements just the SeaweedFilerClient methods that
|
||||
// filerSiblingLister calls (ListEntries + LookupDirectoryEntry). Other
|
||||
// methods on the embedded interface remain nil; calling them panics,
|
||||
// which the tests rely on to catch regressions that try to reach beyond
|
||||
// the documented surface.
|
||||
type fsFakeFiler struct {
|
||||
filer_pb.SeaweedFilerClient
|
||||
|
||||
mu sync.Mutex
|
||||
// tree maps a directory path to its immediate children. A directory
|
||||
// is "missing" when its key is absent from tree (we return ErrNotFound
|
||||
// from ListEntries / LookupDirectoryEntry to mimic the real filer).
|
||||
tree map[string][]*filer_pb.Entry
|
||||
// listErr / lookupErr force the next call to fail with this error.
|
||||
// Reset to nil after use so tests can mix success + failure runs.
|
||||
listErr error
|
||||
lookupErr error
|
||||
// missing controls per-directory NotFound for ListEntries: when a path
|
||||
// is in this set, ListEntries returns filer_pb.ErrNotFound regardless
|
||||
// of whether tree has an entry for it. Lets us test the .versions/
|
||||
// not-yet-created vs. empty cases independently.
|
||||
missing map[string]struct{}
|
||||
}
|
||||
|
||||
func newFakeFiler() *fsFakeFiler {
|
||||
return &fsFakeFiler{
|
||||
tree: map[string][]*filer_pb.Entry{},
|
||||
missing: map[string]struct{}{},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *fsFakeFiler) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.listErr != nil {
|
||||
err := c.listErr
|
||||
c.listErr = nil
|
||||
return nil, err
|
||||
}
|
||||
// Real-filer behavior on a missing parent: stream creation succeeds,
|
||||
// the first Recv() surfaces ErrNotFound. SeaweedList passes that
|
||||
// through unwrapped, which is what filerSiblingLister's
|
||||
// errors.Is(err, filer_pb.ErrNotFound) check depends on.
|
||||
if _, missing := c.missing[in.Directory]; missing {
|
||||
return &fsListStream{recvErr: filer_pb.ErrNotFound, ctx: ctx}, nil
|
||||
}
|
||||
src := c.tree[in.Directory]
|
||||
filtered := make([]*filer_pb.Entry, 0, len(src))
|
||||
for _, e := range src {
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if in.StartFromFileName != "" {
|
||||
if in.InclusiveStartFrom {
|
||||
if e.Name < in.StartFromFileName {
|
||||
continue
|
||||
}
|
||||
} else if e.Name <= in.StartFromFileName {
|
||||
continue
|
||||
}
|
||||
}
|
||||
filtered = append(filtered, e)
|
||||
}
|
||||
sort.SliceStable(filtered, func(i, j int) bool { return filtered[i].Name < filtered[j].Name })
|
||||
if in.Limit > 0 && uint32(len(filtered)) > in.Limit {
|
||||
filtered = filtered[:in.Limit]
|
||||
}
|
||||
resps := make([]*filer_pb.ListEntriesResponse, 0, len(filtered))
|
||||
for _, e := range filtered {
|
||||
resps = append(resps, &filer_pb.ListEntriesResponse{Entry: e})
|
||||
}
|
||||
return &fsListStream{responses: resps, ctx: ctx}, nil
|
||||
}
|
||||
|
||||
func (c *fsFakeFiler) LookupDirectoryEntry(ctx context.Context, in *filer_pb.LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.lookupErr != nil {
|
||||
err := c.lookupErr
|
||||
c.lookupErr = nil
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range c.tree[in.Directory] {
|
||||
if e != nil && e.Name == in.Name {
|
||||
return &filer_pb.LookupDirectoryEntryResponse{Entry: e}, nil
|
||||
}
|
||||
}
|
||||
return nil, filer_pb.ErrNotFound
|
||||
}
|
||||
|
||||
func (c *fsFakeFiler) put(dir string, entries ...*filer_pb.Entry) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.tree[dir] = append(c.tree[dir], entries...)
|
||||
}
|
||||
|
||||
func (c *fsFakeFiler) markMissing(dir string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.missing[dir] = struct{}{}
|
||||
}
|
||||
|
||||
func newLister(c *fsFakeFiler) *filerSiblingLister {
|
||||
return &filerSiblingLister{client: c, bucketsPath: "/buckets"}
|
||||
}
|
||||
|
||||
func versionEntry(name, versionID string) *filer_pb.Entry {
|
||||
return &filer_pb.Entry{
|
||||
Name: name,
|
||||
Attributes: &filer_pb.FuseAttributes{},
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtVersionIdKey: []byte(versionID),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestSurvivors_NoVersionsNoNullReturnsZero(t *testing.T) {
|
||||
f := newFakeFiler()
|
||||
// .versions/ doesn't exist (NotFound) and the bare key doesn't exist either.
|
||||
f.markMissing("/buckets/b/k" + s3_constants.VersionsFolder)
|
||||
l := newLister(f)
|
||||
|
||||
s, err := l.Survivors(context.Background(), "b", "k")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 0, s.Count)
|
||||
assert.False(t, s.HasNullVersion)
|
||||
assert.Nil(t, s.LoneEntry)
|
||||
}
|
||||
|
||||
func TestSurvivors_OneVersionPopulatesLoneEntry(t *testing.T) {
|
||||
// Count == 1 means LoneEntry must be set; the dispatcher's
|
||||
// sole-survivor path uses it to compare against the marker version.
|
||||
f := newFakeFiler()
|
||||
dir := "/buckets/b/k" + s3_constants.VersionsFolder
|
||||
f.put(dir, versionEntry("v_aaa", "aaa"))
|
||||
f.markMissing("/buckets/b") // bare key absent
|
||||
l := newLister(f)
|
||||
|
||||
s, err := l.Survivors(context.Background(), "b", "k")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 1, s.Count)
|
||||
require.NotNil(t, s.LoneEntry)
|
||||
assert.Equal(t, "v_aaa", s.LoneEntry.Name)
|
||||
assert.False(t, s.HasNullVersion)
|
||||
}
|
||||
|
||||
func TestSurvivors_MoreThanOneClearsLoneEntry(t *testing.T) {
|
||||
// Listing caps at 2; once Count > 1, LoneEntry must be reset to nil
|
||||
// so the caller doesn't mistakenly treat the second-arrival entry as
|
||||
// the sole survivor.
|
||||
f := newFakeFiler()
|
||||
dir := "/buckets/b/k" + s3_constants.VersionsFolder
|
||||
f.put(dir, versionEntry("v_aaa", "aaa"), versionEntry("v_bbb", "bbb"))
|
||||
f.markMissing("/buckets/b")
|
||||
l := newLister(f)
|
||||
|
||||
s, err := l.Survivors(context.Background(), "b", "k")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 2, s.Count)
|
||||
assert.Nil(t, s.LoneEntry, "Count > 1 must clear LoneEntry")
|
||||
}
|
||||
|
||||
func TestSurvivors_NullVersionDetectedFromBareRegularFile(t *testing.T) {
|
||||
// The bare <bucket>/<key> path resolves to a regular file when the
|
||||
// object pre-dates versioning; that file is the null version.
|
||||
f := newFakeFiler()
|
||||
f.markMissing("/buckets/b/k" + s3_constants.VersionsFolder)
|
||||
f.put("/buckets/b", &filer_pb.Entry{Name: "k", IsDirectory: false, Attributes: &filer_pb.FuseAttributes{}})
|
||||
l := newLister(f)
|
||||
|
||||
s, err := l.Survivors(context.Background(), "b", "k")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, s.HasNullVersion)
|
||||
}
|
||||
|
||||
func TestSurvivors_DirectoryKeyMarkerCountsAsNull(t *testing.T) {
|
||||
// An explicit S3 directory-key marker (IsDirectory=true with
|
||||
// Attributes.Mime set) is also the null version. The listing path
|
||||
// downstream treats both the regular-file and the directory-marker
|
||||
// shape as null; Survivors must agree. The lister's NewFullPath
|
||||
// trims the trailing slash from objectKey before splitting, so the
|
||||
// stored entry Name has no trailing slash.
|
||||
f := newFakeFiler()
|
||||
f.markMissing("/buckets/b/dir" + s3_constants.VersionsFolder)
|
||||
f.put("/buckets/b", &filer_pb.Entry{
|
||||
Name: "dir",
|
||||
IsDirectory: true,
|
||||
Attributes: &filer_pb.FuseAttributes{Mime: "application/x-directory"},
|
||||
})
|
||||
l := newLister(f)
|
||||
|
||||
s, err := l.Survivors(context.Background(), "b", "dir/")
|
||||
require.NoError(t, err)
|
||||
assert.True(t, s.HasNullVersion, "directory-key marker must count as null version")
|
||||
}
|
||||
|
||||
func TestSurvivors_PlainDirectoryDoesNotCount(t *testing.T) {
|
||||
// A plain directory (no Mime, just IsDirectory=true) is implicit-only
|
||||
// and must NOT register as a null version — otherwise normal prefix
|
||||
// directories would shadow real null-version detection.
|
||||
f := newFakeFiler()
|
||||
f.markMissing("/buckets/b/dir" + s3_constants.VersionsFolder)
|
||||
f.put("/buckets/b", &filer_pb.Entry{
|
||||
Name: "dir",
|
||||
IsDirectory: true,
|
||||
Attributes: &filer_pb.FuseAttributes{}, // no Mime
|
||||
})
|
||||
l := newLister(f)
|
||||
|
||||
s, err := l.Survivors(context.Background(), "b", "dir/")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, s.HasNullVersion, "plain directory must not count as null version")
|
||||
}
|
||||
|
||||
func TestSurvivors_VersionsListingErrorPropagates(t *testing.T) {
|
||||
// A non-NotFound error from the .versions/ listing must surface; the
|
||||
// dispatcher promotes this to a retry instead of a NOOP.
|
||||
f := newFakeFiler()
|
||||
f.listErr = errors.New("transport boom")
|
||||
l := newLister(f)
|
||||
|
||||
_, err := l.Survivors(context.Background(), "b", "k")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "transport boom")
|
||||
}
|
||||
|
||||
func TestSurvivors_NullLookupErrorPropagates(t *testing.T) {
|
||||
// A non-NotFound error from the bare-key lookup must surface; we
|
||||
// can't conclude "no null" from a transport failure.
|
||||
f := newFakeFiler()
|
||||
f.markMissing("/buckets/b/k" + s3_constants.VersionsFolder)
|
||||
f.lookupErr = errors.New("lookup boom")
|
||||
l := newLister(f)
|
||||
|
||||
_, err := l.Survivors(context.Background(), "b", "k")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "lookup boom")
|
||||
}
|
||||
|
||||
func TestListVersions_NotFoundReturnsNil(t *testing.T) {
|
||||
// A hard-deleted .versions/ container collapses to (nil, nil) so
|
||||
// callers can treat it as "no versions" without special-casing.
|
||||
f := newFakeFiler()
|
||||
f.markMissing("/buckets/b/k" + s3_constants.VersionsFolder)
|
||||
l := newLister(f)
|
||||
|
||||
versions, err := l.ListVersions(context.Background(), "b", "k")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, versions)
|
||||
}
|
||||
|
||||
func TestListVersions_FiltersOutDirectoriesAndMissingVersionId(t *testing.T) {
|
||||
// Subdirectories and entries without ExtVersionIdKey aren't real
|
||||
// versions; the bootstrap walker writes them sometimes so the
|
||||
// listing must filter rather than treat them as data.
|
||||
f := newFakeFiler()
|
||||
dir := "/buckets/b/k" + s3_constants.VersionsFolder
|
||||
f.put(dir,
|
||||
versionEntry("v_aaa", "aaa"),
|
||||
&filer_pb.Entry{Name: "subdir", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
|
||||
&filer_pb.Entry{Name: "v_no_attrs"}, // nil Attributes
|
||||
&filer_pb.Entry{Name: "v_no_ext", Attributes: &filer_pb.FuseAttributes{}}, // missing Extended
|
||||
&filer_pb.Entry{Name: "v_empty_ext", Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{s3_constants.ExtVersionIdKey: nil}}, // empty value
|
||||
versionEntry("v_zzz", "zzz"),
|
||||
)
|
||||
l := newLister(f)
|
||||
|
||||
versions, err := l.ListVersions(context.Background(), "b", "k")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, versions, 2)
|
||||
assert.Equal(t, "v_aaa", versions[0].Name)
|
||||
assert.Equal(t, "v_zzz", versions[1].Name)
|
||||
}
|
||||
|
||||
func TestListVersions_PaginatesAcrossPages(t *testing.T) {
|
||||
// pageSize is 1024 inside the lister; populate just over the boundary
|
||||
// to force a second listing call. The fake's StartFromFileName
|
||||
// handling mirrors the real filer.
|
||||
f := newFakeFiler()
|
||||
dir := "/buckets/b/k" + s3_constants.VersionsFolder
|
||||
const total = 1030
|
||||
entries := make([]*filer_pb.Entry, 0, total)
|
||||
for i := 0; i < total; i++ {
|
||||
entries = append(entries, versionEntry(fmt.Sprintf("v_%05d", i), fmt.Sprintf("id_%05d", i)))
|
||||
}
|
||||
f.put(dir, entries...)
|
||||
l := newLister(f)
|
||||
|
||||
versions, err := l.ListVersions(context.Background(), "b", "k")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, versions, total)
|
||||
// Order is preserved across pages.
|
||||
assert.Equal(t, "v_00000", versions[0].Name)
|
||||
assert.Equal(t, fmt.Sprintf("v_%05d", total-1), versions[total-1].Name)
|
||||
}
|
||||
|
||||
func TestListVersions_ErrorPropagates(t *testing.T) {
|
||||
f := newFakeFiler()
|
||||
f.listErr = errors.New("list boom")
|
||||
l := newLister(f)
|
||||
|
||||
versions, err := l.ListVersions(context.Background(), "b", "k")
|
||||
assert.Nil(t, versions)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "list boom")
|
||||
}
|
||||
|
||||
func TestLookupNullVersion_RegularFileExplicitFalse(t *testing.T) {
|
||||
// A bare regular file with no ExtVersionIdKey is the null version
|
||||
// but not "explicit" — the suspended-versioning write path flags
|
||||
// explicit nulls with ExtVersionIdKey="null".
|
||||
f := newFakeFiler()
|
||||
f.put("/buckets/b", &filer_pb.Entry{Name: "k", IsDirectory: false, Attributes: &filer_pb.FuseAttributes{}})
|
||||
l := newLister(f)
|
||||
|
||||
entry, explicit, err := l.LookupNullVersion(context.Background(), "b", "k")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, entry)
|
||||
assert.Equal(t, "k", entry.Name)
|
||||
assert.False(t, explicit)
|
||||
}
|
||||
|
||||
func TestLookupNullVersion_ExplicitNullFlag(t *testing.T) {
|
||||
// ExtVersionIdKey="null" is the marker the suspended-versioning
|
||||
// write path sets; the lister surfaces it via the explicit return.
|
||||
f := newFakeFiler()
|
||||
f.put("/buckets/b", &filer_pb.Entry{
|
||||
Name: "k",
|
||||
Attributes: &filer_pb.FuseAttributes{},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("null")},
|
||||
})
|
||||
l := newLister(f)
|
||||
|
||||
entry, explicit, err := l.LookupNullVersion(context.Background(), "b", "k")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, entry)
|
||||
assert.True(t, explicit)
|
||||
}
|
||||
|
||||
func TestLookupNullVersion_DirectoryKeyMarkerQualifies(t *testing.T) {
|
||||
// NewFullPath strips a trailing slash before splitting; the entry
|
||||
// Name as stored has no slash.
|
||||
f := newFakeFiler()
|
||||
f.put("/buckets/b", &filer_pb.Entry{
|
||||
Name: "dir",
|
||||
IsDirectory: true,
|
||||
Attributes: &filer_pb.FuseAttributes{Mime: "application/x-directory"},
|
||||
})
|
||||
l := newLister(f)
|
||||
|
||||
entry, _, err := l.LookupNullVersion(context.Background(), "b", "dir/")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, entry)
|
||||
}
|
||||
|
||||
func TestLookupNullVersion_PlainDirectoryRejected(t *testing.T) {
|
||||
// A plain directory must not be returned — otherwise the dispatcher
|
||||
// would treat any prefix directory as a null version.
|
||||
f := newFakeFiler()
|
||||
f.put("/buckets/b", &filer_pb.Entry{
|
||||
Name: "dir",
|
||||
IsDirectory: true,
|
||||
Attributes: &filer_pb.FuseAttributes{},
|
||||
})
|
||||
l := newLister(f)
|
||||
|
||||
entry, explicit, err := l.LookupNullVersion(context.Background(), "b", "dir/")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, entry)
|
||||
assert.False(t, explicit)
|
||||
}
|
||||
|
||||
func TestLookupNullVersion_NotFoundCollapses(t *testing.T) {
|
||||
f := newFakeFiler()
|
||||
l := newLister(f) // empty tree -> NotFound on lookup
|
||||
|
||||
entry, explicit, err := l.LookupNullVersion(context.Background(), "b", "k")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, entry)
|
||||
assert.False(t, explicit)
|
||||
}
|
||||
|
||||
func TestLookupNullVersion_TransportErrorPropagates(t *testing.T) {
|
||||
f := newFakeFiler()
|
||||
f.lookupErr = errors.New("lookup boom")
|
||||
l := newLister(f)
|
||||
|
||||
_, _, err := l.LookupNullVersion(context.Background(), "b", "k")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "lookup boom")
|
||||
}
|
||||
|
||||
func TestLookupVersion_EmptyVersionIdReturnsNil(t *testing.T) {
|
||||
// An empty versionID is a no-op — neither error nor result.
|
||||
f := newFakeFiler()
|
||||
l := newLister(f)
|
||||
|
||||
entry, err := l.LookupVersion(context.Background(), "b", "k", "")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, entry)
|
||||
}
|
||||
|
||||
func TestLookupVersion_AppliesVPrefix(t *testing.T) {
|
||||
// File name on disk is "v_<id>"; the caller passes only the id.
|
||||
f := newFakeFiler()
|
||||
dir := "/buckets/b/k" + s3_constants.VersionsFolder
|
||||
f.put(dir, versionEntry("v_abc123", "abc123"))
|
||||
l := newLister(f)
|
||||
|
||||
entry, err := l.LookupVersion(context.Background(), "b", "k", "abc123")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, entry)
|
||||
assert.Equal(t, "v_abc123", entry.Name)
|
||||
}
|
||||
|
||||
func TestLookupVersion_NotFoundCollapses(t *testing.T) {
|
||||
// Hard-delete between pointer update and lookup is the documented
|
||||
// race; collapse to (nil, nil) so the dispatcher continues without
|
||||
// retry-storming.
|
||||
f := newFakeFiler()
|
||||
l := newLister(f)
|
||||
|
||||
entry, err := l.LookupVersion(context.Background(), "b", "k", "abc")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, entry)
|
||||
}
|
||||
|
||||
func TestLookupVersion_TransportErrorPropagates(t *testing.T) {
|
||||
f := newFakeFiler()
|
||||
f.lookupErr = errors.New("lookup boom")
|
||||
l := newLister(f)
|
||||
|
||||
_, err := l.LookupVersion(context.Background(), "b", "k", "abc")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "lookup boom")
|
||||
}
|
||||
Reference in New Issue
Block a user