snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,33 @@
package dispatcher
import (
"context"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
// FilerStore is the small subset of filer-client operations the
// daily-replay cursor persister needs (dailyrun.FilerCursorPersister).
// Tests inject an in-memory fake.
type FilerStore interface {
Read(ctx context.Context, dir, name string) ([]byte, error)
Save(ctx context.Context, dir, name string, content []byte) error
}
// NewFilerStoreClient adapts a SeaweedFilerClient into a FilerStore.
func NewFilerStoreClient(client filer_pb.SeaweedFilerClient) FilerStore {
return &filerStoreClient{client: client}
}
type filerStoreClient struct {
client filer_pb.SeaweedFilerClient
}
func (s *filerStoreClient) Read(ctx context.Context, dir, name string) ([]byte, error) {
return filer.ReadInsideFiler(ctx, s.client, dir, name)
}
func (s *filerStoreClient) Save(ctx context.Context, dir, name string, content []byte) error {
return filer.SaveInsideFiler(ctx, s.client, dir, name, content)
}

View File

@@ -0,0 +1,171 @@
package dispatcher
import (
"context"
"errors"
"strings"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// filerSiblingLister inspects the .versions/<key>/ folder and the bare
// null-version. Listing caps at 2 — callers only distinguish 1 vs >=2.
type filerSiblingLister struct {
client filer_pb.SeaweedFilerClient
bucketsPath string
}
// NewFilerSiblingLister returns a router.SiblingLister backed by the
// filer client. Exported so the daily-replay worker (and any future
// non-dispatcher caller) can reuse the same .versions/ + null-bare
// lookup logic without re-implementing it. Phase 5 deletes the
// dispatcher's internal use; this constructor stays.
func NewFilerSiblingLister(client filer_pb.SeaweedFilerClient, bucketsPath string) router.SiblingLister {
return &filerSiblingLister{client: client, bucketsPath: bucketsPath}
}
func (l *filerSiblingLister) Survivors(ctx context.Context, bucket, objectKey string) (router.Survivors, error) {
bucketPath := strings.TrimSuffix(l.bucketsPath, "/") + "/" + bucket
versionsDir := bucketPath + "/" + objectKey + s3_constants.VersionsFolder
var s router.Survivors
err := filer_pb.SeaweedList(ctx, l.client, versionsDir, "", func(entry *filer_pb.Entry, _ bool) error {
s.Count++
if s.Count == 1 && entry != nil {
s.LoneEntry = entry
} else if s.Count > 1 {
s.LoneEntry = nil
}
return nil
}, "", false, 2)
if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
return router.Survivors{}, err
}
// NewFullPath strips a trailing slash from objectKey so directory-key
// objects (foo/) split the same as regular keys. LookupEntry
// normalizes the gRPC string-mapped not-found into ErrNotFound; a
// raw client.LookupDirectoryEntry would return that as a generic
// error and suppress every otherwise-valid match.
parent, name := util.NewFullPath(bucketPath, objectKey).DirAndName()
resp, err := filer_pb.LookupEntry(ctx, l.client, &filer_pb.LookupDirectoryEntryRequest{
Directory: parent,
Name: name,
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
return s, nil
}
return router.Survivors{}, err
}
// Bare regular file or an explicit S3 directory-marker (an empty
// directory entry with Mime set) both count as the null version.
if resp.Entry != nil && (!resp.Entry.IsDirectory || resp.Entry.IsDirectoryKeyObject()) {
s.HasNullVersion = true
}
return s, nil
}
// ListVersions paginates every file under .versions/<key>/. Used by
// the pointer-transition expansion path when a NewerNoncurrentVersions
// rule needs accurate per-version ranking. Subdirectories and entries
// without ExtVersionIdKey are filtered out. NotFound returns
// (nil, nil) so a hard-deleted .versions/ container collapses cleanly.
func (l *filerSiblingLister) ListVersions(ctx context.Context, bucket, objectKey string) ([]*filer_pb.Entry, error) {
bucketPath := strings.TrimSuffix(l.bucketsPath, "/") + "/" + bucket
dir := bucketPath + "/" + objectKey + s3_constants.VersionsFolder
const pageSize uint32 = 1024
startFrom := ""
var versions []*filer_pb.Entry
for {
var pageCount uint32
var lastName string
err := filer_pb.SeaweedList(ctx, l.client, dir, "", func(e *filer_pb.Entry, _ bool) error {
pageCount++
if e == nil {
return nil
}
lastName = e.Name
if e.Attributes == nil || e.IsDirectory {
return nil
}
if id, ok := e.Extended[s3_constants.ExtVersionIdKey]; !ok || len(id) == 0 {
return nil
}
versions = append(versions, e)
return nil
}, startFrom, false, pageSize)
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
return nil, nil
}
return nil, err
}
if pageCount < pageSize {
return versions, nil
}
startFrom = lastName
}
}
// LookupNullVersion returns the bare-key entry that represents the
// null version of objectKey, if any. Both regular files and explicit
// S3 directory-key markers (an empty directory entry with Mime set)
// qualify. explicit reports whether the entry's Extended map carries
// ExtVersionIdKey="null" — the marker the suspended-versioning write
// path applies. NotFound returns (nil, false, nil).
func (l *filerSiblingLister) LookupNullVersion(ctx context.Context, bucket, objectKey string) (*filer_pb.Entry, bool, error) {
bucketPath := strings.TrimSuffix(l.bucketsPath, "/") + "/" + bucket
parent, name := util.NewFullPath(bucketPath, objectKey).DirAndName()
resp, err := filer_pb.LookupEntry(ctx, l.client, &filer_pb.LookupDirectoryEntryRequest{
Directory: parent,
Name: name,
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
return nil, false, nil
}
return nil, false, err
}
if resp == nil || resp.Entry == nil {
return nil, false, nil
}
e := resp.Entry
if e.IsDirectory && !e.IsDirectoryKeyObject() {
return nil, false, nil
}
explicit := false
if id, ok := e.Extended[s3_constants.ExtVersionIdKey]; ok && string(id) == "null" {
explicit = true
}
return e, explicit, nil
}
// LookupVersion fetches <bucketsPath>/<bucket>/<objectKey>.versions/v_<id>
// for the pointer-transition router branch. NotFound returns (nil, nil)
// — the displaced version may have been hard-deleted between the
// pointer update and the lookup.
func (l *filerSiblingLister) LookupVersion(ctx context.Context, bucket, objectKey, versionID string) (*filer_pb.Entry, error) {
if versionID == "" {
return nil, nil
}
bucketPath := strings.TrimSuffix(l.bucketsPath, "/") + "/" + bucket
dir := bucketPath + "/" + objectKey + s3_constants.VersionsFolder
resp, err := filer_pb.LookupEntry(ctx, l.client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: "v_" + versionID,
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
return nil, nil
}
return nil, err
}
if resp == nil {
return nil, nil
}
return resp.Entry, nil
}

View File

@@ -0,0 +1,502 @@
package dispatcher
import (
"context"
"errors"
"fmt"
"io"
"sort"
"sync"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
)
// Tests for filerSiblingLister cover the four routing-critical surfaces
// the dispatcher pipeline depends on: Survivors (count cap + null
// detection), ListVersions (pagination + filter), LookupNullVersion
// (regular vs directory-key marker, explicit-null flag), and
// LookupVersion (v_ prefix + NotFound collapse).
type fsListStream struct {
responses []*filer_pb.ListEntriesResponse
index int
// recvErr is returned from the FIRST Recv() call before any
// responses are delivered; lets tests inject the gRPC NotFound the
// real filer surfaces when the parent directory doesn't exist.
recvErr error
ctx context.Context
}
func (s *fsListStream) Recv() (*filer_pb.ListEntriesResponse, error) {
if s.ctx != nil {
if err := s.ctx.Err(); err != nil {
return nil, err
}
}
if s.recvErr != nil {
err := s.recvErr
s.recvErr = nil
return nil, err
}
if s.index >= len(s.responses) {
return nil, io.EOF
}
resp := s.responses[s.index]
s.index++
return resp, nil
}
func (s *fsListStream) Header() (metadata.MD, error) { return metadata.MD{}, nil }
func (s *fsListStream) Trailer() metadata.MD { return metadata.MD{} }
func (s *fsListStream) CloseSend() error { return nil }
func (s *fsListStream) Context() context.Context {
if s.ctx != nil {
return s.ctx
}
return context.Background()
}
func (s *fsListStream) SendMsg(any) error { return nil }
func (s *fsListStream) RecvMsg(any) error { return nil }
// fsFakeFiler implements just the SeaweedFilerClient methods that
// filerSiblingLister calls (ListEntries + LookupDirectoryEntry). Other
// methods on the embedded interface remain nil; calling them panics,
// which the tests rely on to catch regressions that try to reach beyond
// the documented surface.
type fsFakeFiler struct {
filer_pb.SeaweedFilerClient
mu sync.Mutex
// tree maps a directory path to its immediate children. A directory
// is "missing" when its key is absent from tree (we return ErrNotFound
// from ListEntries / LookupDirectoryEntry to mimic the real filer).
tree map[string][]*filer_pb.Entry
// listErr / lookupErr force the next call to fail with this error.
// Reset to nil after use so tests can mix success + failure runs.
listErr error
lookupErr error
// missing controls per-directory NotFound for ListEntries: when a path
// is in this set, ListEntries returns filer_pb.ErrNotFound regardless
// of whether tree has an entry for it. Lets us test the .versions/
// not-yet-created vs. empty cases independently.
missing map[string]struct{}
}
func newFakeFiler() *fsFakeFiler {
return &fsFakeFiler{
tree: map[string][]*filer_pb.Entry{},
missing: map[string]struct{}{},
}
}
func (c *fsFakeFiler) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.listErr != nil {
err := c.listErr
c.listErr = nil
return nil, err
}
// Real-filer behavior on a missing parent: stream creation succeeds,
// the first Recv() surfaces ErrNotFound. SeaweedList passes that
// through unwrapped, which is what filerSiblingLister's
// errors.Is(err, filer_pb.ErrNotFound) check depends on.
if _, missing := c.missing[in.Directory]; missing {
return &fsListStream{recvErr: filer_pb.ErrNotFound, ctx: ctx}, nil
}
src := c.tree[in.Directory]
filtered := make([]*filer_pb.Entry, 0, len(src))
for _, e := range src {
if e == nil {
continue
}
if in.StartFromFileName != "" {
if in.InclusiveStartFrom {
if e.Name < in.StartFromFileName {
continue
}
} else if e.Name <= in.StartFromFileName {
continue
}
}
filtered = append(filtered, e)
}
sort.SliceStable(filtered, func(i, j int) bool { return filtered[i].Name < filtered[j].Name })
if in.Limit > 0 && uint32(len(filtered)) > in.Limit {
filtered = filtered[:in.Limit]
}
resps := make([]*filer_pb.ListEntriesResponse, 0, len(filtered))
for _, e := range filtered {
resps = append(resps, &filer_pb.ListEntriesResponse{Entry: e})
}
return &fsListStream{responses: resps, ctx: ctx}, nil
}
func (c *fsFakeFiler) LookupDirectoryEntry(ctx context.Context, in *filer_pb.LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.lookupErr != nil {
err := c.lookupErr
c.lookupErr = nil
return nil, err
}
for _, e := range c.tree[in.Directory] {
if e != nil && e.Name == in.Name {
return &filer_pb.LookupDirectoryEntryResponse{Entry: e}, nil
}
}
return nil, filer_pb.ErrNotFound
}
func (c *fsFakeFiler) put(dir string, entries ...*filer_pb.Entry) {
c.mu.Lock()
defer c.mu.Unlock()
c.tree[dir] = append(c.tree[dir], entries...)
}
func (c *fsFakeFiler) markMissing(dir string) {
c.mu.Lock()
defer c.mu.Unlock()
c.missing[dir] = struct{}{}
}
func newLister(c *fsFakeFiler) *filerSiblingLister {
return &filerSiblingLister{client: c, bucketsPath: "/buckets"}
}
func versionEntry(name, versionID string) *filer_pb.Entry {
return &filer_pb.Entry{
Name: name,
Attributes: &filer_pb.FuseAttributes{},
Extended: map[string][]byte{
s3_constants.ExtVersionIdKey: []byte(versionID),
},
}
}
func TestSurvivors_NoVersionsNoNullReturnsZero(t *testing.T) {
f := newFakeFiler()
// .versions/ doesn't exist (NotFound) and the bare key doesn't exist either.
f.markMissing("/buckets/b/k" + s3_constants.VersionsFolder)
l := newLister(f)
s, err := l.Survivors(context.Background(), "b", "k")
require.NoError(t, err)
assert.Equal(t, 0, s.Count)
assert.False(t, s.HasNullVersion)
assert.Nil(t, s.LoneEntry)
}
func TestSurvivors_OneVersionPopulatesLoneEntry(t *testing.T) {
// Count == 1 means LoneEntry must be set; the dispatcher's
// sole-survivor path uses it to compare against the marker version.
f := newFakeFiler()
dir := "/buckets/b/k" + s3_constants.VersionsFolder
f.put(dir, versionEntry("v_aaa", "aaa"))
f.markMissing("/buckets/b") // bare key absent
l := newLister(f)
s, err := l.Survivors(context.Background(), "b", "k")
require.NoError(t, err)
assert.Equal(t, 1, s.Count)
require.NotNil(t, s.LoneEntry)
assert.Equal(t, "v_aaa", s.LoneEntry.Name)
assert.False(t, s.HasNullVersion)
}
func TestSurvivors_MoreThanOneClearsLoneEntry(t *testing.T) {
// Listing caps at 2; once Count > 1, LoneEntry must be reset to nil
// so the caller doesn't mistakenly treat the second-arrival entry as
// the sole survivor.
f := newFakeFiler()
dir := "/buckets/b/k" + s3_constants.VersionsFolder
f.put(dir, versionEntry("v_aaa", "aaa"), versionEntry("v_bbb", "bbb"))
f.markMissing("/buckets/b")
l := newLister(f)
s, err := l.Survivors(context.Background(), "b", "k")
require.NoError(t, err)
assert.Equal(t, 2, s.Count)
assert.Nil(t, s.LoneEntry, "Count > 1 must clear LoneEntry")
}
func TestSurvivors_NullVersionDetectedFromBareRegularFile(t *testing.T) {
// The bare <bucket>/<key> path resolves to a regular file when the
// object pre-dates versioning; that file is the null version.
f := newFakeFiler()
f.markMissing("/buckets/b/k" + s3_constants.VersionsFolder)
f.put("/buckets/b", &filer_pb.Entry{Name: "k", IsDirectory: false, Attributes: &filer_pb.FuseAttributes{}})
l := newLister(f)
s, err := l.Survivors(context.Background(), "b", "k")
require.NoError(t, err)
assert.True(t, s.HasNullVersion)
}
func TestSurvivors_DirectoryKeyMarkerCountsAsNull(t *testing.T) {
// An explicit S3 directory-key marker (IsDirectory=true with
// Attributes.Mime set) is also the null version. The listing path
// downstream treats both the regular-file and the directory-marker
// shape as null; Survivors must agree. The lister's NewFullPath
// trims the trailing slash from objectKey before splitting, so the
// stored entry Name has no trailing slash.
f := newFakeFiler()
f.markMissing("/buckets/b/dir" + s3_constants.VersionsFolder)
f.put("/buckets/b", &filer_pb.Entry{
Name: "dir",
IsDirectory: true,
Attributes: &filer_pb.FuseAttributes{Mime: "application/x-directory"},
})
l := newLister(f)
s, err := l.Survivors(context.Background(), "b", "dir/")
require.NoError(t, err)
assert.True(t, s.HasNullVersion, "directory-key marker must count as null version")
}
func TestSurvivors_PlainDirectoryDoesNotCount(t *testing.T) {
// A plain directory (no Mime, just IsDirectory=true) is implicit-only
// and must NOT register as a null version — otherwise normal prefix
// directories would shadow real null-version detection.
f := newFakeFiler()
f.markMissing("/buckets/b/dir" + s3_constants.VersionsFolder)
f.put("/buckets/b", &filer_pb.Entry{
Name: "dir",
IsDirectory: true,
Attributes: &filer_pb.FuseAttributes{}, // no Mime
})
l := newLister(f)
s, err := l.Survivors(context.Background(), "b", "dir/")
require.NoError(t, err)
assert.False(t, s.HasNullVersion, "plain directory must not count as null version")
}
func TestSurvivors_VersionsListingErrorPropagates(t *testing.T) {
// A non-NotFound error from the .versions/ listing must surface; the
// dispatcher promotes this to a retry instead of a NOOP.
f := newFakeFiler()
f.listErr = errors.New("transport boom")
l := newLister(f)
_, err := l.Survivors(context.Background(), "b", "k")
assert.Error(t, err)
assert.Contains(t, err.Error(), "transport boom")
}
func TestSurvivors_NullLookupErrorPropagates(t *testing.T) {
// A non-NotFound error from the bare-key lookup must surface; we
// can't conclude "no null" from a transport failure.
f := newFakeFiler()
f.markMissing("/buckets/b/k" + s3_constants.VersionsFolder)
f.lookupErr = errors.New("lookup boom")
l := newLister(f)
_, err := l.Survivors(context.Background(), "b", "k")
assert.Error(t, err)
assert.Contains(t, err.Error(), "lookup boom")
}
func TestListVersions_NotFoundReturnsNil(t *testing.T) {
// A hard-deleted .versions/ container collapses to (nil, nil) so
// callers can treat it as "no versions" without special-casing.
f := newFakeFiler()
f.markMissing("/buckets/b/k" + s3_constants.VersionsFolder)
l := newLister(f)
versions, err := l.ListVersions(context.Background(), "b", "k")
require.NoError(t, err)
assert.Nil(t, versions)
}
func TestListVersions_FiltersOutDirectoriesAndMissingVersionId(t *testing.T) {
// Subdirectories and entries without ExtVersionIdKey aren't real
// versions; the bootstrap walker writes them sometimes so the
// listing must filter rather than treat them as data.
f := newFakeFiler()
dir := "/buckets/b/k" + s3_constants.VersionsFolder
f.put(dir,
versionEntry("v_aaa", "aaa"),
&filer_pb.Entry{Name: "subdir", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
&filer_pb.Entry{Name: "v_no_attrs"}, // nil Attributes
&filer_pb.Entry{Name: "v_no_ext", Attributes: &filer_pb.FuseAttributes{}}, // missing Extended
&filer_pb.Entry{Name: "v_empty_ext", Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{s3_constants.ExtVersionIdKey: nil}}, // empty value
versionEntry("v_zzz", "zzz"),
)
l := newLister(f)
versions, err := l.ListVersions(context.Background(), "b", "k")
require.NoError(t, err)
require.Len(t, versions, 2)
assert.Equal(t, "v_aaa", versions[0].Name)
assert.Equal(t, "v_zzz", versions[1].Name)
}
func TestListVersions_PaginatesAcrossPages(t *testing.T) {
// pageSize is 1024 inside the lister; populate just over the boundary
// to force a second listing call. The fake's StartFromFileName
// handling mirrors the real filer.
f := newFakeFiler()
dir := "/buckets/b/k" + s3_constants.VersionsFolder
const total = 1030
entries := make([]*filer_pb.Entry, 0, total)
for i := 0; i < total; i++ {
entries = append(entries, versionEntry(fmt.Sprintf("v_%05d", i), fmt.Sprintf("id_%05d", i)))
}
f.put(dir, entries...)
l := newLister(f)
versions, err := l.ListVersions(context.Background(), "b", "k")
require.NoError(t, err)
require.Len(t, versions, total)
// Order is preserved across pages.
assert.Equal(t, "v_00000", versions[0].Name)
assert.Equal(t, fmt.Sprintf("v_%05d", total-1), versions[total-1].Name)
}
func TestListVersions_ErrorPropagates(t *testing.T) {
f := newFakeFiler()
f.listErr = errors.New("list boom")
l := newLister(f)
versions, err := l.ListVersions(context.Background(), "b", "k")
assert.Nil(t, versions)
assert.Error(t, err)
assert.Contains(t, err.Error(), "list boom")
}
func TestLookupNullVersion_RegularFileExplicitFalse(t *testing.T) {
// A bare regular file with no ExtVersionIdKey is the null version
// but not "explicit" — the suspended-versioning write path flags
// explicit nulls with ExtVersionIdKey="null".
f := newFakeFiler()
f.put("/buckets/b", &filer_pb.Entry{Name: "k", IsDirectory: false, Attributes: &filer_pb.FuseAttributes{}})
l := newLister(f)
entry, explicit, err := l.LookupNullVersion(context.Background(), "b", "k")
require.NoError(t, err)
require.NotNil(t, entry)
assert.Equal(t, "k", entry.Name)
assert.False(t, explicit)
}
func TestLookupNullVersion_ExplicitNullFlag(t *testing.T) {
// ExtVersionIdKey="null" is the marker the suspended-versioning
// write path sets; the lister surfaces it via the explicit return.
f := newFakeFiler()
f.put("/buckets/b", &filer_pb.Entry{
Name: "k",
Attributes: &filer_pb.FuseAttributes{},
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("null")},
})
l := newLister(f)
entry, explicit, err := l.LookupNullVersion(context.Background(), "b", "k")
require.NoError(t, err)
require.NotNil(t, entry)
assert.True(t, explicit)
}
func TestLookupNullVersion_DirectoryKeyMarkerQualifies(t *testing.T) {
// NewFullPath strips a trailing slash before splitting; the entry
// Name as stored has no slash.
f := newFakeFiler()
f.put("/buckets/b", &filer_pb.Entry{
Name: "dir",
IsDirectory: true,
Attributes: &filer_pb.FuseAttributes{Mime: "application/x-directory"},
})
l := newLister(f)
entry, _, err := l.LookupNullVersion(context.Background(), "b", "dir/")
require.NoError(t, err)
require.NotNil(t, entry)
}
func TestLookupNullVersion_PlainDirectoryRejected(t *testing.T) {
// A plain directory must not be returned — otherwise the dispatcher
// would treat any prefix directory as a null version.
f := newFakeFiler()
f.put("/buckets/b", &filer_pb.Entry{
Name: "dir",
IsDirectory: true,
Attributes: &filer_pb.FuseAttributes{},
})
l := newLister(f)
entry, explicit, err := l.LookupNullVersion(context.Background(), "b", "dir/")
require.NoError(t, err)
assert.Nil(t, entry)
assert.False(t, explicit)
}
func TestLookupNullVersion_NotFoundCollapses(t *testing.T) {
f := newFakeFiler()
l := newLister(f) // empty tree -> NotFound on lookup
entry, explicit, err := l.LookupNullVersion(context.Background(), "b", "k")
require.NoError(t, err)
assert.Nil(t, entry)
assert.False(t, explicit)
}
func TestLookupNullVersion_TransportErrorPropagates(t *testing.T) {
f := newFakeFiler()
f.lookupErr = errors.New("lookup boom")
l := newLister(f)
_, _, err := l.LookupNullVersion(context.Background(), "b", "k")
assert.Error(t, err)
assert.Contains(t, err.Error(), "lookup boom")
}
func TestLookupVersion_EmptyVersionIdReturnsNil(t *testing.T) {
// An empty versionID is a no-op — neither error nor result.
f := newFakeFiler()
l := newLister(f)
entry, err := l.LookupVersion(context.Background(), "b", "k", "")
require.NoError(t, err)
assert.Nil(t, entry)
}
func TestLookupVersion_AppliesVPrefix(t *testing.T) {
// File name on disk is "v_<id>"; the caller passes only the id.
f := newFakeFiler()
dir := "/buckets/b/k" + s3_constants.VersionsFolder
f.put(dir, versionEntry("v_abc123", "abc123"))
l := newLister(f)
entry, err := l.LookupVersion(context.Background(), "b", "k", "abc123")
require.NoError(t, err)
require.NotNil(t, entry)
assert.Equal(t, "v_abc123", entry.Name)
}
func TestLookupVersion_NotFoundCollapses(t *testing.T) {
// Hard-delete between pointer update and lookup is the documented
// race; collapse to (nil, nil) so the dispatcher continues without
// retry-storming.
f := newFakeFiler()
l := newLister(f)
entry, err := l.LookupVersion(context.Background(), "b", "k", "abc")
require.NoError(t, err)
assert.Nil(t, entry)
}
func TestLookupVersion_TransportErrorPropagates(t *testing.T) {
f := newFakeFiler()
f.lookupErr = errors.New("lookup boom")
l := newLister(f)
_, err := l.LookupVersion(context.Background(), "b", "k", "abc")
assert.Error(t, err)
assert.Contains(t, err.Error(), "lookup boom")
}