snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,311 @@
package s3tables
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
var (
ErrAttributeNotFound = errors.New("attribute not found")
ErrConcurrentUpdate = errors.New("attribute changed concurrently")
)
// extendedAttributeUpdateAttempts bounds the read-modify-write retry loop in
// updateExtendedAttribute.
const extendedAttributeUpdateAttempts = 5
// Filer operations - Common functions for interacting with the filer
// createDirectory creates a new directory at the specified path
func (h *S3TablesHandler) createDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
dir, name := splitPath(path)
now := time.Now().Unix()
return filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
Directory: dir,
Entry: &filer_pb.Entry{
Name: name,
IsDirectory: true,
Attributes: &filer_pb.FuseAttributes{
Mtime: now,
Crtime: now,
FileMode: uint32(0755 | os.ModeDir),
},
},
})
}
// ensureDirectory ensures a directory exists at the specified path
func (h *S3TablesHandler) ensureDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
dir, name := splitPath(path)
_, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if err == nil {
return nil
}
if errors.Is(err, filer_pb.ErrNotFound) {
return h.createDirectory(ctx, client, path)
}
return err
}
// deleteEntryIfExists removes an entry if it exists, ignoring missing errors
func (h *S3TablesHandler) deleteEntryIfExists(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
dir, name := splitPath(path)
return filer_pb.DoRemove(ctx, client, dir, name, true, false, true, false, nil)
}
// mutateEntryExtended applies mutate to an entry's extended attributes and
// writes the entry back, asserting that nothing on it changed in between.
// UpdateEntry rewrites the whole entry from the snapshot read here, so an
// unconditional write would delete an attribute a concurrent writer created.
func (h *S3TablesHandler) mutateEntryExtended(
ctx context.Context,
client filer_pb.SeaweedFilerClient,
path string,
mutate func(extended map[string][]byte) error,
) error {
dir, name := splitPath(path)
for attempt := 0; attempt < extendedAttributeUpdateAttempts; attempt++ {
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if err != nil {
return err
}
entry := resp.Entry
expected := snapshotExtended(entry.Extended)
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
if err := mutate(entry.Extended); err != nil {
return err
}
_, err = client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{
Directory: dir,
Entry: entry,
ExpectedExtended: expected,
})
if err == nil {
return nil
}
if status.Code(err) != codes.FailedPrecondition {
return err
}
}
return fmt.Errorf("%w: %s", ErrConcurrentUpdate, path)
}
// setExtendedAttribute sets an extended attribute on an existing entry
func (h *S3TablesHandler) setExtendedAttribute(ctx context.Context, client filer_pb.SeaweedFilerClient, path, key string, data []byte) error {
return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
extended[key] = data
return nil
})
}
// setExtendedAttributes sets multiple extended attributes on an existing entry
// in a single UpdateEntry, so callers don't leave the entry partially tagged.
func (h *S3TablesHandler) setExtendedAttributes(ctx context.Context, client filer_pb.SeaweedFilerClient, path string, attrs map[string][]byte) error {
return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
for key, data := range attrs {
extended[key] = data
}
return nil
})
}
// removeExtendedAttributes deletes the given extended attributes from an entry,
// leaving the directory and its children intact.
func (h *S3TablesHandler) removeExtendedAttributes(ctx context.Context, client filer_pb.SeaweedFilerClient, path string, keys ...string) error {
return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
for _, key := range keys {
delete(extended, key)
}
return nil
})
}
// removeExtendedAttributesIf drops the given attributes only while they still
// hold the values the caller captured. A rename copies attributes to the
// destination and then clears the source; without this, a write that landed in
// between is deleted here and never reaches the new name.
func (h *S3TablesHandler) removeExtendedAttributesIf(
ctx context.Context,
client filer_pb.SeaweedFilerClient,
path string,
expected map[string][]byte,
keys ...string,
) error {
return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
for key, want := range expected {
if !bytes.Equal(extended[key], want) {
return fmt.Errorf("%w: %s changed on %s", ErrConcurrentUpdate, key, path)
}
}
for _, key := range keys {
delete(extended, key)
}
return nil
})
}
// updateExtendedAttribute applies mutate to the current value of key and writes
// the result back. mutate runs again on each retry, so it always sees the
// current value.
func (h *S3TablesHandler) updateExtendedAttribute(
ctx context.Context,
client filer_pb.SeaweedFilerClient,
path, key string,
mutate func(current []byte) ([]byte, error),
) error {
return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
updated, err := mutate(extended[key])
if err != nil {
return err
}
extended[key] = updated
return nil
})
}
// s3tablesExtendedKeys is every attribute this package stores on a bucket or
// table entry. A whole-entry write has to assert the absent ones too: a key
// missing from the precondition is a key a concurrent writer can create and
// this write will then delete.
var s3tablesExtendedKeys = []string{
ExtendedKeyTableBucket,
ExtendedKeyMetadata,
ExtendedKeyMetadataVersion,
ExtendedKeyPolicy,
ExtendedKeyTags,
ExtendedKeyMaintenance,
ExtendedKeyMaintenanceStatus,
ExtendedKeyEntryType,
}
// SnapshotExtended captures an entry's attributes as an UpdateEntry
// precondition, so a whole-entry write cannot silently revert or delete an
// attribute a concurrent writer touched. Attributes absent at read time are
// asserted absent, which fails the precondition if someone creates one first.
func SnapshotExtended(extended map[string][]byte, keys ...string) map[string][]byte {
return snapshotExtended(extended, keys...)
}
func snapshotExtended(extended map[string][]byte, keys ...string) map[string][]byte {
expected := make(map[string][]byte, len(extended)+len(s3tablesExtendedKeys))
for k, v := range extended {
expected[k] = v
}
for _, k := range s3tablesExtendedKeys {
if _, ok := expected[k]; !ok {
expected[k] = nil
}
}
for _, k := range keys {
if _, ok := expected[k]; !ok {
expected[k] = nil
}
}
return expected
}
// getExtendedAttribute gets an extended attribute from an entry
func (h *S3TablesHandler) getExtendedAttribute(ctx context.Context, client filer_pb.SeaweedFilerClient, path, key string) ([]byte, error) {
dir, name := splitPath(path)
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if err != nil {
return nil, err
}
if resp.Entry.Extended == nil {
return nil, fmt.Errorf("%w: %s", ErrAttributeNotFound, key)
}
data, ok := resp.Entry.Extended[key]
if !ok {
return nil, fmt.Errorf("%w: %s", ErrAttributeNotFound, key)
}
return data, nil
}
// loadNamespaceMetadata resolves a namespace to its metadata. A directory that
// carries no namespace metadata is not a namespace, so a missing attribute
// reports the same absence as a missing entry and every caller tests one
// condition instead of forgetting the second.
func (h *S3TablesHandler) loadNamespaceMetadata(ctx context.Context, filerClient FilerClient, bucketName, namespaceName string) (*namespaceMetadata, error) {
var metadata namespaceMetadata
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(ctx, client, GetNamespacePath(bucketName, namespaceName), ExtendedKeyMetadata)
if err != nil {
if errors.Is(err, ErrAttributeNotFound) {
return filer_pb.ErrNotFound
}
return err
}
if err := json.Unmarshal(data, &metadata); err != nil {
return fmt.Errorf("failed to unmarshal namespace metadata: %w", err)
}
return nil
})
if err != nil {
return nil, err
}
return &metadata, nil
}
// lookupEntry returns the filer entry at the given path.
func (h *S3TablesHandler) lookupEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) (*filer_pb.Entry, error) {
dir, name := splitPath(path)
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if err != nil {
return nil, err
}
return resp.Entry, nil
}
// deleteExtendedAttribute deletes an extended attribute from an entry
func (h *S3TablesHandler) deleteExtendedAttribute(ctx context.Context, client filer_pb.SeaweedFilerClient, path, key string) error {
return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
delete(extended, key)
return nil
})
}
// deleteDirectory deletes a directory and all its contents
func (h *S3TablesHandler) deleteDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
dir, name := splitPath(path)
return filer_pb.DoRemove(ctx, client, dir, name, true, true, true, false, nil)
}
// entryExists checks if an entry exists at the given path
func (h *S3TablesHandler) entryExists(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) bool {
dir, name := splitPath(path)
_, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
return err == nil
}

View File

@@ -0,0 +1,102 @@
package s3tables
import (
"context"
"encoding/json"
"errors"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// deleteEntryStub answers DeleteEntry the way the filer does: a rejected delete
// comes back as a nil transport error with the reason in the response.
type deleteEntryStub struct {
filer_pb.SeaweedFilerClient
resp *filer_pb.DeleteEntryResponse
err error
req *filer_pb.DeleteEntryRequest
}
func (s *deleteEntryStub) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest, _ ...grpc.CallOption) (*filer_pb.DeleteEntryResponse, error) {
s.req = req
return s.resp, s.err
}
func TestDeleteDirectoryReportsRejectedDelete(t *testing.T) {
stub := &deleteEntryStub{resp: &filer_pb.DeleteEntryResponse{Error: "fail to delete non-empty folder"}}
err := (&S3TablesHandler{}).deleteDirectory(context.Background(), stub, "/buckets/b/ns/t")
require.Error(t, err, "a delete the filer rejected must not be reported as success")
assert.Contains(t, err.Error(), "fail to delete non-empty folder")
}
func TestDeleteDirectoryToleratesMissingEntry(t *testing.T) {
t.Run("reported in the response", func(t *testing.T) {
stub := &deleteEntryStub{resp: &filer_pb.DeleteEntryResponse{Error: filer_pb.ErrNotFound.Error()}}
assert.NoError(t, (&S3TablesHandler{}).deleteDirectory(context.Background(), stub, "/buckets/b/ns/t"))
})
t.Run("reported as a transport error", func(t *testing.T) {
stub := &deleteEntryStub{err: status.Error(codes.NotFound, filer_pb.ErrNotFound.Error())}
assert.NoError(t, (&S3TablesHandler{}).deleteDirectory(context.Background(), stub, "/buckets/b/ns/t"))
})
}
func TestDeleteDirectoryPropagatesTransportError(t *testing.T) {
stub := &deleteEntryStub{err: errors.New("filer unreachable")}
err := (&S3TablesHandler{}).deleteDirectory(context.Background(), stub, "/buckets/b/ns/t")
require.Error(t, err)
assert.Contains(t, err.Error(), "filer unreachable")
}
// deleteDirectory drops a whole subtree, so the delete has to stay recursive and
// take the data with it.
func TestDeleteDirectoryDeletesTheSubtree(t *testing.T) {
stub := &deleteEntryStub{resp: &filer_pb.DeleteEntryResponse{}}
require.NoError(t, (&S3TablesHandler{}).deleteDirectory(context.Background(), stub, "/buckets/b/ns/t"))
require.NotNil(t, stub.req)
assert.Equal(t, "/buckets/b/ns", stub.req.Directory)
assert.Equal(t, "t", stub.req.Name)
assert.True(t, stub.req.IsDeleteData)
assert.True(t, stub.req.IsRecursive)
assert.True(t, stub.req.IgnoreRecursiveError)
}
// The bucket is the directory, so DeleteTableBucket must not answer success
// when the filer refused to remove it.
func TestDeleteTableBucketReportsRejectedDirectoryDelete(t *testing.T) {
fs := s3tablestest.Start(t)
bucketMeta, err := json.Marshal(tableBucketMetadata{Name: renameTestBucket, OwnerAccountID: DefaultAccountID})
require.NoError(t, err)
fs.Put(TablesPath, renameTestBucket, map[string][]byte{
ExtendedKeyTableBucket: []byte("{}"),
ExtendedKeyMetadata: bucketMeta,
})
bucketPath := GetTableBucketPath(renameTestBucket)
fs.RejectDelete = func(dir, name string) string {
if dir+"/"+name == bucketPath {
return "fail to delete non-empty folder"
}
return ""
}
m := NewManager()
m.SetTrusted(true)
err = m.Execute(context.Background(), NewManagerClient(fs.Client), "DeleteTableBucket",
&DeleteTableBucketRequest{TableBucketARN: mustBucketARN(t)}, nil, "")
require.Error(t, err, "a bucket the filer refused to delete must not be reported as deleted")
assert.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is still there")
}

View File

@@ -0,0 +1,444 @@
package s3tables
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"reflect"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
const (
TablesPath = s3_constants.DefaultBucketsPath
DefaultAccountID = "000000000000"
DefaultRegion = "us-east-1"
// Extended entry attributes for metadata storage
ExtendedKeyTableBucket = s3_constants.ExtS3TablesPrefix + "tableBucket"
ExtendedKeyMetadata = s3_constants.ExtS3TablesPrefix + "metadata"
ExtendedKeyMetadataVersion = s3_constants.ExtS3TablesPrefix + "metadataVersion"
ExtendedKeyPolicy = s3_constants.ExtS3TablesPrefix + "policy"
ExtendedKeyTags = s3_constants.ExtS3TablesPrefix + "tags"
ExtendedKeyMaintenance = s3_constants.ExtS3TablesPrefix + "maintenance"
// Written by the maintenance worker, read by GetTableMaintenanceJobStatus.
// Separate from ExtendedKeyMaintenance so worker and operator writes do not
// contend on the same attribute.
ExtendedKeyMaintenanceStatus = s3_constants.ExtS3TablesPrefix + "maintenanceStatus"
ExtendedKeyEntryType = s3_constants.ExtS3TablesPrefix + "entryType"
// Entry-type marker values for ExtendedKeyEntryType. Absent or "table" means
// a table; views are stored like tables but tagged "view".
EntryTypeTable = "table"
EntryTypeView = "view"
// Maximum request body size (10MB)
maxRequestBodySize = 10 * 1024 * 1024
)
var (
ErrVersionTokenMismatch = errors.New("version token mismatch")
ErrAccessDenied = errors.New("access denied")
ErrTableAlreadyExists = errors.New("table already exists")
)
type ResourceType string
const (
ResourceTypeBucket ResourceType = "bucket"
ResourceTypeTable ResourceType = "table"
)
// S3TablesHandler handles S3 Tables API requests
type S3TablesHandler struct {
region string
accountID string
defaultAllow bool // Whether to allow access by default (for zero-config IAM)
trusted bool // Trusted local tooling (shell/admin) bypasses authorization
iamAuthorizer IAMAuthorizer
}
// NewS3TablesHandler creates a new S3 Tables handler
func NewS3TablesHandler() *S3TablesHandler {
return &S3TablesHandler{
region: DefaultRegion,
accountID: DefaultAccountID,
defaultAllow: false,
}
}
// SetRegion sets the AWS region for ARN generation
func (h *S3TablesHandler) SetRegion(region string) {
if region != "" {
h.region = region
}
}
// SetAccountID sets the AWS account ID for ARN generation
func (h *S3TablesHandler) SetAccountID(accountID string) {
if accountID != "" {
h.accountID = accountID
}
}
// SetDefaultAllow sets whether to allow access by default
func (h *S3TablesHandler) SetDefaultAllow(allow bool) {
h.defaultAllow = allow
}
// SetTrusted lets local tooling that talks to the filer directly (shell, admin
// console) bypass authorization. HTTP-facing callers must not set it.
func (h *S3TablesHandler) SetTrusted(trusted bool) {
h.trusted = trusted
}
// FilerClient interface for filer operations
type FilerClient interface {
WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error
}
// HandleRequest is the main entry point for S3 Tables API requests
func (h *S3TablesHandler) HandleRequest(w http.ResponseWriter, r *http.Request, filerClient FilerClient) {
operation := r.Header.Get("X-Amz-Target")
if operation != "" {
if idx := strings.LastIndex(operation, "."); idx != -1 {
operation = operation[idx+1:]
}
}
if operation == "" {
glog.V(1).Infof("S3Tables: missing X-Amz-Target header")
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "Missing X-Amz-Target header")
return
}
glog.V(3).Infof("S3Tables: handling operation %s", operation)
var err error
switch operation {
// Table Bucket operations
case "CreateTableBucket":
err = h.handleCreateTableBucket(w, r, filerClient)
case "GetTableBucket":
err = h.handleGetTableBucket(w, r, filerClient)
case "ListTableBuckets":
err = h.handleListTableBuckets(w, r, filerClient)
case "DeleteTableBucket":
err = h.handleDeleteTableBucket(w, r, filerClient)
// Table Bucket Policy operations
case "PutTableBucketPolicy":
err = h.handlePutTableBucketPolicy(w, r, filerClient)
case "GetTableBucketPolicy":
err = h.handleGetTableBucketPolicy(w, r, filerClient)
case "DeleteTableBucketPolicy":
err = h.handleDeleteTableBucketPolicy(w, r, filerClient)
// Namespace operations
case "CreateNamespace":
err = h.handleCreateNamespace(w, r, filerClient)
case "GetNamespace":
err = h.handleGetNamespace(w, r, filerClient)
case "UpdateNamespace":
err = h.handleUpdateNamespace(w, r, filerClient)
case "ListNamespaces":
err = h.handleListNamespaces(w, r, filerClient)
case "DeleteNamespace":
err = h.handleDeleteNamespace(w, r, filerClient)
// Table operations
case "CreateTable":
err = h.handleCreateTable(w, r, filerClient)
case "RegisterTable":
err = h.handleRegisterTable(w, r, filerClient)
case "GetTable":
err = h.handleGetTable(w, r, filerClient)
case "ListTables":
err = h.handleListTables(w, r, filerClient)
case "UpdateTable":
err = h.handleUpdateTable(w, r, filerClient)
case "DeleteTable":
err = h.handleDeleteTable(w, r, filerClient)
case "RenameTable":
err = h.handleRenameTable(w, r, filerClient)
// View operations
case "CreateView":
err = h.handleCreateView(w, r, filerClient)
case "GetView":
err = h.handleGetView(w, r, filerClient)
case "ListViews":
err = h.handleListViews(w, r, filerClient)
case "UpdateView":
err = h.handleUpdateView(w, r, filerClient)
case "DeleteView":
err = h.handleDeleteView(w, r, filerClient)
case "RenameView":
err = h.handleRenameView(w, r, filerClient)
// Table Policy operations
case "PutTablePolicy":
err = h.handlePutTablePolicy(w, r, filerClient)
case "GetTablePolicy":
err = h.handleGetTablePolicy(w, r, filerClient)
case "DeleteTablePolicy":
err = h.handleDeleteTablePolicy(w, r, filerClient)
// Maintenance configuration operations
case "PutTableBucketMaintenanceConfiguration":
err = h.handlePutTableBucketMaintenanceConfiguration(w, r, filerClient)
case "GetTableBucketMaintenanceConfiguration":
err = h.handleGetTableBucketMaintenanceConfiguration(w, r, filerClient)
case "PutTableMaintenanceConfiguration":
err = h.handlePutTableMaintenanceConfiguration(w, r, filerClient)
case "GetTableMaintenanceConfiguration":
err = h.handleGetTableMaintenanceConfiguration(w, r, filerClient)
case "GetTableMaintenanceJobStatus":
err = h.handleGetTableMaintenanceJobStatus(w, r, filerClient)
// Tagging operations
case "TagResource":
err = h.handleTagResource(w, r, filerClient)
case "ListTagsForResource":
err = h.handleListTagsForResource(w, r, filerClient)
case "UntagResource":
err = h.handleUntagResource(w, r, filerClient)
default:
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, fmt.Sprintf("Unknown operation: %s", operation))
return
}
if err != nil {
glog.Errorf("S3Tables: error handling %s: %v", operation, err)
}
}
// Principal/authorization helpers
// getAccountID returns a stable caller identifier for ownership and permission checks.
// Reflection depends on the identity shape produced by JWT/STS auth (Account *struct{Id string},
// Claims map[string]interface{} containing string values for preferred_username/sub, and optional
// identity name/header values). Changing those fields without updating the reflection here will
// break the handler, so refactorers should replace this with a typed interface if needed.
func (h *S3TablesHandler) getAccountID(r *http.Request) string {
identityRaw := s3_constants.GetIdentityFromContext(r)
if identityRaw != nil {
// Use reflection to access identity fields and avoid import cycles.
val := reflect.ValueOf(identityRaw)
if val.Kind() == reflect.Ptr {
val = val.Elem()
}
if val.Kind() == reflect.Struct {
// Prefer stable claims from JWT/STS identities. Only "sub" is guaranteed durable per OIDC;
// preferred_username is ergonomic but can rotate and may orphan ownership data, while email
// is explicitly excluded to avoid storing PII in metadata.
claimsField := val.FieldByName("Claims")
if claimsField.IsValid() && claimsField.Kind() == reflect.Map && !claimsField.IsNil() && claimsField.Type().Key().Kind() == reflect.String {
for _, claimKey := range []string{"sub", "preferred_username"} {
claimVal := claimsField.MapIndex(reflect.ValueOf(claimKey))
if !claimVal.IsValid() {
continue
}
if claimVal.Kind() == reflect.Interface && !claimVal.IsNil() {
claimVal = claimVal.Elem()
}
if claimVal.Kind() == reflect.String {
if principal := normalizePrincipalID(claimVal.String()); principal != "" {
return principal
}
}
}
}
accountField := val.FieldByName("Account")
if accountField.IsValid() && !accountField.IsNil() {
accountVal := accountField.Elem()
if accountVal.Kind() == reflect.Struct {
idField := accountVal.FieldByName("Id")
if idField.IsValid() && idField.Kind() == reflect.String {
if principal := normalizePrincipalID(idField.String()); principal != "" {
// Account-less identities default to the admin account; only
// keep it for real admins, else use the unique identity name.
if principal != s3_constants.AccountAdminId || hasAdminAction(getIdentityActions(r)) {
return principal
}
}
}
}
}
}
}
if identityName := s3_constants.GetIdentityNameFromContext(r); identityName != "" {
if principal := normalizePrincipalID(identityName); principal != "" {
return principal
}
}
if accountID := r.Header.Get(s3_constants.AmzAccountId); accountID != "" {
if principal := normalizePrincipalID(accountID); principal != "" {
return principal
}
}
return h.accountID
}
// normalizePrincipalID collapses ARN and identity strings to a key that is stable within a single account.
// WARNING: this assumes identity names are unique per account; distinct principals such as
// arn:aws:iam::111:user/alice and arn:aws:iam::222:user/alice will both normalize to "alice".
// If future work adds multi-account support, revisit this function to include the account ID or full ARN
// so ownership checks remain correct.
func normalizePrincipalID(id string) string {
id = strings.TrimSpace(id)
if id == "" {
return ""
}
// If this is an ARN (common for assumed roles), use the trailing segment as a
// stable-ish principal key instead of embedding the full ARN in ownership fields.
if strings.HasPrefix(id, "arn:") {
if idx := strings.LastIndex(id, "/"); idx >= 0 && idx+1 < len(id) {
return strings.TrimSpace(id[idx+1:])
}
if idx := strings.LastIndex(id, ":"); idx >= 0 && idx+1 < len(id) {
return strings.TrimSpace(id[idx+1:])
}
return strings.TrimSpace(id)
}
return id
}
// getIdentityActions extracts the action list from the identity object in the request context.
// Uses reflection to avoid import cycles with s3api package.
func getIdentityActions(r *http.Request) []string {
val, ok := getIdentityStructValue(r)
if !ok {
return nil
}
actionsField := val.FieldByName("Actions")
if !actionsField.IsValid() || actionsField.Kind() != reflect.Slice {
return nil
}
// Convert actions to string slice
actions := make([]string, actionsField.Len())
for i := 0; i < actionsField.Len(); i++ {
action := actionsField.Index(i)
// Action is likely a custom type (e.g., type Action string)
// Convert to string using String() or direct string conversion
if action.Kind() == reflect.String {
actions[i] = action.String()
} else if action.CanInterface() {
// Try to convert via fmt.Sprint
actions[i] = fmt.Sprint(action.Interface())
}
}
return actions
}
// Request/Response helpers
func (h *S3TablesHandler) readRequestBody(r *http.Request, v interface{}) error {
defer r.Body.Close()
// Limit request body size to prevent unbounded reads
limitedReader := io.LimitReader(r.Body, maxRequestBodySize+1)
body, err := io.ReadAll(limitedReader)
if err != nil {
return fmt.Errorf("failed to read request body: %w", err)
}
// Check if body exceeds size limit
if len(body) > maxRequestBodySize {
return fmt.Errorf("request body too large: exceeds maximum size of %d bytes", maxRequestBodySize)
}
if len(body) == 0 {
return nil
}
if err := json.Unmarshal(body, v); err != nil {
return fmt.Errorf("failed to decode request: %w", err)
}
return nil
}
// Response writing helpers
func (h *S3TablesHandler) writeJSON(w http.ResponseWriter, status int, data interface{}) {
w.Header().Set("Content-Type", "application/x-amz-json-1.1")
w.WriteHeader(status)
if data != nil {
if err := json.NewEncoder(w).Encode(data); err != nil {
glog.Errorf("S3Tables: failed to encode response: %v", err)
}
}
}
func (h *S3TablesHandler) writeError(w http.ResponseWriter, status int, code, message string) {
w.Header().Set("Content-Type", "application/x-amz-json-1.1")
w.WriteHeader(status)
errorResponse := map[string]interface{}{
"__type": code,
"message": message,
}
if err := json.NewEncoder(w).Encode(errorResponse); err != nil {
glog.Errorf("S3Tables: failed to encode error response: %v", err)
}
}
// ARN generation helpers
func (h *S3TablesHandler) generateTableBucketARN(ownerAccountID, bucketName string) string {
return buildARN(h.region, ownerAccountID, fmt.Sprintf("bucket/%s", bucketName))
}
func (h *S3TablesHandler) generateTableARN(ownerAccountID, bucketName, tableID string) string {
return buildARN(h.region, ownerAccountID, fmt.Sprintf("bucket/%s/table/%s", bucketName, tableID))
}
func (h *S3TablesHandler) generateViewARN(ownerAccountID, bucketName, viewID string) string {
return buildARN(h.region, ownerAccountID, fmt.Sprintf("bucket/%s/view/%s", bucketName, viewID))
}
// generateS3BucketARN builds the plain S3 ARN used for IAM resource matching.
func (h *S3TablesHandler) generateS3BucketARN(bucketName string) string {
return fmt.Sprintf("arn:%s:s3:::%s", arnPartitionForRegion(h.region), bucketName)
}
func isAuthError(err error) bool {
var authErr *AuthError
return errors.As(err, &authErr) || errors.Is(err, ErrAccessDenied)
}
func (h *S3TablesHandler) readTags(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) (map[string]string, error) {
data, err := h.getExtendedAttribute(ctx, client, path, ExtendedKeyTags)
if err != nil {
if errors.Is(err, ErrAttributeNotFound) {
return nil, nil
}
return nil, err
}
tags := make(map[string]string)
if err := json.Unmarshal(data, &tags); err != nil {
return nil, fmt.Errorf("failed to unmarshal tags: %w", err)
}
return tags, nil
}
func mapKeys(tags map[string]string) []string {
if len(tags) == 0 {
return nil
}
keys := make([]string, 0, len(tags))
for key := range tags {
keys = append(keys, key)
}
return keys
}

View File

@@ -0,0 +1,190 @@
package s3tables
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
// handleCreateTableBucket creates a new table bucket
func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req CreateTableBucketRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
// Validate bucket name
if err := validateBucketName(req.Name); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
// A bucket is a catalog, and a catalog serves one protocol. Saying which one
// at creation is what lets everything downstream - the endpoint the UI
// shows, the tables the bucket accepts - be answered without opening a table.
bucketFormat := FormatIceberg
if req.Format != "" {
normalized, ok := NormalizeFormat(req.Format)
if !ok {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, fmt.Sprintf("unsupported format %q", req.Format))
return fmt.Errorf("invalid format")
}
bucketFormat = normalized
}
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
identityPolicyNames := getIdentityPolicyNames(r)
useIAM := h.shouldUseIAM(r, identityActions, identityPolicyNames)
useLegacy := !useIAM
if useIAM {
allowed, err := h.authorizeIAMAction(r, identityPolicyNames, "CreateTableBucket", h.generateTableBucketARN(principal, req.Name), h.generateS3BucketARN(req.Name))
if err != nil {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create table buckets")
return NewAuthError("CreateTableBucket", principal, "not authorized to create table buckets")
}
if !allowed {
if h.defaultAllow && len(identityActions) == 0 && len(identityPolicyNames) == 0 {
useLegacy = true
} else {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create table buckets")
return NewAuthError("CreateTableBucket", principal, "not authorized to create table buckets")
}
}
}
if useLegacy {
owner := h.accountID
if owner == "" {
owner = DefaultAccountID
}
if !CheckPermissionWithContext("CreateTableBucket", principal, owner, "", "", &PolicyContext{
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create table buckets")
return NewAuthError("CreateTableBucket", principal, "not authorized to create table buckets")
}
}
bucketPath := GetTableBucketPath(req.Name)
// Check if bucket already exists and ensure no conflict with object store buckets
tableBucketExists := false
s3BucketExists := false
bucketsPath := s3_constants.DefaultBucketsPath
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.GetFilerConfiguration(r.Context(), &filer_pb.GetFilerConfigurationRequest{})
if err != nil {
return err
}
if resp.DirBuckets != "" {
bucketsPath = resp.DirBuckets
}
entryResp, err := filer_pb.LookupEntry(r.Context(), client, &filer_pb.LookupDirectoryEntryRequest{
Directory: bucketsPath,
Name: req.Name,
})
if err != nil {
if !errors.Is(err, filer_pb.ErrNotFound) {
return err
}
return nil
}
if entryResp != nil && entryResp.Entry != nil {
if IsTableBucketEntry(entryResp.Entry) {
tableBucketExists = true
} else {
s3BucketExists = true
}
}
return nil
})
if err != nil {
glog.Errorf("S3Tables: failed to check bucket existence: %v", err)
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to check bucket existence")
return err
}
if tableBucketExists {
h.writeError(w, http.StatusConflict, ErrCodeBucketAlreadyExists, fmt.Sprintf("table bucket %s already exists", req.Name))
return fmt.Errorf("bucket already exists")
}
if s3BucketExists {
h.writeError(w, http.StatusConflict, ErrCodeBucketAlreadyExists, fmt.Sprintf("bucket %s already exists", req.Name))
return fmt.Errorf("bucket already exists")
}
// Create the bucket directory and set metadata as extended attributes
now := time.Now()
metadata := &tableBucketMetadata{
Name: req.Name,
CreatedAt: now,
OwnerAccountID: principal,
Format: bucketFormat,
}
metadataBytes, err := json.Marshal(metadata)
if err != nil {
glog.Errorf("S3Tables: failed to marshal metadata: %v", err)
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal metadata")
return err
}
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
// Ensure root tables directory exists
if !h.entryExists(r.Context(), client, TablesPath) {
if err := h.createDirectory(r.Context(), client, TablesPath); err != nil {
return fmt.Errorf("failed to create root tables directory: %w", err)
}
}
// Create bucket directory
if err := h.createDirectory(r.Context(), client, bucketPath); err != nil {
return err
}
// Mark as a table bucket
if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyTableBucket, []byte("true")); err != nil {
return err
}
// Set metadata as extended attribute
if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata, metadataBytes); err != nil {
return err
}
// Set tags if provided
if len(req.Tags) > 0 {
tagsBytes, err := json.Marshal(req.Tags)
if err != nil {
return fmt.Errorf("failed to marshal tags: %w", err)
}
if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyTags, tagsBytes); err != nil {
return err
}
}
return nil
})
if err != nil {
glog.Errorf("S3Tables: failed to create table bucket %s: %v", req.Name, err)
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to create table bucket")
return err
}
resp := &CreateTableBucketResponse{
ARN: h.generateTableBucketARN(metadata.OwnerAccountID, req.Name),
}
h.writeJSON(w, http.StatusOK, resp)
return nil
}

View File

@@ -0,0 +1,109 @@
package s3tables
import (
"context"
"encoding/json"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
"github.com/stretchr/testify/require"
)
const formatTestBucket = "formats"
// bucketWithFormat lays down a table bucket holding one namespace, declared as
// the given format. An empty format is a bucket from before the declaration
// existed.
func bucketWithFormat(t *testing.T, format string) (*s3tablestest.MemFiler, *Manager) {
t.Helper()
fs := s3tablestest.Start(t)
m := NewManager()
bucketMeta, _ := json.Marshal(tableBucketMetadata{
Name: formatTestBucket,
OwnerAccountID: DefaultAccountID,
Format: format,
})
fs.Put(TablesPath, formatTestBucket, map[string][]byte{
ExtendedKeyTableBucket: []byte("{}"),
ExtendedKeyMetadata: bucketMeta,
})
nsMeta, _ := json.Marshal(namespaceMetadata{Namespace: []string{"ns"}, OwnerAccountID: DefaultAccountID})
fs.Put(GetTableBucketPath(formatTestBucket), "ns", map[string][]byte{ExtendedKeyMetadata: nsMeta})
return fs, m
}
func createTableOfFormat(t *testing.T, m *Manager, fs *s3tablestest.MemFiler, name, format string) error {
t.Helper()
return m.Execute(context.Background(), NewManagerClient(fs.Client), "CreateTable", &CreateTableRequest{
TableBucketARN: "arn:aws:s3tables:::bucket/" + formatTestBucket,
Namespace: []string{"ns"},
Name: name,
Format: format,
}, nil, "")
}
// The declaration is the point: a bucket that says it holds Lance cannot be
// handed an Iceberg table, because the catalog serving it would never show one.
func TestCreateTableRefusesAForeignFormat(t *testing.T) {
fs, m := bucketWithFormat(t, FormatLance)
require.NoError(t, createTableOfFormat(t, m, fs, "vectors", FormatLance))
err := createTableOfFormat(t, m, fs, "events", FormatIceberg)
require.Error(t, err, "an Iceberg table in a Lance bucket should be refused")
require.Contains(t, err.Error(), "holds LANCE")
}
func TestCreateTableRefusesLanceInAnIcebergBucket(t *testing.T) {
fs, m := bucketWithFormat(t, FormatIceberg)
err := createTableOfFormat(t, m, fs, "vectors", FormatLance)
require.Error(t, err)
require.Contains(t, err.Error(), "holds ICEBERG")
}
// A bucket made before formats were declared keeps taking anything. Nothing is
// migrated, so nothing that worked stops working.
func TestUndeclaredBucketAcceptsEitherFormat(t *testing.T) {
fs, m := bucketWithFormat(t, "")
require.NoError(t, createTableOfFormat(t, m, fs, "events", FormatIceberg))
require.NoError(t, createTableOfFormat(t, m, fs, "vectors", FormatLance))
}
// A view is Iceberg metadata, so it has no meaning in a bucket of another format.
func TestCreateViewRefusedInALanceBucket(t *testing.T) {
fs, m := bucketWithFormat(t, FormatLance)
err := m.Execute(context.Background(), NewManagerClient(fs.Client), "CreateView", &CreateViewRequest{
TableBucketARN: "arn:aws:s3tables:::bucket/" + formatTestBucket,
Namespace: []string{"ns"},
Name: "v",
}, nil, "")
require.Error(t, err)
require.Contains(t, err.Error(), "cannot hold views")
}
func TestNormalizeFormat(t *testing.T) {
cases := []struct {
in string
want string
ok bool
}{
{"ICEBERG", FormatIceberg, true},
{"iceberg", FormatIceberg, true},
{" Lance ", FormatLance, true},
{"delta", "", false},
{"", "", false},
}
for _, c := range cases {
got, ok := NormalizeFormat(c.in)
if got != c.want || ok != c.ok {
t.Errorf("NormalizeFormat(%q) = %q,%v; want %q,%v", c.in, got, ok, c.want, c.ok)
}
}
}

View File

@@ -0,0 +1,366 @@
package s3tables
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
// handleGetTableBucket gets details of a table bucket
func (h *S3TablesHandler) handleGetTableBucket(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
// Check permission
var req GetTableBucketRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
return fmt.Errorf("tableBucketARN is required")
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketPath := GetTableBucketPath(bucketName)
var metadata tableBucketMetadata
var bucketPolicy string
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &metadata); err != nil {
return fmt.Errorf("failed to unmarshal metadata: %w", err)
}
// Fetch bucket policy if it exists
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err == nil {
bucketPolicy = string(policyData)
} else if !errors.Is(err, ErrAttributeNotFound) {
return fmt.Errorf("failed to fetch bucket policy: %v", err)
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get table bucket: %v", err))
}
return err
}
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, bucketName)
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("GetTableBucket", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table bucket details")
return ErrAccessDenied
}
resp := &GetTableBucketResponse{
ARN: h.generateTableBucketARN(metadata.OwnerAccountID, bucketName),
Name: metadata.Name,
OwnerAccountID: metadata.OwnerAccountID,
CreatedAt: metadata.CreatedAt,
Format: metadata.Format,
}
h.writeJSON(w, http.StatusOK, resp)
return nil
}
// handleListTableBuckets lists all table buckets
func (h *S3TablesHandler) handleListTableBuckets(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req ListTableBucketsRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
// No account-level gate: visibility is enforced per bucket below, so an
// owner can always list its own buckets and others are filtered out.
accountID := h.getAccountID(r)
maxBuckets := req.MaxBuckets
if maxBuckets <= 0 {
maxBuckets = 100
}
// Cap to prevent uint32 overflow when used in uint32(maxBuckets*2)
const maxBucketsLimit = 1000
if maxBuckets > maxBucketsLimit {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "MaxBuckets exceeds maximum allowed value")
return fmt.Errorf("invalid maxBuckets value: %d", maxBuckets)
}
var buckets []TableBucketSummary
lastFileName := req.ContinuationToken
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
for len(buckets) < maxBuckets {
resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{
Directory: TablesPath,
Limit: uint32(maxBuckets * 2), // Fetch more than needed to account for filtering
StartFromFileName: lastFileName,
InclusiveStartFrom: lastFileName == "" || lastFileName == req.ContinuationToken,
})
if err != nil {
return err
}
hasMore := false
for {
entry, respErr := resp.Recv()
if respErr != nil {
if respErr == io.EOF {
break
}
return respErr
}
if entry.Entry == nil {
continue
}
// Skip the start item if it was included in the previous page
if len(buckets) == 0 && req.ContinuationToken != "" && entry.Entry.Name == req.ContinuationToken {
continue
}
hasMore = true
lastFileName = entry.Entry.Name
if !entry.Entry.IsDirectory {
continue
}
// Skip entries starting with "."
if strings.HasPrefix(entry.Entry.Name, ".") {
continue
}
// Apply prefix filter
if req.Prefix != "" && !strings.HasPrefix(entry.Entry.Name, req.Prefix) {
continue
}
if !IsTableBucketEntry(entry.Entry) {
continue
}
// Read metadata from extended attribute
data, ok := entry.Entry.Extended[ExtendedKeyMetadata]
if !ok {
continue
}
var metadata tableBucketMetadata
if err := json.Unmarshal(data, &metadata); err != nil {
continue
}
bucketPath := GetTableBucketPath(entry.Entry.Name)
bucketPolicy := ""
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err != nil {
if !errors.Is(err, ErrAttributeNotFound) {
continue
}
} else {
bucketPolicy = string(policyData)
}
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, entry.Entry.Name)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("GetTableBucket", accountID, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: entry.Entry.Name,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
continue
}
buckets = append(buckets, TableBucketSummary{
ARN: bucketARN,
Name: entry.Entry.Name,
CreatedAt: metadata.CreatedAt,
Format: metadata.Format,
})
if len(buckets) >= maxBuckets {
return nil
}
}
if !hasMore {
break
}
}
return nil
})
if err != nil {
// Check if it's a "not found" error - return empty list in that case
if errors.Is(err, filer_pb.ErrNotFound) {
buckets = []TableBucketSummary{}
} else {
// For other errors, return error response
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list table buckets: %v", err))
return err
}
}
paginationToken := ""
if len(buckets) >= maxBuckets {
paginationToken = lastFileName
}
resp := &ListTableBucketsResponse{
TableBuckets: buckets,
ContinuationToken: paginationToken,
}
h.writeJSON(w, http.StatusOK, resp)
return nil
}
// handleDeleteTableBucket deletes a table bucket
func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req DeleteTableBucketRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
return fmt.Errorf("tableBucketARN is required")
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketPath := GetTableBucketPath(bucketName)
// Check if bucket exists and perform ownership + emptiness check in one block
var metadata tableBucketMetadata
var bucketPolicy string
hasChildren := false
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
// 1. Get metadata for ownership check
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &metadata); err != nil {
return fmt.Errorf("failed to unmarshal metadata: %w", err)
}
// Fetch bucket policy if it exists
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err != nil {
if errors.Is(err, ErrAttributeNotFound) {
// No bucket policy set; proceed with empty bucketPolicy
} else {
return fmt.Errorf("failed to fetch bucket policy: %w", err)
}
} else {
bucketPolicy = string(policyData)
}
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, bucketName)
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("DeleteTableBucket", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
return NewAuthError("DeleteTableBucket", principal, fmt.Sprintf("not authorized to delete bucket %s", bucketName))
}
// 3. Check if bucket is empty
resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{
Directory: bucketPath,
Limit: 10,
})
if err != nil {
return err
}
for {
entry, err := resp.Recv()
if err != nil {
if err == io.EOF {
break
}
return err
}
if entry.Entry != nil && !strings.HasPrefix(entry.Entry.Name, ".") {
hasChildren = true
break
}
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
} else if isAuthError(err) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, err.Error())
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to delete table bucket: %v", err))
}
return err
}
if hasChildren {
h.writeError(w, http.StatusConflict, ErrCodeBucketNotEmpty, "table bucket is not empty")
return fmt.Errorf("bucket not empty")
}
// Delete the bucket
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
// Drop the leaf entry first, so it does not outlive the directory.
if tableObjErr := h.deleteEntryIfExists(r.Context(), client, GetTableObjectBucketPath(bucketName)); tableObjErr != nil {
glog.V(1).Infof("failed to delete table object for %s: %v", bucketName, tableObjErr)
}
// The bucket is the directory, so a refused delete leaves it in place
// and the caller must hear about it.
return h.deleteDirectory(r.Context(), client, bucketPath)
})
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table bucket")
return err
}
h.writeJSON(w, http.StatusOK, nil)
return nil
}

View File

@@ -0,0 +1,122 @@
package s3tables
import (
"context"
"encoding/json"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func runDeleteTable(t *testing.T, m *Manager, fs *s3tablestest.MemFiler, namespace, name string) error {
t.Helper()
return m.Execute(context.Background(), NewManagerClient(fs.Client), "DeleteTable", &DeleteTableRequest{
TableBucketARN: mustBucketARN(t),
Namespace: []string{namespace},
Name: name,
}, nil, "")
}
// A table whose data was decoupled from its name (created over a leftover from a
// rename): catalog marker at ns/newt, data at ns/newt-x, and ns/newt still holds
// another table's leftover data. Dropping it must purge only its own data and
// clear the marker, never the data under the reused name path.
func TestDeleteTableDecoupledKeepsReusedNamePath(t *testing.T) {
fs, m := startRenameManager(t)
newtMeta, _ := json.Marshal(tableMetadataInternal{
Name: "newt",
Namespace: "ns",
Format: "ICEBERG",
OwnerAccountID: DefaultAccountID,
MetadataLocation: "s3://" + renameTestBucket + "/ns/newt-x/metadata/v1.metadata.json",
})
markerKeys := []string{ExtendedKeyMetadata, ExtendedKeyMetadataVersion, ExtendedKeyPolicy, ExtendedKeyTags, ExtendedKeyEntryType}
fs.Put(GetNamespacePath(renameTestBucket, "ns"), "newt", map[string][]byte{
ExtendedKeyMetadata: newtMeta,
ExtendedKeyMetadataVersion: []byte("v1"),
ExtendedKeyPolicy: []byte(`{"Version":"2012-10-17"}`),
ExtendedKeyTags: []byte(`{"k":"v"}`),
ExtendedKeyEntryType: []byte(EntryTypeTable),
})
fs.Put(GetTablePath(renameTestBucket, "ns", "newt"), "leftover", nil) // another table's data under the name path
fs.Put(GetNamespacePath(renameTestBucket, "ns"), "newt-x", nil) // this table's own (decoupled) data
fs.Put(GetTablePath(renameTestBucket, "ns", "newt-x"), "metadata", nil)
require.NoError(t, runDeleteTable(t, m, fs, "ns", "newt"))
assert.Nil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "newt-x"),
"the table's own data location must be purged")
assert.NotNil(t, fs.Get(GetTablePath(renameTestBucket, "ns", "newt"), "leftover"),
"data under the reused name path must survive")
marker := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "newt")
require.NotNil(t, marker)
for _, key := range markerKeys {
_, present := marker.Extended[key]
assert.Falsef(t, present, "catalog attribute %s must be cleared", key)
}
}
// A table whose MetadataLocation resolves to an ancestor of its own name path
// (here the namespace root, e.g. from corrupt metadata) must not be deleted: a
// recursive purge of that data path would wipe sibling tables. The delete is
// refused and the namespace's other tables survive.
func TestDeleteTableRefusesAncestorDataPath(t *testing.T) {
fs, m := startRenameManager(t)
badMeta, _ := json.Marshal(tableMetadataInternal{
Name: "badt",
Namespace: "ns",
Format: "ICEBERG",
OwnerAccountID: DefaultAccountID,
MetadataLocation: "s3://" + renameTestBucket + "/ns/metadata/v1.metadata.json",
})
fs.Put(GetNamespacePath(renameTestBucket, "ns"), "badt", map[string][]byte{ExtendedKeyMetadata: badMeta})
require.Error(t, runDeleteTable(t, m, fs, "ns", "badt"))
// The sibling table seeded by startRenameManager and its data must survive.
assert.NotNil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t"),
"sibling table marker must survive a refused delete")
assert.NotNil(t, fs.Get(GetTablePath(renameTestBucket, "ns", "t"), "data"),
"sibling table data must survive a refused delete")
}
// A normal colocated table (data under its own name path) is removed wholesale.
func TestDeleteTableColocatedRemovesData(t *testing.T) {
fs, m := startRenameManager(t)
require.NoError(t, runDeleteTable(t, m, fs, "ns", "t"))
assert.Nil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t"), "colocated table entry must be deleted")
assert.Nil(t, fs.Get(GetTablePath(renameTestBucket, "ns", "t"), "metadata"), "colocated table data must be deleted")
}
// A table whose MetadataLocation points at a sibling that is still a live
// catalog entry must not be deleted: only the named table was authorized, so a
// recursive purge of that location would destroy another tenant's table.
func TestDeleteTableRefusesLiveSiblingDataPath(t *testing.T) {
fs, m := startRenameManager(t)
nsMeta, _ := json.Marshal(namespaceMetadata{Namespace: []string{"attackerns"}, OwnerAccountID: DefaultAccountID})
fs.Put(GetTableBucketPath(renameTestBucket), "attackerns", map[string][]byte{ExtendedKeyMetadata: nsMeta})
decoyMeta, _ := json.Marshal(tableMetadataInternal{
Name: "decoy",
Namespace: "attackerns",
Format: "ICEBERG",
OwnerAccountID: DefaultAccountID,
MetadataLocation: "s3://" + renameTestBucket + "/ns/t/metadata/v3.metadata.json",
})
fs.Put(GetNamespacePath(renameTestBucket, "attackerns"), "decoy", map[string][]byte{ExtendedKeyMetadata: decoyMeta})
require.Error(t, runDeleteTable(t, m, fs, "attackerns", "decoy"))
assert.NotNil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t"),
"the targeted table's catalog entry must survive")
assert.NotNil(t, fs.Get(GetTablePath(renameTestBucket, "ns", "t"), "data"),
"the targeted table's data must survive")
}

View File

@@ -0,0 +1,173 @@
package s3tables
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/stretchr/testify/assert"
)
// testIdentity/testIdentityAccount mirror the production identity shape used via reflection.
// Keep these field names in sync with getAccountID to avoid silent breaks.
type testIdentityAccount struct {
Id string
}
type testIdentity struct {
Name string
Account *testIdentityAccount
Actions []string
Claims map[string]interface{}
}
func TestGetAccountIDPrefersClaimsOverAccountID(t *testing.T) {
h := NewS3TablesHandler()
id := &testIdentity{
Account: &testIdentityAccount{Id: s3_constants.AccountAdminId},
Claims: map[string]interface{}{
"preferred_username": "alice",
"sub": "alice-sub",
},
}
req := httptest.NewRequest(http.MethodGet, "/", nil)
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id))
got := h.getAccountID(req)
assert.Equal(t, "alice-sub", got, "expected sub claim to be used before preferred_username")
assert.NotEqual(t, DefaultAccountID, got, "claims should override default handler account")
}
func TestGetAccountIDUsesSubWhenPreferredUsernameMissing(t *testing.T) {
h := NewS3TablesHandler()
id := &testIdentity{
Account: &testIdentityAccount{Id: s3_constants.AccountAdminId},
Claims: map[string]interface{}{
"sub": "user-123",
},
}
req := httptest.NewRequest(http.MethodGet, "/", nil)
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id))
got := h.getAccountID(req)
assert.Equal(t, "user-123", got, "expected sub claim to be used when preferred_username missing")
assert.NotEqual(t, DefaultAccountID, got, "claims should override default handler account")
}
func TestGetAccountIDFallsBackToHandlerDefaultAccount(t *testing.T) {
h := NewS3TablesHandler()
req := httptest.NewRequest(http.MethodGet, "/", nil)
assert.Equal(t, DefaultAccountID, h.getAccountID(req), "expected handler default account to be returned when no identity is set")
}
func TestGetAccountIDIgnoresEmptyClaimValues(t *testing.T) {
h := NewS3TablesHandler()
id := &testIdentity{
Account: &testIdentityAccount{Id: s3_constants.AccountAdminId},
Claims: map[string]interface{}{
"preferred_username": " ",
"sub": "user-123",
},
}
req := httptest.NewRequest(http.MethodGet, "/", nil)
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id))
assert.Equal(t, "user-123", h.getAccountID(req), "expected whitespace preferred_username to be ignored")
}
func TestGetAccountIDFallsBackToIdentityName(t *testing.T) {
h := NewS3TablesHandler()
req := httptest.NewRequest(http.MethodGet, "/", nil)
req = req.WithContext(s3_constants.SetIdentityNameInContext(req.Context(), "arn:aws:sts::123456789012:assumed-role/S3UserRole/alice-session"))
assert.Equal(t, "alice-session", h.getAccountID(req), "expected ARN session suffix to be extracted")
}
func TestGetAccountIDFallsBackToARNColonSegment(t *testing.T) {
h := NewS3TablesHandler()
req := httptest.NewRequest(http.MethodGet, "/", nil)
req = req.WithContext(s3_constants.SetIdentityNameInContext(req.Context(), "arn:aws:iam::123456789012:root"))
assert.Equal(t, "root", h.getAccountID(req), "expected ARN colon segment to be returned as principal")
}
func TestGetAccountIDFallsBackToAmzAccountIdHeader(t *testing.T) {
h := NewS3TablesHandler()
req := httptest.NewRequest(http.MethodGet, "/", nil)
req.Header.Set(s3_constants.AmzAccountId, "header-account")
assert.Equal(t, "header-account", h.getAccountID(req), "expected header value to be used when no identity is present")
}
func TestGetAccountIDFallsBackToAccountID(t *testing.T) {
h := NewS3TablesHandler()
id := &testIdentity{
Account: &testIdentityAccount{Id: "my-account-id"},
}
req := httptest.NewRequest(http.MethodGet, "/", nil)
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id))
assert.Equal(t, "my-account-id", h.getAccountID(req), "expected Account.Id to be returned when claims are missing")
}
func TestGetAccountIDNonAdminDoesNotInheritAdminAccount(t *testing.T) {
h := NewS3TablesHandler()
id := &testIdentity{
Account: &testIdentityAccount{Id: s3_constants.AccountAdminId},
Actions: []string{"Read", "List"},
}
req := httptest.NewRequest(http.MethodGet, "/", nil)
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id))
req = req.WithContext(s3_constants.SetIdentityNameInContext(req.Context(), "readonly"))
assert.Equal(t, "readonly", h.getAccountID(req), "a non-admin identity must not inherit the shared admin account")
}
func TestGetAccountIDAdminActionKeepsAdminAccount(t *testing.T) {
h := NewS3TablesHandler()
id := &testIdentity{
Account: &testIdentityAccount{Id: s3_constants.AccountAdminId},
Actions: []string{s3_constants.ACTION_ADMIN},
}
req := httptest.NewRequest(http.MethodGet, "/", nil)
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id))
req = req.WithContext(s3_constants.SetIdentityNameInContext(req.Context(), "root"))
assert.Equal(t, s3_constants.AccountAdminId, h.getAccountID(req), "an admin identity keeps the admin account as principal")
}
func TestDefaultAllowForOnlyAppliesToUnauthenticatedOrAnonymous(t *testing.T) {
h := NewS3TablesHandler()
h.SetDefaultAllow(true)
noIdentity := httptest.NewRequest(http.MethodGet, "/", nil)
assert.True(t, h.defaultAllowFor(noIdentity), "zero-config requests with no identity keep the open default")
anon := httptest.NewRequest(http.MethodGet, "/", nil)
anon = anon.WithContext(s3_constants.SetIdentityInContext(anon.Context(),
&testIdentity{Name: s3_constants.AccountAnonymousId}))
assert.True(t, h.defaultAllowFor(anon), "anonymous requests keep the open default")
authed := httptest.NewRequest(http.MethodGet, "/", nil)
authed = authed.WithContext(s3_constants.SetIdentityInContext(authed.Context(),
&testIdentity{Name: "readonly", Account: &testIdentityAccount{Id: s3_constants.AccountAdminId}, Actions: []string{"Read"}}))
assert.False(t, h.defaultAllowFor(authed), "an authenticated identity must not benefit from the open default")
h.SetDefaultAllow(false)
assert.False(t, h.defaultAllowFor(noIdentity), "default-allow disabled is never open")
}
func TestGetAccountIDNormalizesAccountIDARN(t *testing.T) {
h := NewS3TablesHandler()
id := &testIdentity{
Account: &testIdentityAccount{Id: "arn:aws:iam::123456789012:user/bob"},
}
req := httptest.NewRequest(http.MethodGet, "/", nil)
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id))
assert.Equal(t, "bob", h.getAccountID(req), "expected ARN account ID to be normalized to the suffix")
}

View File

@@ -0,0 +1,529 @@
package s3tables
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
// Maintenance types are scoped: AWS configures unreferenced file removal on the
// table bucket and compaction and snapshot management on the table.
var (
bucketMaintenanceTypes = map[string]bool{
MaintenanceTypeIcebergUnreferencedFileRemoval: true,
}
tableMaintenanceTypes = map[string]bool{
MaintenanceTypeIcebergCompaction: true,
MaintenanceTypeIcebergSnapshotManagement: true,
}
)
// handlePutTableBucketMaintenanceConfiguration sets a maintenance configuration on a table bucket
func (h *S3TablesHandler) handlePutTableBucketMaintenanceConfiguration(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req PutTableBucketMaintenanceConfigurationRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
return fmt.Errorf("tableBucketARN is required")
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if err := validateMaintenanceValue(req.Type, bucketMaintenanceTypes, req.Value); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if _, err := h.authorizeMaintenanceBucket(r, filerClient, "PutTableBucketMaintenanceConfiguration", bucketName); err != nil {
h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
return err
}
if err := h.putMaintenanceConfiguration(r, filerClient, GetTableBucketPath(bucketName), req.Type, req.Value); err != nil {
h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
return err
}
h.writeJSON(w, http.StatusOK, nil)
return nil
}
// handleGetTableBucketMaintenanceConfiguration gets the maintenance configuration of a table bucket
func (h *S3TablesHandler) handleGetTableBucketMaintenanceConfiguration(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req GetTableBucketMaintenanceConfigurationRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
return fmt.Errorf("tableBucketARN is required")
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketARN, err := h.authorizeMaintenanceBucket(r, filerClient, "GetTableBucketMaintenanceConfiguration", bucketName)
if err != nil {
h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
return err
}
config, err := h.readMaintenanceConfiguration(r, filerClient, GetTableBucketPath(bucketName))
if err != nil {
h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
return err
}
h.writeJSON(w, http.StatusOK, &GetTableBucketMaintenanceConfigurationResponse{
TableBucketARN: bucketARN,
Configuration: config,
})
return nil
}
// handlePutTableMaintenanceConfiguration sets a maintenance configuration on a table
func (h *S3TablesHandler) handlePutTableMaintenanceConfiguration(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req PutTableMaintenanceConfigurationRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketName, namespaceName, tableName, err := h.parseTableTarget(req.TableBucketARN, req.Namespace, req.Name)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if err := validateMaintenanceValue(req.Type, tableMaintenanceTypes, req.Value); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if _, err := h.authorizeMaintenanceTable(r, filerClient, "PutTableMaintenanceConfiguration", bucketName, namespaceName, tableName); err != nil {
h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
return err
}
tablePath := GetTablePath(bucketName, namespaceName, tableName)
if err := h.putMaintenanceConfiguration(r, filerClient, tablePath, req.Type, req.Value); err != nil {
h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
return err
}
h.writeJSON(w, http.StatusOK, nil)
return nil
}
// handleGetTableMaintenanceConfiguration gets the maintenance configuration of a table
func (h *S3TablesHandler) handleGetTableMaintenanceConfiguration(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req GetTableMaintenanceConfigurationRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketName, namespaceName, tableName, err := h.parseTableTarget(req.TableBucketARN, req.Namespace, req.Name)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
tableARN, err := h.authorizeMaintenanceTable(r, filerClient, "GetTableMaintenanceConfiguration", bucketName, namespaceName, tableName)
if err != nil {
h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
return err
}
tablePath := GetTablePath(bucketName, namespaceName, tableName)
config, err := h.readMaintenanceConfiguration(r, filerClient, tablePath)
if err != nil {
h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
return err
}
h.writeJSON(w, http.StatusOK, &GetTableMaintenanceConfigurationResponse{
TableARN: tableARN,
Namespace: []string{namespaceName},
Name: tableName,
Configuration: config,
})
return nil
}
// handleGetTableMaintenanceJobStatus reports the outcome of the last maintenance run on a table
func (h *S3TablesHandler) handleGetTableMaintenanceJobStatus(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req GetTableMaintenanceJobStatusRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketName, namespaceName, tableName, err := h.parseTableTarget(req.TableBucketARN, req.Namespace, req.Name)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
tableARN, err := h.authorizeMaintenanceTable(r, filerClient, "GetTableMaintenanceJobStatus", bucketName, namespaceName, tableName)
if err != nil {
h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
return err
}
tablePath := GetTablePath(bucketName, namespaceName, tableName)
// Unreferenced file removal is configured on the bucket, so a bucket-level
// disable has to show up here as Disabled rather than a stale table status.
bucketConfig, err := h.readMaintenanceConfiguration(r, filerClient, GetTableBucketPath(bucketName))
if err != nil {
h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
return err
}
tableConfig, err := h.readMaintenanceConfiguration(r, filerClient, tablePath)
if err != nil {
h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
return err
}
config := MergeMaintenanceConfiguration(bucketConfig, tableConfig)
recorded := MaintenanceJobStatus{}
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMaintenanceStatus)
if err != nil {
if errors.Is(err, ErrAttributeNotFound) {
return nil
}
return err
}
return json.Unmarshal(data, &recorded)
})
if err != nil {
h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
return err
}
h.writeJSON(w, http.StatusOK, &GetTableMaintenanceJobStatusResponse{
TableARN: tableARN,
Status: buildJobStatusResponse(recorded, config),
})
return nil
}
// buildJobStatusResponse reports every job type: what the worker recorded, or
// Disabled when the configuration switched the type off, or Not_Yet_Run.
func buildJobStatusResponse(recorded MaintenanceJobStatus, config MaintenanceConfiguration) MaintenanceJobStatus {
jobTypes := []string{
MaintenanceTypeIcebergCompaction,
MaintenanceTypeIcebergSnapshotManagement,
MaintenanceTypeIcebergUnreferencedFileRemoval,
}
status := make(MaintenanceJobStatus, len(jobTypes))
for _, jobType := range jobTypes {
if value, ok := config[jobType]; ok && value != nil && value.Status == MaintenanceStatusDisabled {
status[jobType] = &MaintenanceJobStatusValue{Status: MaintenanceJobStatusDisabled}
continue
}
if value, ok := recorded[jobType]; ok && value != nil {
status[jobType] = value
continue
}
status[jobType] = &MaintenanceJobStatusValue{Status: MaintenanceJobStatusNotYetRun}
}
return status
}
// putMaintenanceConfiguration merges one maintenance type into the stored
// configuration, leaving the other types alone. The write checks the catalog
// identity in the same conditional mutation, so it cannot land on a name a
// concurrent rename or delete has already soft-deleted.
func (h *S3TablesHandler) putMaintenanceConfiguration(
r *http.Request,
filerClient FilerClient,
resourcePath, maintenanceType string,
value *MaintenanceConfigurationValue,
) error {
return filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return h.mutateEntryExtended(r.Context(), client, resourcePath, func(extended map[string][]byte) error {
if len(extended[ExtendedKeyMetadata]) == 0 {
return fmt.Errorf("%w: %s", filer_pb.ErrNotFound, resourcePath)
}
config := MaintenanceConfiguration{}
if current := extended[ExtendedKeyMaintenance]; len(current) > 0 {
if err := json.Unmarshal(current, &config); err != nil {
return fmt.Errorf("failed to unmarshal maintenance configuration: %w", err)
}
}
config[maintenanceType] = value
data, err := json.Marshal(config)
if err != nil {
return err
}
extended[ExtendedKeyMaintenance] = data
return nil
})
})
}
// readMaintenanceConfiguration returns the stored configuration, or an empty
// one when the resource has never been configured.
func (h *S3TablesHandler) readMaintenanceConfiguration(r *http.Request, filerClient FilerClient, resourcePath string) (MaintenanceConfiguration, error) {
config := MaintenanceConfiguration{}
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMaintenance)
if err != nil {
if errors.Is(err, ErrAttributeNotFound) {
return nil
}
return err
}
return json.Unmarshal(data, &config)
})
if err != nil {
return nil, err
}
return config, nil
}
// parseTableTarget validates the bucket/namespace/table triple every
// table-scoped maintenance request carries.
func (h *S3TablesHandler) parseTableTarget(tableBucketARN string, namespace []string, name string) (bucketName, namespaceName, tableName string, err error) {
if tableBucketARN == "" || len(namespace) == 0 || name == "" {
return "", "", "", fmt.Errorf("tableBucketARN, namespace, and name are required")
}
if namespaceName, err = validateNamespace(namespace); err != nil {
return "", "", "", err
}
if bucketName, err = parseBucketNameFromARN(tableBucketARN); err != nil {
return "", "", "", err
}
if tableName, err = validateTableName(name); err != nil {
return "", "", "", err
}
return bucketName, namespaceName, tableName, nil
}
// authorizeMaintenanceBucket checks the caller may perform operation on the
// table bucket, returning the bucket ARN.
func (h *S3TablesHandler) authorizeMaintenanceBucket(r *http.Request, filerClient FilerClient, operation, bucketName string) (string, error) {
bucketPath := GetTableBucketPath(bucketName)
var bucketMetadata tableBucketMetadata
var bucketPolicy string
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
}
bucketPolicy, err = h.readBucketPolicy(r, client, bucketPath)
return err
})
if err != nil {
return "", err
}
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
principal := h.getAccountID(r)
if !CheckPermissionWithContext(operation, principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: getIdentityActions(r),
DefaultAllow: h.defaultAllowFor(r),
}) {
return "", NewAuthError(operation, principal, "not authorized to "+operation)
}
return bucketARN, nil
}
// authorizeMaintenanceTable checks the caller may perform operation on the
// table, returning the table ARN.
func (h *S3TablesHandler) authorizeMaintenanceTable(r *http.Request, filerClient FilerClient, operation, bucketName, namespaceName, tableName string) (string, error) {
tablePath := GetTablePath(bucketName, namespaceName, tableName)
bucketPath := GetTableBucketPath(bucketName)
var metadata tableMetadataInternal
var bucketPolicy string
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &metadata); err != nil {
return fmt.Errorf("failed to unmarshal table metadata: %w", err)
}
bucketPolicy, err = h.readBucketPolicy(r, client, bucketPath)
return err
})
if err != nil {
return "", err
}
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName)
principal := h.getAccountID(r)
if !CheckPermissionWithContext(operation, principal, metadata.OwnerAccountID, bucketPolicy, tableARN, &PolicyContext{
TableBucketName: bucketName,
Namespace: namespaceName,
TableName: tableName,
IdentityActions: getIdentityActions(r),
DefaultAllow: h.defaultAllowFor(r),
}) {
return "", NewAuthError(operation, principal, "not authorized to "+operation)
}
return tableARN, nil
}
// readBucketPolicy returns the bucket policy, or an empty string when the
// bucket has none.
func (h *S3TablesHandler) readBucketPolicy(r *http.Request, client filer_pb.SeaweedFilerClient, bucketPath string) (string, error) {
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err != nil {
if errors.Is(err, ErrAttributeNotFound) {
return "", nil
}
return "", fmt.Errorf("failed to read bucket policy: %w", err)
}
return string(data), nil
}
// writeMaintenanceError maps the shared failure modes of the maintenance
// handlers onto responses.
func (h *S3TablesHandler) writeMaintenanceError(w http.ResponseWriter, err error, notFoundCode, notFoundMessage string) {
switch {
case errors.Is(err, filer_pb.ErrNotFound):
h.writeError(w, http.StatusNotFound, notFoundCode, notFoundMessage)
case isAuthError(err):
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, err.Error())
case errors.Is(err, ErrConcurrentUpdate):
h.writeError(w, http.StatusConflict, ErrCodeConflict, "maintenance configuration changed concurrently, retry the request")
default:
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, err.Error())
}
}
// validateMaintenanceValue rejects a type that does not belong to the scope and
// settings that do not name that same type.
func validateMaintenanceValue(maintenanceType string, allowed map[string]bool, value *MaintenanceConfigurationValue) error {
if maintenanceType == "" {
return fmt.Errorf("type is required")
}
if !allowed[maintenanceType] {
return fmt.Errorf("unsupported maintenance type %q", maintenanceType)
}
if value == nil {
return fmt.Errorf("value is required")
}
switch value.Status {
case MaintenanceStatusEnabled, MaintenanceStatusDisabled:
case "":
return fmt.Errorf("value.status is required")
default:
return fmt.Errorf("invalid value.status %q, expected %q or %q", value.Status, MaintenanceStatusEnabled, MaintenanceStatusDisabled)
}
if value.Settings != nil && !settingsMatchType(maintenanceType, value.Settings) {
return fmt.Errorf("value.settings must only contain %s", maintenanceType)
}
return validateMaintenanceSettings(value.Settings)
}
// AWS bounds every maintenance setting to a positive 32-bit value. Accepting
// anything else would store a number the worker then ignores or saturates, so
// the configuration read back would not be the one that runs.
const (
maintenanceSettingMin int64 = 1
maintenanceSettingMax int64 = 2147483647
)
func validateMaintenanceSettings(settings *MaintenanceSettings) error {
if settings == nil {
return nil
}
if c := settings.IcebergCompaction; c != nil {
if err := validateCompactionStrategy(c.Strategy); err != nil {
return err
}
if err := validateMaintenanceSetting("targetFileSizeMB", c.TargetFileSizeMB); err != nil {
return err
}
}
if m := settings.IcebergSnapshotManagement; m != nil {
if err := validateMaintenanceSetting("minSnapshotsToKeep", m.MinSnapshotsToKeep); err != nil {
return err
}
if err := validateMaintenanceSetting("maxSnapshotAgeHours", m.MaxSnapshotAgeHours); err != nil {
return err
}
}
if u := settings.IcebergUnreferencedFileRemoval; u != nil {
if err := validateMaintenanceSetting("unreferencedDays", u.UnreferencedDays); err != nil {
return err
}
if err := validateMaintenanceSetting("nonCurrentDays", u.NonCurrentDays); err != nil {
return err
}
}
return nil
}
func validateMaintenanceSetting(name string, value *int64) error {
if value == nil {
return nil
}
if *value < maintenanceSettingMin || *value > maintenanceSettingMax {
return fmt.Errorf("%s must be between %d and %d, got %d", name, maintenanceSettingMin, maintenanceSettingMax, *value)
}
return nil
}
// validateCompactionStrategy rejects a strategy the maintenance worker cannot
// carry out, rather than accepting it and quietly compacting some other way.
func validateCompactionStrategy(strategy string) error {
switch strategy {
case "", CompactionStrategyAuto, CompactionStrategyBinpack, CompactionStrategySort:
return nil
case CompactionStrategyZOrder:
return fmt.Errorf("compaction strategy %q is not supported", strategy)
default:
return fmt.Errorf("invalid compaction strategy %q, expected one of %q, %q, %q",
strategy, CompactionStrategyAuto, CompactionStrategyBinpack, CompactionStrategySort)
}
}
func settingsMatchType(maintenanceType string, settings *MaintenanceSettings) bool {
switch maintenanceType {
case MaintenanceTypeIcebergCompaction:
return settings.IcebergSnapshotManagement == nil && settings.IcebergUnreferencedFileRemoval == nil
case MaintenanceTypeIcebergSnapshotManagement:
return settings.IcebergCompaction == nil && settings.IcebergUnreferencedFileRemoval == nil
case MaintenanceTypeIcebergUnreferencedFileRemoval:
return settings.IcebergCompaction == nil && settings.IcebergSnapshotManagement == nil
}
return false
}

View File

@@ -0,0 +1,476 @@
package s3tables
import (
"encoding/json"
"strings"
"testing"
)
func settingPtr(v int64) *int64 { return &v }
func TestValidateMaintenanceValueAcceptsScopedTypes(t *testing.T) {
err := validateMaintenanceValue(MaintenanceTypeIcebergCompaction, tableMaintenanceTypes, &MaintenanceConfigurationValue{
Status: MaintenanceStatusEnabled,
Settings: &MaintenanceSettings{IcebergCompaction: &IcebergCompactionSettings{TargetFileSizeMB: settingPtr(512)}},
})
if err != nil {
t.Errorf("expected compaction accepted on a table, got %v", err)
}
err = validateMaintenanceValue(MaintenanceTypeIcebergUnreferencedFileRemoval, bucketMaintenanceTypes, &MaintenanceConfigurationValue{
Status: MaintenanceStatusDisabled,
})
if err != nil {
t.Errorf("expected unreferenced file removal accepted on a bucket, got %v", err)
}
}
func TestValidateMaintenanceValueRejectsWrongScope(t *testing.T) {
if err := validateMaintenanceValue(MaintenanceTypeIcebergCompaction, bucketMaintenanceTypes, &MaintenanceConfigurationValue{
Status: MaintenanceStatusEnabled,
}); err == nil {
t.Error("expected compaction rejected on a table bucket")
}
if err := validateMaintenanceValue(MaintenanceTypeIcebergUnreferencedFileRemoval, tableMaintenanceTypes, &MaintenanceConfigurationValue{
Status: MaintenanceStatusEnabled,
}); err == nil {
t.Error("expected unreferenced file removal rejected on a table")
}
}
func TestValidateMaintenanceValueRejectsBadInput(t *testing.T) {
cases := map[string]struct {
maintenanceType string
value *MaintenanceConfigurationValue
}{
"empty type": {"", &MaintenanceConfigurationValue{Status: MaintenanceStatusEnabled}},
"unknown type": {"icebergSomethingElse", &MaintenanceConfigurationValue{Status: MaintenanceStatusEnabled}},
"nil value": {MaintenanceTypeIcebergCompaction, nil},
"no status": {MaintenanceTypeIcebergCompaction, &MaintenanceConfigurationValue{}},
"bad status": {MaintenanceTypeIcebergCompaction, &MaintenanceConfigurationValue{Status: "Enabled"}},
"settings for another type": {MaintenanceTypeIcebergCompaction, &MaintenanceConfigurationValue{
Status: MaintenanceStatusEnabled,
Settings: &MaintenanceSettings{IcebergSnapshotManagement: &IcebergSnapshotManagementSettings{MinSnapshotsToKeep: settingPtr(3)}},
}},
}
for name, tc := range cases {
t.Run(name, func(t *testing.T) {
if err := validateMaintenanceValue(tc.maintenanceType, tableMaintenanceTypes, tc.value); err == nil {
t.Error("expected rejection")
}
})
}
}
// The stored attribute is the wire shape, so Get can hand it straight back.
func TestMaintenanceConfigurationRoundTrip(t *testing.T) {
original := MaintenanceConfiguration{
MaintenanceTypeIcebergCompaction: {
Status: MaintenanceStatusEnabled,
Settings: &MaintenanceSettings{IcebergCompaction: &IcebergCompactionSettings{TargetFileSizeMB: settingPtr(512)}},
},
MaintenanceTypeIcebergSnapshotManagement: {
Status: MaintenanceStatusDisabled,
Settings: &MaintenanceSettings{IcebergSnapshotManagement: &IcebergSnapshotManagementSettings{
MinSnapshotsToKeep: settingPtr(3),
MaxSnapshotAgeHours: settingPtr(120),
}},
},
}
data, err := json.Marshal(original)
if err != nil {
t.Fatalf("marshal: %v", err)
}
var decoded MaintenanceConfiguration
if err := json.Unmarshal(data, &decoded); err != nil {
t.Fatalf("unmarshal: %v", err)
}
compaction := decoded[MaintenanceTypeIcebergCompaction]
if compaction == nil || compaction.Settings == nil || compaction.Settings.IcebergCompaction == nil {
t.Fatalf("expected compaction settings, got %s", data)
}
if got := compaction.Settings.IcebergCompaction.TargetFileSizeMB; got == nil || *got != 512 {
t.Errorf("expected targetFileSizeMB=512, got %v", got)
}
snapshots := decoded[MaintenanceTypeIcebergSnapshotManagement]
if snapshots == nil || snapshots.Status != MaintenanceStatusDisabled {
t.Fatalf("expected snapshot management disabled, got %s", data)
}
if got := snapshots.Settings.IcebergSnapshotManagement.MaxSnapshotAgeHours; got == nil || *got != 120 {
t.Errorf("expected maxSnapshotAgeHours=120, got %v", got)
}
}
// Putting one type must not drop the others already stored.
func TestMaintenanceConfigurationMergeKeepsOtherTypes(t *testing.T) {
stored, err := json.Marshal(MaintenanceConfiguration{
MaintenanceTypeIcebergCompaction: {Status: MaintenanceStatusEnabled},
})
if err != nil {
t.Fatalf("marshal: %v", err)
}
config := MaintenanceConfiguration{}
if err := json.Unmarshal(stored, &config); err != nil {
t.Fatalf("unmarshal: %v", err)
}
config[MaintenanceTypeIcebergSnapshotManagement] = &MaintenanceConfigurationValue{Status: MaintenanceStatusDisabled}
if len(config) != 2 {
t.Errorf("expected both types retained, got %+v", config)
}
if config[MaintenanceTypeIcebergCompaction].Status != MaintenanceStatusEnabled {
t.Error("expected the existing compaction entry preserved")
}
}
func TestParseTableTarget(t *testing.T) {
h := NewS3TablesHandler()
arn := h.generateTableBucketARN(DefaultAccountID, "analytics")
bucketName, namespaceName, tableName, err := h.parseTableTarget(arn, []string{"sales"}, "orders")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if bucketName != "analytics" || namespaceName != "sales" || tableName != "orders" {
t.Errorf("got %q/%q/%q", bucketName, namespaceName, tableName)
}
for name, tc := range map[string]struct {
arn string
namespace []string
table string
}{
"missing arn": {"", []string{"sales"}, "orders"},
"missing namespace": {arn, nil, "orders"},
"missing name": {arn, []string{"sales"}, ""},
"bad arn": {"not-an-arn", []string{"sales"}, "orders"},
} {
t.Run(name, func(t *testing.T) {
if _, _, _, err := h.parseTableTarget(tc.arn, tc.namespace, tc.table); err == nil {
t.Error("expected rejection")
}
})
}
}
func TestBuildJobStatusResponseReportsEveryType(t *testing.T) {
status := buildJobStatusResponse(nil, nil)
if len(status) != 3 {
t.Fatalf("expected all three job types reported, got %v", status)
}
for jobType, value := range status {
if value.Status != MaintenanceJobStatusNotYetRun {
t.Errorf("expected %s not yet run, got %q", jobType, value.Status)
}
}
}
func TestBuildJobStatusResponseDisabledBeatsRecorded(t *testing.T) {
recorded := MaintenanceJobStatus{
MaintenanceTypeIcebergCompaction: {Status: MaintenanceJobStatusSuccessful},
MaintenanceTypeIcebergSnapshotManagement: {Status: MaintenanceJobStatusFailed, FailureMessage: "boom"},
}
config := MaintenanceConfiguration{
MaintenanceTypeIcebergCompaction: {Status: MaintenanceStatusDisabled},
}
status := buildJobStatusResponse(recorded, config)
if status[MaintenanceTypeIcebergCompaction].Status != MaintenanceJobStatusDisabled {
t.Errorf("expected a disabled type reported as disabled, got %q", status[MaintenanceTypeIcebergCompaction].Status)
}
if status[MaintenanceTypeIcebergSnapshotManagement].FailureMessage != "boom" {
t.Error("expected the recorded failure preserved")
}
if status[MaintenanceTypeIcebergUnreferencedFileRemoval].Status != MaintenanceJobStatusNotYetRun {
t.Error("expected an unrun type reported as not yet run")
}
}
// A strategy the worker cannot carry out must be rejected, not accepted and
// silently compacted some other way.
func TestValidateMaintenanceValueCompactionStrategy(t *testing.T) {
for _, strategy := range []string{"", CompactionStrategyAuto, CompactionStrategyBinpack, CompactionStrategySort} {
if err := validateMaintenanceValue(MaintenanceTypeIcebergCompaction, tableMaintenanceTypes, &MaintenanceConfigurationValue{
Status: MaintenanceStatusEnabled,
Settings: &MaintenanceSettings{IcebergCompaction: &IcebergCompactionSettings{Strategy: strategy}},
}); err != nil {
t.Errorf("expected strategy %q accepted, got %v", strategy, err)
}
}
for _, strategy := range []string{CompactionStrategyZOrder, "nonsense"} {
if err := validateMaintenanceValue(MaintenanceTypeIcebergCompaction, tableMaintenanceTypes, &MaintenanceConfigurationValue{
Status: MaintenanceStatusEnabled,
Settings: &MaintenanceSettings{IcebergCompaction: &IcebergCompactionSettings{Strategy: strategy}},
}); err == nil {
t.Errorf("expected strategy %q rejected", strategy)
}
}
}
// Put then Get must return the strategy unchanged.
func TestMaintenanceConfigurationPreservesStrategy(t *testing.T) {
data, err := json.Marshal(MaintenanceConfiguration{
MaintenanceTypeIcebergCompaction: {
Status: MaintenanceStatusEnabled,
Settings: &MaintenanceSettings{IcebergCompaction: &IcebergCompactionSettings{Strategy: CompactionStrategySort, TargetFileSizeMB: settingPtr(512)}},
},
})
if err != nil {
t.Fatalf("marshal: %v", err)
}
var decoded MaintenanceConfiguration
if err := json.Unmarshal(data, &decoded); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if got := decoded[MaintenanceTypeIcebergCompaction].Settings.IcebergCompaction.Strategy; got != CompactionStrategySort {
t.Errorf("expected the strategy preserved, got %q", got)
}
}
// Unreferenced file removal is configured on the bucket, so a bucket-level
// disable has to surface here rather than a stale table status.
func TestBuildJobStatusResponseHonoursBucketDisable(t *testing.T) {
recorded := MaintenanceJobStatus{
MaintenanceTypeIcebergUnreferencedFileRemoval: {Status: MaintenanceJobStatusSuccessful},
}
bucketConfig := MaintenanceConfiguration{
MaintenanceTypeIcebergUnreferencedFileRemoval: {Status: MaintenanceStatusDisabled},
}
status := buildJobStatusResponse(recorded, MergeMaintenanceConfiguration(bucketConfig, nil))
if got := status[MaintenanceTypeIcebergUnreferencedFileRemoval].Status; got != MaintenanceJobStatusDisabled {
t.Errorf("expected Disabled from the bucket configuration, got %q", got)
}
}
func TestMergeMaintenanceConfiguration(t *testing.T) {
bucket := MaintenanceConfiguration{
MaintenanceTypeIcebergUnreferencedFileRemoval: {Status: MaintenanceStatusEnabled},
MaintenanceTypeIcebergCompaction: {Status: MaintenanceStatusEnabled},
}
table := MaintenanceConfiguration{
MaintenanceTypeIcebergCompaction: {Status: MaintenanceStatusDisabled},
}
merged := MergeMaintenanceConfiguration(bucket, table)
if merged[MaintenanceTypeIcebergCompaction].Status != MaintenanceStatusDisabled {
t.Error("expected the table entry to win")
}
if merged[MaintenanceTypeIcebergUnreferencedFileRemoval].Status != MaintenanceStatusEnabled {
t.Error("expected the bucket-only entry retained")
}
if got := MergeMaintenanceConfiguration(nil, table); len(got) != 1 {
t.Errorf("expected the table configuration passed through, got %v", got)
}
if got := MergeMaintenanceConfiguration(bucket, nil); len(got) != 2 {
t.Errorf("expected the bucket configuration passed through, got %v", got)
}
}
// UpdateEntry rewrites the whole entry, so the precondition has to cover every
// attribute this package stores — including the ones absent right now, which a
// concurrent writer can create and this write would then delete.
func TestSnapshotExtendedCoversEveryAttribute(t *testing.T) {
extended := map[string][]byte{
ExtendedKeyMetadata: []byte(`{"a":1}`),
ExtendedKeyMaintenance: []byte(`{"icebergCompaction":{"status":"disabled"}}`),
}
expected := SnapshotExtended(extended, ExtendedKeyMaintenanceStatus)
for key, want := range extended {
if string(expected[key]) != string(want) {
t.Errorf("attribute %s not carried into the precondition", key)
}
}
for _, key := range s3tablesExtendedKeys {
if _, ok := expected[key]; !ok {
t.Errorf("attribute %s missing from the precondition", key)
}
}
// The absent ones are asserted absent, so a concurrent create fails the write.
for _, key := range []string{ExtendedKeyMaintenanceStatus, ExtendedKeyPolicy, ExtendedKeyTags} {
if got, ok := expected[key]; !ok || len(got) != 0 {
t.Errorf("expected %s asserted absent, got %v", key, got)
}
}
// Mutating the snapshot must not disturb the entry it came from.
expected[ExtendedKeyMetadata] = []byte("changed")
if string(extended[ExtendedKeyMetadata]) != `{"a":1}` {
t.Error("expected the source map left alone")
}
}
// The regression this closes: a maintenance configuration that did not exist
// when the writer read the entry must still be asserted, or a first-time
// disable lands between the read and the write and gets erased.
func TestSnapshotExtendedAssertsAbsentMaintenanceKey(t *testing.T) {
expected := SnapshotExtended(map[string][]byte{ExtendedKeyMetadata: []byte(`{}`)})
value, ok := expected[ExtendedKeyMaintenance]
if !ok {
t.Fatal("expected the maintenance key asserted even when absent")
}
if len(value) != 0 {
t.Errorf("expected an absent assertion, got %v", value)
}
}
// AWS bounds every numeric setting to 1..2147483647. Accepting anything else
// would store a value the worker ignores or saturates, so the configuration
// read back would not be the one that runs.
func TestValidateMaintenanceSettingRange(t *testing.T) {
compaction := func(v *int64) error {
return validateMaintenanceValue(MaintenanceTypeIcebergCompaction, tableMaintenanceTypes, &MaintenanceConfigurationValue{
Status: MaintenanceStatusEnabled,
Settings: &MaintenanceSettings{IcebergCompaction: &IcebergCompactionSettings{TargetFileSizeMB: v}},
})
}
if err := compaction(nil); err != nil {
t.Errorf("expected an unset setting accepted, got %v", err)
}
for _, v := range []int64{1, 512, maintenanceSettingMax} {
if err := compaction(settingPtr(v)); err != nil {
t.Errorf("expected %d accepted, got %v", v, err)
}
}
for _, v := range []int64{0, -1, maintenanceSettingMax + 1, 1 << 62} {
if err := compaction(settingPtr(v)); err == nil {
t.Errorf("expected %d rejected", v)
}
}
}
func TestValidateMaintenanceSettingRangeCoversEveryField(t *testing.T) {
cases := map[string]*MaintenanceSettings{
"minSnapshotsToKeep": {IcebergSnapshotManagement: &IcebergSnapshotManagementSettings{MinSnapshotsToKeep: settingPtr(0)}},
"maxSnapshotAgeHours": {IcebergSnapshotManagement: &IcebergSnapshotManagementSettings{MaxSnapshotAgeHours: settingPtr(-5)}},
"unreferencedDays": {IcebergUnreferencedFileRemoval: &IcebergUnreferencedFileRemovalSettings{UnreferencedDays: settingPtr(maintenanceSettingMax + 1)}},
"nonCurrentDays": {IcebergUnreferencedFileRemoval: &IcebergUnreferencedFileRemovalSettings{NonCurrentDays: settingPtr(0)}},
}
for name, settings := range cases {
t.Run(name, func(t *testing.T) {
if err := validateMaintenanceSettings(settings); err == nil {
t.Errorf("expected %s out of range to be rejected", name)
}
})
}
}
// An explicit zero has to be distinguishable from an omitted field, or it is
// silently accepted and then ignored.
func TestMaintenanceSettingsDistinguishZeroFromUnset(t *testing.T) {
var decoded MaintenanceSettings
if err := json.Unmarshal([]byte(`{"icebergCompaction":{"targetFileSizeMB":0}}`), &decoded); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if decoded.IcebergCompaction.TargetFileSizeMB == nil {
t.Fatal("expected an explicit zero to survive decoding")
}
var omitted MaintenanceSettings
if err := json.Unmarshal([]byte(`{"icebergCompaction":{}}`), &omitted); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if omitted.IcebergCompaction.TargetFileSizeMB != nil {
t.Error("expected an omitted field to stay unset")
}
}
// aws-cn and aws-us-gov ARNs are valid and must parse; the emitted ARN has to
// use the partition its region belongs to so it round-trips.
func TestARNPartitionsBeyondCommercial(t *testing.T) {
for _, arn := range []string{
"arn:aws:s3tables:us-east-1:123456789012:bucket/analytics",
"arn:aws-cn:s3tables:cn-north-1:123456789012:bucket/analytics",
"arn:aws-us-gov:s3tables:us-gov-west-1:123456789012:bucket/analytics",
} {
got, err := parseBucketNameFromARN(arn)
if err != nil {
t.Errorf("expected %s to parse, got %v", arn, err)
continue
}
if got != "analytics" {
t.Errorf("expected bucket analytics from %s, got %q", arn, got)
}
}
if _, err := parseBucketNameFromARN("arn:notaws:s3tables:us-east-1:123456789012:bucket/analytics"); err == nil {
t.Error("expected a non-AWS partition to be rejected")
}
}
func TestARNPartitionForRegion(t *testing.T) {
for region, want := range map[string]string{
"us-east-1": "aws",
"eu-west-2": "aws",
"cn-north-1": "aws-cn",
"cn-northwest-1": "aws-cn",
"us-gov-west-1": "aws-us-gov",
"us-iso-east-1": "aws-iso",
"us-isob-east-1": "aws-iso-b",
"eu-isoe-west-1": "aws-iso-e",
"us-isof-south-1": "aws-iso-f",
"eusc-de-east-1": "aws-eusc",
} {
if got := arnPartitionForRegion(region); got != want {
t.Errorf("region %s: expected partition %q, got %q", region, want, got)
}
}
}
// An ARN this handler emits has to carry the partition its region belongs to,
// not just parse back — a commercial ARN returned for a gov region still parses
// but is wrong for IAM matching and for the caller.
func TestGeneratedARNsUseTheRegionPartition(t *testing.T) {
for region, partition := range map[string]string{
"us-east-1": "aws",
"cn-north-1": "aws-cn",
"us-gov-west-1": "aws-us-gov",
} {
h := NewS3TablesHandler()
h.SetRegion(region)
bucketARN := h.generateTableBucketARN(DefaultAccountID, "analytics")
tableARN := h.generateTableARN(DefaultAccountID, "analytics", "sales/orders")
viewARN := h.generateViewARN(DefaultAccountID, "analytics", "sales/v1")
s3ARN := h.generateS3BucketARN("analytics")
for _, arn := range []string{bucketARN, tableARN, viewARN, s3ARN} {
if !strings.HasPrefix(arn, "arn:"+partition+":") {
t.Errorf("region %s: expected partition %q, got %s", region, partition, arn)
}
}
got, err := parseBucketNameFromARN(bucketARN)
if err != nil {
t.Errorf("generated ARN %s does not parse: %v", bucketARN, err)
continue
}
if got != "analytics" {
t.Errorf("expected analytics from %s, got %q", bucketARN, got)
}
bucket, namespace, table, err := parseTableFromARN(tableARN)
if err != nil {
t.Errorf("generated table ARN %s does not parse: %v", tableARN, err)
continue
}
if bucket != "analytics" || namespace != "sales" || table != "orders" {
t.Errorf("unexpected parse of %s: %s/%s/%s", tableARN, bucket, namespace, table)
}
}
}

View File

@@ -0,0 +1,37 @@
package s3tables
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestNameValidationAllowsHyphens(t *testing.T) {
// Hyphen interior is accepted (Iceberg REST clients use names like these).
require.NoError(t, validateNamespacePart("rest-integration-test"))
if _, err := validateTableName("test-create-table"); err != nil {
t.Fatalf("hyphenated table name rejected: %v", err)
}
// A leading or trailing hyphen on a namespace is still rejected (must start
// and end with a letter or digit), as are characters outside the charset.
require.Error(t, validateNamespacePart("-leading"))
require.Error(t, validateNamespacePart("trailing-"))
require.Error(t, validateNamespacePart("Upper"))
// Table names must still start with a letter or digit and stay in charset.
if _, err := validateTableName("-leading"); err == nil {
t.Fatal("table name with leading hyphen should be rejected")
}
if _, err := validateTableName("Upper"); err == nil {
t.Fatal("uppercase table name should be rejected")
}
}
func TestParseTableFromARNAllowsHyphens(t *testing.T) {
bucket, ns, table, err := parseTableFromARN("arn:aws:s3tables:us-east-1:123456789012:bucket/my-bucket/table/rest-integration-test/test-create-table")
require.NoError(t, err)
require.Equal(t, "my-bucket", bucket)
require.Equal(t, "rest-integration-test", ns)
require.Equal(t, "test-create-table", table)
}

View File

@@ -0,0 +1,704 @@
package s3tables
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
// handleCreateNamespace creates a new namespace in a table bucket
func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
glog.Errorf("S3Tables: handleCreateNamespace called")
var req CreateNamespaceRequest
if err := h.readRequestBody(r, &req); err != nil {
glog.Errorf("S3Tables: handleCreateNamespace failed to read request body: %v", err)
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
return fmt.Errorf("tableBucketARN is required")
}
if len(req.Namespace) == 0 {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "namespace is required")
return fmt.Errorf("namespace is required")
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
namespaceName, err := validateNamespace(req.Namespace)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
// Check if table bucket exists
bucketPath := GetTableBucketPath(bucketName)
var bucketMetadata tableBucketMetadata
var bucketPolicy string
var bucketTags map[string]string
ownerAccountID := h.getAccountID(r)
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
if err != nil {
if errors.Is(err, ErrAttributeNotFound) {
dir, name := splitPath(bucketPath)
entryResp, lookupErr := filer_pb.LookupEntry(r.Context(), client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if lookupErr != nil {
return lookupErr
}
if entryResp.Entry == nil || !IsTableBucketEntry(entryResp.Entry) {
return filer_pb.ErrNotFound
}
bucketMetadata = tableBucketMetadata{
Name: bucketName,
CreatedAt: time.Now(),
OwnerAccountID: ownerAccountID,
}
metadataBytes, err := json.Marshal(&bucketMetadata)
if err != nil {
return fmt.Errorf("failed to marshal bucket metadata: %w", err)
}
if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata, metadataBytes); err != nil {
return err
}
} else {
return err
}
} else if err := json.Unmarshal(data, &bucketMetadata); err != nil {
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
}
// Fetch bucket policy if it exists
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err == nil {
bucketPolicy = string(policyData)
} else if !errors.Is(err, ErrAttributeNotFound) {
return fmt.Errorf("failed to fetch bucket policy: %v", err)
}
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
if err != nil {
return err
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table bucket: %v", err))
}
return err
}
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
glog.Infof("S3Tables: CreateNamespace permission check - principal=%s, owner=%s, actions=%v", principal, bucketMetadata.OwnerAccountID, identityActions)
if !CheckPermissionWithContext("CreateNamespace", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
Namespace: namespaceName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
glog.Infof("S3Tables: Permission denied for CreateNamespace - principal=%s, owner=%s", principal, bucketMetadata.OwnerAccountID)
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create namespace in this bucket")
return ErrAccessDenied
}
namespacePath := GetNamespacePath(bucketName, namespaceName)
// Check if namespace already exists
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
_, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
return err
})
if err == nil {
h.writeError(w, http.StatusConflict, ErrCodeNamespaceAlreadyExists, fmt.Sprintf("namespace %s already exists", namespaceName))
return fmt.Errorf("namespace already exists")
} else if !errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check namespace: %v", err))
return err
}
// Create the namespace with bucket owner to maintain consistency
// (authorization above ensures the caller has permission to create in this bucket)
now := time.Now()
metadata := &namespaceMetadata{
Namespace: req.Namespace,
CreatedAt: now,
OwnerAccountID: bucketMetadata.OwnerAccountID,
Properties: req.Properties,
}
metadataBytes, err := json.Marshal(metadata)
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal namespace metadata")
return fmt.Errorf("failed to marshal metadata: %w", err)
}
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
// Create namespace directory
if err := h.createDirectory(r.Context(), client, namespacePath); err != nil {
return err
}
// Set metadata as extended attribute
if err := h.setExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata, metadataBytes); err != nil {
return err
}
return nil
})
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to create namespace")
return err
}
resp := &CreateNamespaceResponse{
Namespace: req.Namespace,
TableBucketARN: req.TableBucketARN,
Properties: req.Properties,
}
h.writeJSON(w, http.StatusOK, resp)
return nil
}
// handleGetNamespace gets details of a namespace
func (h *S3TablesHandler) handleGetNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req GetNamespaceRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
return fmt.Errorf("tableBucketARN is required")
}
namespaceName, err := validateNamespace(req.Namespace)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
namespacePath := GetNamespacePath(bucketName, namespaceName)
bucketPath := GetTableBucketPath(bucketName)
// Get namespace and bucket policy
var metadata namespaceMetadata
var bucketPolicy string
var bucketTags map[string]string
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &metadata); err != nil {
return err
}
// Fetch bucket policy if it exists
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err == nil {
bucketPolicy = string(policyData)
} else if !errors.Is(err, ErrAttributeNotFound) {
return fmt.Errorf("failed to fetch bucket policy: %v", err)
}
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
if err != nil {
return err
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, fmt.Sprintf("namespace %s not found", flattenNamespace(req.Namespace)))
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get namespace: %v", err))
}
return err
}
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, bucketName)
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("GetNamespace", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
Namespace: namespaceName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, "namespace not found")
return ErrAccessDenied
}
resp := &GetNamespaceResponse{
Namespace: metadata.Namespace,
CreatedAt: metadata.CreatedAt,
OwnerAccountID: metadata.OwnerAccountID,
Properties: metadata.Properties,
}
h.writeJSON(w, http.StatusOK, resp)
return nil
}
// handleUpdateNamespace replaces the stored properties of an existing namespace.
// AWS S3 Tables namespaces have no properties; this backs the Iceberg REST
// catalog's namespace-property updates, which carry the merged map.
func (h *S3TablesHandler) handleUpdateNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req UpdateNamespaceRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
return fmt.Errorf("tableBucketARN is required")
}
namespaceName, err := validateNamespace(req.Namespace)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
namespacePath := GetNamespacePath(bucketName, namespaceName)
bucketPath := GetTableBucketPath(bucketName)
var metadata namespaceMetadata
var bucketPolicy string
var bucketTags map[string]string
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &metadata); err != nil {
return err
}
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err == nil {
bucketPolicy = string(policyData)
} else if !errors.Is(err, ErrAttributeNotFound) {
return fmt.Errorf("failed to fetch bucket policy: %v", err)
}
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
if err != nil {
return err
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, fmt.Sprintf("namespace %s not found", flattenNamespace(req.Namespace)))
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get namespace: %v", err))
}
return err
}
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, bucketName)
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("UpdateNamespace", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
Namespace: namespaceName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to update namespace")
return ErrAccessDenied
}
metadata.Properties = req.Properties
metadataBytes, err := json.Marshal(&metadata)
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal namespace metadata")
return fmt.Errorf("failed to marshal metadata: %w", err)
}
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return h.setExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata, metadataBytes)
})
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to update namespace")
return err
}
resp := &UpdateNamespaceResponse{
Namespace: metadata.Namespace,
TableBucketARN: req.TableBucketARN,
Properties: metadata.Properties,
}
h.writeJSON(w, http.StatusOK, resp)
return nil
}
// handleListNamespaces lists all namespaces in a table bucket
func (h *S3TablesHandler) handleListNamespaces(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req ListNamespacesRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
return fmt.Errorf("tableBucketARN is required")
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
maxNamespaces := req.MaxNamespaces
if maxNamespaces <= 0 {
maxNamespaces = 100
}
bucketPath := GetTableBucketPath(bucketName)
// Check permission (check bucket ownership)
var bucketMetadata tableBucketMetadata
var bucketPolicy string
var bucketTags map[string]string
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
}
// Fetch bucket policy if it exists
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err == nil {
bucketPolicy = string(policyData)
} else if !errors.Is(err, ErrAttributeNotFound) {
return fmt.Errorf("failed to fetch bucket policy: %v", err)
}
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
if err != nil {
return err
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list namespaces: %v", err))
}
return err
}
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("ListNamespaces", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
return ErrAccessDenied
}
var namespaces []NamespaceSummary
lastFileName := req.ContinuationToken
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
for len(namespaces) < maxNamespaces {
resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{
Directory: bucketPath,
Limit: uint32(maxNamespaces * 2),
StartFromFileName: lastFileName,
InclusiveStartFrom: lastFileName == "" || lastFileName == req.ContinuationToken,
})
if err != nil {
return err
}
hasMore := false
for {
entry, respErr := resp.Recv()
if respErr != nil {
if respErr == io.EOF {
break
}
return respErr
}
if entry.Entry == nil {
continue
}
// Skip the start item if it was included in the previous page
if len(namespaces) == 0 && req.ContinuationToken != "" && entry.Entry.Name == req.ContinuationToken {
continue
}
hasMore = true
lastFileName = entry.Entry.Name
if !entry.Entry.IsDirectory {
continue
}
// Skip hidden entries
if strings.HasPrefix(entry.Entry.Name, ".") {
continue
}
// Apply prefix filter
if req.Prefix != "" && !strings.HasPrefix(entry.Entry.Name, req.Prefix) {
continue
}
// Read metadata from extended attribute
data, ok := entry.Entry.Extended[ExtendedKeyMetadata]
if !ok {
continue
}
var metadata namespaceMetadata
if err := json.Unmarshal(data, &metadata); err != nil {
continue
}
if metadata.OwnerAccountID != bucketMetadata.OwnerAccountID {
continue
}
namespaces = append(namespaces, NamespaceSummary{
Namespace: metadata.Namespace,
CreatedAt: metadata.CreatedAt,
})
if len(namespaces) >= maxNamespaces {
return nil
}
}
if !hasMore {
break
}
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
namespaces = []NamespaceSummary{}
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list namespaces: %v", err))
return err
}
}
paginationToken := ""
if len(namespaces) >= maxNamespaces {
paginationToken = lastFileName
}
resp := &ListNamespacesResponse{
Namespaces: namespaces,
ContinuationToken: paginationToken,
}
h.writeJSON(w, http.StatusOK, resp)
return nil
}
// handleDeleteNamespace deletes a namespace from a table bucket
func (h *S3TablesHandler) handleDeleteNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req DeleteNamespaceRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
return fmt.Errorf("tableBucketARN is required")
}
namespaceName, err := validateNamespace(req.Namespace)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
namespacePath := GetNamespacePath(bucketName, namespaceName)
bucketPath := GetTableBucketPath(bucketName)
// Check if namespace exists and get metadata for permission check
var metadata namespaceMetadata
var bucketPolicy string
var bucketTags map[string]string
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &metadata); err != nil {
return fmt.Errorf("failed to unmarshal metadata: %w", err)
}
// Fetch bucket policy if it exists
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err == nil {
bucketPolicy = string(policyData)
} else if !errors.Is(err, ErrAttributeNotFound) {
return fmt.Errorf("failed to fetch bucket policy: %v", err)
}
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
if err != nil {
return err
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, fmt.Sprintf("namespace %s not found", flattenNamespace(req.Namespace)))
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get namespace metadata: %v", err))
}
return err
}
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, bucketName)
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("DeleteNamespace", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
Namespace: namespaceName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, "namespace not found")
return ErrAccessDenied
}
// Check if namespace is empty
hasChildren := false
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{
Directory: namespacePath,
Limit: 10,
})
if err != nil {
return err
}
for {
entry, err := resp.Recv()
if err != nil {
if err == io.EOF {
break
}
return err
}
if entry.Entry != nil && !strings.HasPrefix(entry.Entry.Name, ".") {
// Only consider it a child if it's a table (has metadata) or a file.
// Empty directories left by dropped tables or S3 clients should not
// prevent the namespace from being deleted.
_, hasMetadata := entry.Entry.Extended[ExtendedKeyMetadata]
if hasMetadata || !entry.Entry.IsDirectory {
hasChildren = true
break
}
}
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, fmt.Sprintf("namespace %s not found", flattenNamespace(req.Namespace)))
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list namespace entries: %v", err))
}
return err
}
if hasChildren {
h.writeError(w, http.StatusConflict, ErrCodeNamespaceNotEmpty, "namespace is not empty")
return fmt.Errorf("namespace not empty")
}
// Delete the namespace
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return h.deleteDirectory(r.Context(), client, namespacePath)
})
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete namespace")
return err
}
h.writeJSON(w, http.StatusOK, nil)
return nil
}

View File

@@ -0,0 +1,45 @@
package s3tables
import (
"context"
"errors"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
)
// A directory under a table bucket that carries no namespace metadata is not a
// namespace. Reporting that as an internal error, which is what the three
// callers of this did while each testing only for a missing entry, turns a
// client mistake into a 500 and hides it behind "attribute not found".
func TestLoadNamespaceMetadataTreatsMissingAttributeAsAbsent(t *testing.T) {
filer := s3tablestest.Start(t)
handler := NewS3TablesHandler()
client := NewManagerClient(filer.Client)
ctx := context.Background()
filer.Put(TablesPath, "bkt", map[string][]byte{ExtendedKeyTableBucket: []byte("{}")})
// A stray directory where a namespace would live, with no metadata on it.
filer.Put(GetTableBucketPath("bkt"), "stray", nil)
if _, err := handler.loadNamespaceMetadata(ctx, client, "bkt", "stray"); !errors.Is(err, filer_pb.ErrNotFound) {
t.Fatalf("loadNamespaceMetadata on a directory without metadata = %v, want ErrNotFound", err)
}
if _, err := handler.loadNamespaceMetadata(ctx, client, "bkt", "absent"); !errors.Is(err, filer_pb.ErrNotFound) {
t.Fatalf("loadNamespaceMetadata on a missing directory = %v, want ErrNotFound", err)
}
filer.Put(GetTableBucketPath("bkt"), "real", map[string][]byte{
ExtendedKeyMetadata: []byte(`{"namespace":["real"],"ownerAccountId":"000000000000"}`),
})
metadata, err := handler.loadNamespaceMetadata(ctx, client, "bkt", "real")
if err != nil {
t.Fatalf("loadNamespaceMetadata on a real namespace: %v", err)
}
if metadata.OwnerAccountID != "000000000000" {
t.Fatalf("owner = %q, want the stored one", metadata.OwnerAccountID)
}
}

View File

@@ -0,0 +1,936 @@
package s3tables
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
// extractResourceOwnerAndBucket extracts ownership info and bucket name from resource metadata.
// This helper consolidates the repeated pattern used in handleTagResource, handleListTagsForResource,
// and handleUntagResource.
func (h *S3TablesHandler) extractResourceOwnerAndBucket(
data []byte,
resourcePath string,
rType ResourceType,
) (ownerAccountID, bucketName string, err error) {
// Extract bucket name from resource path (format: /buckets/{bucket}/... for both tables and buckets)
parts := strings.Split(strings.Trim(resourcePath, "/"), "/")
if len(parts) >= 2 {
bucketName = parts[1]
}
if rType == ResourceTypeTable {
var meta tableMetadataInternal
if err := json.Unmarshal(data, &meta); err != nil {
return "", "", err
}
ownerAccountID = meta.OwnerAccountID
} else {
var meta tableBucketMetadata
if err := json.Unmarshal(data, &meta); err != nil {
return "", "", err
}
ownerAccountID = meta.OwnerAccountID
}
return ownerAccountID, bucketName, nil
}
// handlePutTableBucketPolicy puts a policy on a table bucket
func (h *S3TablesHandler) handlePutTableBucketPolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req PutTableBucketPolicyRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
return fmt.Errorf("tableBucketARN is required")
}
if req.ResourcePolicy == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourcePolicy is required")
return fmt.Errorf("resourcePolicy is required")
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
// Check if bucket exists and get metadata for ownership check
bucketPath := GetTableBucketPath(bucketName)
var bucketMetadata tableBucketMetadata
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table bucket: %v", err))
}
return err
}
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("PutTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, "", bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to put table bucket policy")
return NewAuthError("PutTableBucketPolicy", principal, "not authorized to put table bucket policy")
}
// Write policy
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy, []byte(req.ResourcePolicy))
})
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to put table bucket policy")
return err
}
h.writeJSON(w, http.StatusOK, nil)
return nil
}
// handleGetTableBucketPolicy gets the policy of a table bucket
func (h *S3TablesHandler) handleGetTableBucketPolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req GetTableBucketPolicyRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
return fmt.Errorf("tableBucketARN is required")
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketPath := GetTableBucketPath(bucketName)
var policy []byte
var bucketMetadata tableBucketMetadata
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
// Get metadata for ownership check
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
}
// Get policy
policy, err = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
return err
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
return err
}
if errors.Is(err, ErrAttributeNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchPolicy, "table bucket policy not found")
return err
}
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get table bucket policy: %v", err))
return err
}
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("GetTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, string(policy), bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table bucket policy")
return NewAuthError("GetTableBucketPolicy", principal, "not authorized to get table bucket policy")
}
resp := &GetTableBucketPolicyResponse{
ResourcePolicy: string(policy),
}
h.writeJSON(w, http.StatusOK, resp)
return nil
}
// handleDeleteTableBucketPolicy deletes the policy of a table bucket
func (h *S3TablesHandler) handleDeleteTableBucketPolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req DeleteTableBucketPolicyRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
return fmt.Errorf("tableBucketARN is required")
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketPath := GetTableBucketPath(bucketName)
// Check if bucket exists and get metadata for ownership check
var bucketMetadata tableBucketMetadata
var bucketPolicy string
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
}
// Fetch bucket policy if it exists
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err != nil {
if !errors.Is(err, ErrAttributeNotFound) {
return fmt.Errorf("failed to read bucket policy: %w", err)
}
// Policy not found is not an error; bucketPolicy remains empty
} else {
bucketPolicy = string(policyData)
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table bucket: %v", err))
}
return err
}
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("DeleteTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table bucket policy")
return NewAuthError("DeleteTableBucketPolicy", principal, "not authorized to delete table bucket policy")
}
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return h.deleteExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
})
if err != nil && !errors.Is(err, ErrAttributeNotFound) {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table bucket policy")
return err
}
h.writeJSON(w, http.StatusOK, nil)
return nil
}
// handlePutTablePolicy puts a policy on a table
func (h *S3TablesHandler) handlePutTablePolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req PutTablePolicyRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" || len(req.Namespace) == 0 || req.Name == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN, namespace, and name are required")
return fmt.Errorf("missing required parameters")
}
namespaceName, err := validateNamespace(req.Namespace)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.ResourcePolicy == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourcePolicy is required")
return fmt.Errorf("resourcePolicy is required")
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
// Check if table exists
tableName, err := validateTableName(req.Name)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
tablePath := GetTablePath(bucketName, namespaceName, tableName)
bucketPath := GetTableBucketPath(bucketName)
var metadata tableMetadataInternal
var bucketPolicy string
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &metadata); err != nil {
return fmt.Errorf("failed to unmarshal table metadata: %w", err)
}
// Fetch bucket policy if it exists
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err != nil {
if !errors.Is(err, ErrAttributeNotFound) {
return fmt.Errorf("failed to read bucket policy: %w", err)
}
// Policy not found is not an error; bucketPolicy remains empty
} else {
bucketPolicy = string(policyData)
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table: %v", err))
}
return err
}
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName)
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("PutTablePolicy", principal, metadata.OwnerAccountID, bucketPolicy, tableARN, &PolicyContext{
TableBucketName: bucketName,
Namespace: namespaceName,
TableName: tableName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to put table policy")
return NewAuthError("PutTablePolicy", principal, "not authorized to put table policy")
}
// Write policy
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return h.setExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyPolicy, []byte(req.ResourcePolicy))
})
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to put table policy")
return err
}
h.writeJSON(w, http.StatusOK, nil)
return nil
}
// handleGetTablePolicy gets the policy of a table
func (h *S3TablesHandler) handleGetTablePolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req GetTablePolicyRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" || len(req.Namespace) == 0 || req.Name == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN, namespace, and name are required")
return fmt.Errorf("missing required parameters")
}
namespaceName, err := validateNamespace(req.Namespace)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
tableName, err := validateTableName(req.Name)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
tablePath := GetTablePath(bucketName, namespaceName, tableName)
bucketPath := GetTableBucketPath(bucketName)
var policy []byte
var metadata tableMetadataInternal
var bucketPolicy string
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
// Get metadata for ownership check
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &metadata); err != nil {
return fmt.Errorf("failed to unmarshal table metadata: %w", err)
}
// Get policy
policy, err = h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyPolicy)
if err != nil {
return err
}
// Fetch bucket policy if it exists
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err != nil {
if !errors.Is(err, ErrAttributeNotFound) {
return fmt.Errorf("failed to read bucket policy: %w", err)
}
// Policy not found is not an error; bucketPolicy remains empty
} else {
bucketPolicy = string(policyData)
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
return err
}
if errors.Is(err, ErrAttributeNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchPolicy, "table policy not found")
return err
}
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get table policy: %v", err))
return err
}
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName)
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("GetTablePolicy", principal, metadata.OwnerAccountID, bucketPolicy, tableARN, &PolicyContext{
TableBucketName: bucketName,
Namespace: namespaceName,
TableName: tableName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table policy")
return NewAuthError("GetTablePolicy", principal, "not authorized to get table policy")
}
resp := &GetTablePolicyResponse{
ResourcePolicy: string(policy),
}
h.writeJSON(w, http.StatusOK, resp)
return nil
}
// handleDeleteTablePolicy deletes the policy of a table
func (h *S3TablesHandler) handleDeleteTablePolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req DeleteTablePolicyRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" || len(req.Namespace) == 0 || req.Name == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN, namespace, and name are required")
return fmt.Errorf("missing required parameters")
}
namespaceName, err := validateNamespace(req.Namespace)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
tableName, err := validateTableName(req.Name)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
tablePath := GetTablePath(bucketName, namespaceName, tableName)
bucketPath := GetTableBucketPath(bucketName)
// Check if table exists
var metadata tableMetadataInternal
var bucketPolicy string
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &metadata); err != nil {
return fmt.Errorf("failed to unmarshal table metadata: %w", err)
}
// Fetch bucket policy if it exists
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err != nil {
if !errors.Is(err, ErrAttributeNotFound) {
return fmt.Errorf("failed to read bucket policy: %w", err)
}
// Policy not found is not an error; bucketPolicy remains empty
} else {
bucketPolicy = string(policyData)
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table: %v", err))
}
return err
}
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName)
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("DeleteTablePolicy", principal, metadata.OwnerAccountID, bucketPolicy, tableARN, &PolicyContext{
TableBucketName: bucketName,
Namespace: namespaceName,
TableName: tableName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table policy")
return NewAuthError("DeleteTablePolicy", principal, "not authorized to delete table policy")
}
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return h.deleteExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyPolicy)
})
if err != nil && !errors.Is(err, ErrAttributeNotFound) {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table policy")
return err
}
h.writeJSON(w, http.StatusOK, nil)
return nil
}
// handleTagResource adds tags to a resource
func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req TagResourceRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.ResourceARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourceArn is required")
return fmt.Errorf("resourceArn is required")
}
if len(req.Tags) == 0 {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tags are required")
return fmt.Errorf("tags are required")
}
// Parse resource ARN to determine if it's a bucket or table
resourcePath, extendedKey, rType, err := h.resolveResourcePath(req.ResourceARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
// Read existing tags and merge, AND check permissions based on metadata ownership
existingTags := make(map[string]string)
var bucketPolicy string
var bucketTags map[string]string
requestTagKeys := mapKeys(req.Tags)
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
// Read metadata for ownership check
data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata)
if err != nil {
return err
}
ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType)
if err != nil {
return err
}
// Fetch bucket policy if we have a bucket name
if bucketName != "" {
bucketPath := GetTableBucketPath(bucketName)
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err != nil {
if !errors.Is(err, ErrAttributeNotFound) {
return fmt.Errorf("failed to read bucket policy: %w", err)
}
// Policy not found is not an error; bucketPolicy remains empty
} else {
bucketPolicy = string(policyData)
}
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
if err != nil {
return err
}
}
// Read existing tags
data, err = h.getExtendedAttribute(r.Context(), client, resourcePath, extendedKey)
if err != nil {
if !errors.Is(err, ErrAttributeNotFound) {
return err
}
} else if err := json.Unmarshal(data, &existingTags); err != nil {
return err
}
resourceARN := req.ResourceARN
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("TagResource", principal, ownerAccountID, bucketPolicy, resourceARN, &PolicyContext{
TableBucketName: bucketName,
TableBucketTags: bucketTags,
RequestTags: req.Tags,
TagKeys: requestTagKeys,
ResourceTags: existingTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
return NewAuthError("TagResource", principal, "not authorized to tag resource")
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
errorCode := ErrCodeNoSuchBucket
if rType == ResourceTypeTable {
errorCode = ErrCodeNoSuchTable
}
h.writeError(w, http.StatusNotFound, errorCode, "resource not found")
} else if isAuthError(err) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, err.Error())
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to read existing tags: %v", err))
}
return err
}
// Merge new tags
for k, v := range req.Tags {
existingTags[k] = v
}
// Write merged tags
tagsBytes, err := json.Marshal(existingTags)
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal tags")
return fmt.Errorf("failed to marshal tags: %w", err)
}
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return h.setExtendedAttribute(r.Context(), client, resourcePath, extendedKey, tagsBytes)
})
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to tag resource")
return err
}
h.writeJSON(w, http.StatusOK, nil)
return nil
}
// handleListTagsForResource lists tags for a resource
func (h *S3TablesHandler) handleListTagsForResource(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req ListTagsForResourceRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.ResourceARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourceArn is required")
return fmt.Errorf("resourceArn is required")
}
resourcePath, extendedKey, rType, err := h.resolveResourcePath(req.ResourceARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
tags := make(map[string]string)
var bucketPolicy string
var bucketTags map[string]string
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
// Read metadata for ownership check
data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata)
if err != nil {
return err
}
ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType)
if err != nil {
return err
}
// Fetch bucket policy if we have a bucket name
if bucketName != "" {
bucketPath := GetTableBucketPath(bucketName)
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err != nil {
if !errors.Is(err, ErrAttributeNotFound) {
return fmt.Errorf("failed to read bucket policy: %w", err)
}
// Policy not found is not an error; bucketPolicy remains empty
} else {
bucketPolicy = string(policyData)
}
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
if err != nil {
return err
}
}
data, err = h.getExtendedAttribute(r.Context(), client, resourcePath, extendedKey)
if err != nil {
if errors.Is(err, ErrAttributeNotFound) {
return nil // No tags is not an error.
}
return err // Propagate other errors.
}
if err := json.Unmarshal(data, &tags); err != nil {
return err
}
resourceARN := req.ResourceARN
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("ListTagsForResource", principal, ownerAccountID, bucketPolicy, resourceARN, &PolicyContext{
TableBucketName: bucketName,
TableBucketTags: bucketTags,
ResourceTags: tags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
return NewAuthError("ListTagsForResource", principal, "not authorized to list tags for resource")
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
errorCode := ErrCodeNoSuchBucket
if rType == ResourceTypeTable {
errorCode = ErrCodeNoSuchTable
}
h.writeError(w, http.StatusNotFound, errorCode, "resource not found")
} else if isAuthError(err) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, err.Error())
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list tags: %v", err))
}
return err
}
resp := &ListTagsForResourceResponse{
Tags: tags,
}
h.writeJSON(w, http.StatusOK, resp)
return nil
}
// handleUntagResource removes tags from a resource
func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req UntagResourceRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.ResourceARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourceArn is required")
return fmt.Errorf("resourceArn is required")
}
if len(req.TagKeys) == 0 {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tagKeys are required")
return fmt.Errorf("tagKeys are required")
}
resourcePath, extendedKey, rType, err := h.resolveResourcePath(req.ResourceARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
// Read existing tags, check permission
tags := make(map[string]string)
var bucketPolicy string
var bucketTags map[string]string
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
// Read metadata for ownership check
data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata)
if err != nil {
return err
}
ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType)
if err != nil {
return err
}
// Fetch bucket policy if we have a bucket name
if bucketName != "" {
bucketPath := GetTableBucketPath(bucketName)
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if err != nil {
if !errors.Is(err, ErrAttributeNotFound) {
return fmt.Errorf("failed to read bucket policy: %w", err)
}
// Policy not found is not an error; bucketPolicy remains empty
} else {
bucketPolicy = string(policyData)
}
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
if err != nil {
return err
}
}
data, err = h.getExtendedAttribute(r.Context(), client, resourcePath, extendedKey)
if err != nil {
if errors.Is(err, ErrAttributeNotFound) {
return nil
}
return err
}
if err := json.Unmarshal(data, &tags); err != nil {
return err
}
resourceARN := req.ResourceARN
principal := h.getAccountID(r)
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("UntagResource", principal, ownerAccountID, bucketPolicy, resourceARN, &PolicyContext{
TableBucketName: bucketName,
TableBucketTags: bucketTags,
TagKeys: req.TagKeys,
ResourceTags: tags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}) {
return NewAuthError("UntagResource", principal, "not authorized to untag resource")
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
errorCode := ErrCodeNoSuchBucket
if rType == ResourceTypeTable {
errorCode = ErrCodeNoSuchTable
}
h.writeError(w, http.StatusNotFound, errorCode, "resource not found")
} else if isAuthError(err) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, err.Error())
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to read existing tags")
}
return err
}
// Remove specified tags
for _, key := range req.TagKeys {
delete(tags, key)
}
// Write updated tags
tagsBytes, err := json.Marshal(tags)
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal tags")
return fmt.Errorf("failed to marshal tags: %w", err)
}
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return h.setExtendedAttribute(r.Context(), client, resourcePath, extendedKey, tagsBytes)
})
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to untag resource")
return err
}
h.writeJSON(w, http.StatusOK, nil)
return nil
}
// resolveResourcePath determines the resource path and extended attribute key from a resource ARN
func (h *S3TablesHandler) resolveResourcePath(resourceARN string) (path string, key string, rType ResourceType, err error) {
// Try parsing as table ARN first
bucketName, namespace, tableName, err := parseTableFromARN(resourceARN)
if err == nil {
return GetTablePath(bucketName, namespace, tableName), ExtendedKeyTags, ResourceTypeTable, nil
}
// Try parsing as bucket ARN
bucketName, err = parseBucketNameFromARN(resourceARN)
if err == nil {
return GetTableBucketPath(bucketName), ExtendedKeyTags, ResourceTypeBucket, nil
}
return "", "", "", fmt.Errorf("invalid resource ARN: %s", resourceARN)
}

View File

@@ -0,0 +1,289 @@
package s3tables
import (
"context"
"encoding/json"
"path"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const renameTestBucket = "renamebkt"
func mustBucketARN(t *testing.T) string {
t.Helper()
arn, err := BuildBucketARN(DefaultRegion, DefaultAccountID, renameTestBucket)
require.NoError(t, err)
return arn
}
// startRenameManager seeds a bucket/namespace/table and returns a trusted Manager.
func startRenameManager(t *testing.T) (*s3tablestest.MemFiler, *Manager) {
t.Helper()
fs := s3tablestest.Start(t)
bucketMeta, _ := json.Marshal(tableBucketMetadata{Name: renameTestBucket, OwnerAccountID: DefaultAccountID})
fs.Put(TablesPath, renameTestBucket, map[string][]byte{
ExtendedKeyTableBucket: []byte("{}"),
ExtendedKeyMetadata: bucketMeta,
})
nsMeta, _ := json.Marshal(namespaceMetadata{Namespace: []string{"ns"}, OwnerAccountID: DefaultAccountID})
fs.Put(GetTableBucketPath(renameTestBucket), "ns", map[string][]byte{ExtendedKeyMetadata: nsMeta})
tableMeta, _ := json.Marshal(tableMetadataInternal{
Name: "t",
Namespace: "ns",
Format: "ICEBERG",
OwnerAccountID: DefaultAccountID,
MetadataVersion: 3,
MetadataLocation: "s3://" + renameTestBucket + "/ns/t/metadata/v3.metadata.json",
})
fs.Put(GetNamespacePath(renameTestBucket, "ns"), "t", map[string][]byte{
ExtendedKeyMetadata: tableMeta,
ExtendedKeyMetadataVersion: []byte("3"),
})
// Physical metadata.json and data files live under the table directory.
tablePath := GetTablePath(renameTestBucket, "ns", "t")
fs.Put(tablePath, "metadata", nil)
fs.Put(tablePath, "data", nil)
fs.Put(path.Join(tablePath, "metadata"), "v3.metadata.json", nil)
m := NewManager()
m.SetTrusted(true)
return fs, m
}
func runRename(t *testing.T, m *Manager, fs *s3tablestest.MemFiler, req *RenameTableRequest) error {
t.Helper()
return m.Execute(context.Background(), NewManagerClient(fs.Client), "RenameTable", req, nil, "")
}
func runGetTable(t *testing.T, m *Manager, fs *s3tablestest.MemFiler, namespace, name string) (*GetTableResponse, error) {
t.Helper()
resp := &GetTableResponse{}
err := m.Execute(context.Background(), NewManagerClient(fs.Client), "GetTable", &GetTableRequest{
TableBucketARN: mustBucketARN(t),
Namespace: []string{namespace},
Name: name,
}, resp, "")
return resp, err
}
func TestRenameTablePreservesData(t *testing.T) {
fs, m := startRenameManager(t)
req := &RenameTableRequest{
TableBucketARN: mustBucketARN(t),
SourceNamespace: []string{"ns"},
SourceName: "t",
DestNamespace: []string{"ns"},
DestName: "t2",
}
require.NoError(t, runRename(t, m, fs, req))
// The source directory and its metadata.json/data children must survive: rename
// is catalog-only and the destination still points at the original location.
srcPath := GetTablePath(renameTestBucket, "ns", "t")
assert.NotNil(t, fs.Get(srcPath, "metadata"), "source metadata dir must survive")
assert.NotNil(t, fs.Get(srcPath, "data"), "source data dir must survive")
assert.NotNil(t, fs.Get(path.Join(srcPath, "metadata"), "v3.metadata.json"), "metadata.json must survive")
// Source catalog xattrs are dropped so the name stops resolving.
src := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
require.NotNil(t, src, "source directory must remain to hold the data children")
_, hasMeta := src.Extended[ExtendedKeyMetadata]
assert.False(t, hasMeta, "source table-metadata xattr must be removed")
_, err := runGetTable(t, m, fs, "ns", "t")
require.Error(t, err)
var s3Err *S3TablesError
require.ErrorAs(t, err, &s3Err)
assert.Equal(t, ErrCodeNoSuchTable, s3Err.Type)
// The destination resolves to the preserved (original) MetadataLocation.
got, err := runGetTable(t, m, fs, "ns", "t2")
require.NoError(t, err)
assert.Equal(t, "t2", got.Name)
assert.Equal(t, "s3://"+renameTestBucket+"/ns/t/metadata/v3.metadata.json", got.MetadataLocation)
dest := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t2")
require.NotNil(t, dest)
assert.Equal(t, []byte("3"), dest.Extended[ExtendedKeyMetadataVersion])
}
func TestRenameTableSourceMissing(t *testing.T) {
fs, m := startRenameManager(t)
err := runRename(t, m, fs, &RenameTableRequest{
TableBucketARN: mustBucketARN(t),
SourceNamespace: []string{"ns"},
SourceName: "ghost",
DestNamespace: []string{"ns"},
DestName: "t2",
})
require.Error(t, err)
var s3Err *S3TablesError
require.ErrorAs(t, err, &s3Err)
assert.Equal(t, ErrCodeNoSuchTable, s3Err.Type)
}
func TestRenameTableDestExists(t *testing.T) {
fs, m := startRenameManager(t)
existing, _ := json.Marshal(tableMetadataInternal{Name: "t2", Namespace: "ns", OwnerAccountID: DefaultAccountID})
fs.Put(GetNamespacePath(renameTestBucket, "ns"), "t2", map[string][]byte{ExtendedKeyMetadata: existing})
err := runRename(t, m, fs, &RenameTableRequest{
TableBucketARN: mustBucketARN(t),
SourceNamespace: []string{"ns"},
SourceName: "t",
DestNamespace: []string{"ns"},
DestName: "t2",
})
require.Error(t, err)
var s3Err *S3TablesError
require.ErrorAs(t, err, &s3Err)
assert.Equal(t, ErrCodeTableAlreadyExists, s3Err.Type)
assert.NotNil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t"), "source must be untouched on conflict")
}
func TestRenameTableDestNamespaceMissing(t *testing.T) {
fs, m := startRenameManager(t)
err := runRename(t, m, fs, &RenameTableRequest{
TableBucketARN: mustBucketARN(t),
SourceNamespace: []string{"ns"},
SourceName: "t",
DestNamespace: []string{"other"},
DestName: "t2",
})
require.Error(t, err)
var s3Err *S3TablesError
require.ErrorAs(t, err, &s3Err)
assert.Equal(t, ErrCodeNoSuchNamespace, s3Err.Type)
assert.NotNil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t"), "source must be untouched")
}
// A principal allowed to rename the source must still be denied when it cannot
// create a table in the destination namespace.
func TestRenameTableDestNamespaceUnauthorized(t *testing.T) {
fs, m := startRenameManager(t)
m.SetTrusted(false)
m.SetDefaultAllow(false)
// "mover" may rename the source table but holds no rights on "dest".
srcPolicy, _ := json.Marshal(map[string]interface{}{
"Version": "2012-10-17",
"Statement": []map[string]interface{}{{
"Effect": "Allow",
"Principal": "mover",
"Action": "s3tables:RenameTable",
"Resource": "*",
}},
})
srcEntry := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
require.NotNil(t, srcEntry)
srcEntry.Extended[ExtendedKeyPolicy] = srcPolicy
destNsMeta, _ := json.Marshal(namespaceMetadata{Namespace: []string{"dest"}, OwnerAccountID: DefaultAccountID})
fs.Put(GetTableBucketPath(renameTestBucket), "dest", map[string][]byte{ExtendedKeyMetadata: destNsMeta})
mover := &testIdentity{Name: "mover", Account: &testIdentityAccount{Id: "mover"}}
ctx := s3_constants.SetIdentityInContext(context.Background(), mover)
err := m.Execute(ctx, NewManagerClient(fs.Client), "RenameTable", &RenameTableRequest{
TableBucketARN: mustBucketARN(t),
SourceNamespace: []string{"ns"},
SourceName: "t",
DestNamespace: []string{"dest"},
DestName: "t2",
}, nil, "mover")
require.Error(t, err)
var s3Err *S3TablesError
require.ErrorAs(t, err, &s3Err)
assert.Equal(t, ErrCodeAccessDenied, s3Err.Type)
assert.NotNil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t"), "source must be untouched")
assert.Nil(t, fs.Get(GetNamespacePath(renameTestBucket, "dest"), "t2"), "destination must not be written")
}
func TestRenameTableInvalidName(t *testing.T) {
fs, m := startRenameManager(t)
err := runRename(t, m, fs, &RenameTableRequest{
TableBucketARN: mustBucketARN(t),
SourceNamespace: []string{"ns"},
SourceName: "t",
DestNamespace: []string{"ns"},
DestName: "Bad/Name",
})
require.Error(t, err)
var s3Err *S3TablesError
require.ErrorAs(t, err, &s3Err)
assert.Equal(t, ErrCodeInvalidRequest, s3Err.Type)
}
// A disabled maintenance configuration has to move with the table. Leaving it
// behind re-enables maintenance under the new name, and leaks the old settings
// onto whatever is created at the old one.
func TestRenameTableCarriesMaintenanceConfiguration(t *testing.T) {
fs, m := startRenameManager(t)
src := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
require.NotNil(t, src)
config := []byte(`{"icebergSnapshotManagement":{"status":"disabled"}}`)
status := []byte(`{"icebergCompaction":{"status":"Successful"}}`)
src.Extended[ExtendedKeyMaintenance] = config
src.Extended[ExtendedKeyMaintenanceStatus] = status
require.NoError(t, runRename(t, m, fs, &RenameTableRequest{
TableBucketARN: mustBucketARN(t),
SourceNamespace: []string{"ns"},
SourceName: "t",
DestNamespace: []string{"ns"},
DestName: "t2",
}))
dest := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t2")
require.NotNil(t, dest)
assert.Equal(t, config, dest.Extended[ExtendedKeyMaintenance], "the disable must move with the table")
assert.Equal(t, status, dest.Extended[ExtendedKeyMaintenanceStatus])
// And must not linger on the old name, where a reused name would inherit it.
moved := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
require.NotNil(t, moved)
_, hasConfig := moved.Extended[ExtendedKeyMaintenance]
assert.False(t, hasConfig, "source maintenance configuration must be cleared")
_, hasStatus := moved.Extended[ExtendedKeyMaintenanceStatus]
assert.False(t, hasStatus, "source maintenance status must be cleared")
}
// A maintenance configuration written after the rename copied the source must
// not be deleted by the source cleanup: it would vanish without ever reaching
// the destination.
func TestRenameTableRejectsConcurrentMaintenanceWrite(t *testing.T) {
fs, _ := startRenameManager(t)
src := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
require.NotNil(t, src)
copied := []byte(`{"icebergCompaction":{"status":"enabled"}}`)
src.Extended[ExtendedKeyMaintenance] = copied
// Stand in for the Put that lands between the copy and the cleanup.
landedLate := []byte(`{"icebergSnapshotManagement":{"status":"disabled"}}`)
expected := map[string][]byte{ExtendedKeyMaintenance: copied}
src.Extended[ExtendedKeyMaintenance] = landedLate
h := NewS3TablesHandler()
err := NewManagerClient(fs.Client).WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return h.removeExtendedAttributesIf(context.Background(),
client, GetTablePath(renameTestBucket, "ns", "t"), expected, renamedTableAttributes...)
})
require.Error(t, err, "cleanup must refuse to delete a configuration written after the copy")
assert.ErrorIs(t, err, ErrConcurrentUpdate)
assert.Equal(t, landedLate, src.Extended[ExtendedKeyMaintenance], "the late write must survive")
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,25 @@
package s3tables
import "testing"
func TestMetadataVersionFromLocation(t *testing.T) {
cases := []struct {
location string
want int
}{
{"s3://bucket/ns/tbl/metadata/v1.metadata.json", 1},
{"s3://bucket/ns/tbl/metadata/v7.metadata.json", 7},
{"v42.metadata.json", 42},
{"s3://bucket/ns/tbl/metadata/00003-9f1c2b3a-4d5e-6f70-8192-a3b4c5d6e7f8.metadata.json", 3},
{"00012-abcdef.metadata.json", 12},
{"s3://bucket/ns/tbl/metadata/00000-abc.metadata.json", 1},
{"s3://bucket/ns/tbl/metadata/v0.metadata.json", 1},
{"s3://bucket/ns/tbl/metadata/garbage.metadata.json", 1},
{"", 1},
}
for _, c := range cases {
if got := metadataVersionFromLocation(c.location); got != c.want {
t.Errorf("metadataVersionFromLocation(%q) = %d, want %d", c.location, got, c.want)
}
}
}

View File

@@ -0,0 +1,98 @@
package s3tables
import (
"context"
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func updateTableRequest(t *testing.T, metadataLocation string, version int) *UpdateTableRequest {
t.Helper()
return &UpdateTableRequest{
TableBucketARN: mustBucketARN(t),
Namespace: []string{"ns"},
Name: "t",
MetadataVersion: version,
MetadataLocation: metadataLocation,
}
}
// A second commit that reads the same base must not overwrite the first one's
// metadata pointer: the snapshot the first commit published would be lost.
func TestUpdateTableRejectsLostUpdate(t *testing.T) {
fs, m := startRenameManager(t)
winnerLocation := "s3://" + renameTestBucket + "/ns/t/metadata/v4-winner.metadata.json"
fs.BeforeUpdate = func() {
winner, err := json.Marshal(tableMetadataInternal{
Name: "t",
Namespace: "ns",
Format: "ICEBERG",
OwnerAccountID: DefaultAccountID,
MetadataVersion: 4,
MetadataLocation: winnerLocation,
VersionToken: generateVersionToken(),
})
require.NoError(t, err)
entry := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
require.NotNil(t, entry)
entry.Extended[ExtendedKeyMetadata] = winner
}
loserLocation := "s3://" + renameTestBucket + "/ns/t/metadata/v4-loser.metadata.json"
err := m.Execute(context.Background(), NewManagerClient(fs.Client), "UpdateTable",
updateTableRequest(t, loserLocation, 4), nil, "")
require.Error(t, err)
var s3Err *S3TablesError
require.ErrorAs(t, err, &s3Err)
assert.Equal(t, ErrCodeConflict, s3Err.Type)
got, err := runGetTable(t, m, fs, "ns", "t")
require.NoError(t, err)
assert.Equal(t, winnerLocation, got.MetadataLocation, "the first commit must survive")
}
// A policy written between the authorization check and the write decides
// whether this caller may still commit, so the commit is rejected rather than
// applied on a decision that has been overtaken - and the policy itself must
// survive, not be reverted by the commit's stale copy of the entry.
func TestUpdateTableRejectsWhenTheAuthorizingPolicyChanges(t *testing.T) {
fs, m := startRenameManager(t)
policy := []byte(`{"Version":"2012-10-17","Statement":[]}`)
fs.BeforeUpdate = func() {
entry := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
require.NotNil(t, entry)
entry.Extended[ExtendedKeyPolicy] = policy
}
location := "s3://" + renameTestBucket + "/ns/t/metadata/v4.metadata.json"
err := m.Execute(context.Background(), NewManagerClient(fs.Client), "UpdateTable",
updateTableRequest(t, location, 4), nil, "")
require.Error(t, err)
var s3Err *S3TablesError
require.ErrorAs(t, err, &s3Err)
assert.Equal(t, ErrCodeConflict, s3Err.Type)
entry := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
require.NotNil(t, entry)
assert.Equal(t, policy, entry.Extended[ExtendedKeyPolicy], "the concurrent policy write must survive")
}
func TestUpdateTableAppliesWithoutContention(t *testing.T) {
fs, m := startRenameManager(t)
location := "s3://" + renameTestBucket + "/ns/t/metadata/v4.metadata.json"
require.NoError(t, m.Execute(context.Background(), NewManagerClient(fs.Client), "UpdateTable",
updateTableRequest(t, location, 4), nil, ""))
got, err := runGetTable(t, m, fs, "ns", "t")
require.NoError(t, err)
assert.Equal(t, location, got.MetadataLocation)
assert.Equal(t, 4, got.MetadataVersion)
}

View File

@@ -0,0 +1,659 @@
package s3tables
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
// handleCreateView creates a new view in a namespace. Views share the table
// storage layout but are tagged ExtendedKeyEntryType="view". Names are unique
// across tables and views in a namespace.
func (h *S3TablesHandler) handleCreateView(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req CreateViewRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
return fmt.Errorf("tableBucketARN is required")
}
namespaceName, err := validateNamespace(req.Namespace)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.Name == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "name is required")
return fmt.Errorf("name is required")
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
viewName, err := validateTableName(req.Name)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
// Check if namespace exists
namespacePath := GetNamespacePath(bucketName, namespaceName)
namespaceMetadata, err := h.loadNamespaceMetadata(r.Context(), filerClient, bucketName, namespaceName)
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, fmt.Sprintf("namespace %s not found", namespaceName))
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check namespace: %v", err))
}
return err
}
accountID := h.getAccountID(r)
bucketPath := GetTableBucketPath(bucketName)
namespacePolicy, bucketPolicy, bucketTags, bucketMetadata, err := h.loadAuthContext(r, filerClient, namespacePath, bucketPath)
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to fetch policies: %v", err))
return err
}
// A view is Iceberg metadata, so it belongs only in a bucket that holds
// Iceberg tables.
if bucketMetadata.Format != "" && bucketMetadata.Format != FormatIceberg {
message := fmt.Sprintf("table bucket %s holds %s tables and cannot hold views", bucketName, bucketMetadata.Format)
h.writeError(w, http.StatusConflict, ErrCodeConflict, message)
return fmt.Errorf("%s", message)
}
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
if !h.authorizeViewOp(r, "CreateView", accountID, namespaceMetadata.OwnerAccountID, bucketMetadata.OwnerAccountID, namespacePolicy, bucketPolicy, bucketARN, bucketName, namespaceName, viewName, bucketTags) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create view in this namespace")
return ErrAccessDenied
}
viewPath := GetTablePath(bucketName, namespaceName, viewName)
// Reject if a table or view already exists at this name.
var existingMetadata tableMetadataInternal
var existingIsTable bool
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
entry, err := h.lookupEntry(r.Context(), client, viewPath)
if err != nil {
return err
}
if EntryType(entry.Extended) != EntryTypeView {
existingIsTable = true
return nil
}
data, ok := entry.Extended[ExtendedKeyMetadata]
if !ok {
return fmt.Errorf("%w: %s", ErrAttributeNotFound, ExtendedKeyMetadata)
}
return json.Unmarshal(data, &existingMetadata)
})
if err == nil {
if existingIsTable {
h.writeError(w, http.StatusConflict, ErrCodeViewAlreadyExists, fmt.Sprintf("a table named %s already exists", viewName))
return fmt.Errorf("table name conflict: %s", viewName)
}
viewARN := h.generateViewARN(existingMetadata.OwnerAccountID, bucketName, namespaceName+"/"+viewName)
h.writeJSON(w, http.StatusOK, &CreateViewResponse{
ViewARN: viewARN,
VersionToken: existingMetadata.VersionToken,
MetadataLocation: existingMetadata.MetadataLocation,
})
return nil
} else if !errors.Is(err, filer_pb.ErrNotFound) && !errors.Is(err, ErrAttributeNotFound) {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check view: %v", err))
return err
}
if err := ValidateMetadataLocation(req.MetadataLocation, bucketName); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
now := time.Now()
versionToken := generateVersionToken()
metadata := &tableMetadataInternal{
Name: viewName,
Namespace: namespaceName,
Format: FormatIceberg,
CreatedAt: now,
ModifiedAt: now,
OwnerAccountID: namespaceMetadata.OwnerAccountID,
VersionToken: versionToken,
MetadataVersion: max(req.MetadataVersion, 1),
MetadataLocation: req.MetadataLocation,
Metadata: req.Metadata,
}
metadataBytes, err := json.Marshal(metadata)
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal view metadata")
return err
}
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
if err := h.ensureDirectory(r.Context(), client, viewPath); err != nil {
return err
}
return h.setExtendedAttributes(r.Context(), client, viewPath, map[string][]byte{
ExtendedKeyMetadata: metadataBytes,
ExtendedKeyEntryType: []byte(EntryTypeView),
})
})
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to create view")
return err
}
viewARN := h.generateViewARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+viewName)
h.writeJSON(w, http.StatusOK, &CreateViewResponse{
ViewARN: viewARN,
VersionToken: versionToken,
MetadataLocation: metadata.MetadataLocation,
})
return nil
}
// handleGetView returns the metadata pointer for a view.
func (h *S3TablesHandler) handleGetView(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req GetViewRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketName, namespace, viewName, err := h.parseViewRef(req.TableBucketARN, req.Namespace, req.Name)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
viewPath := GetTablePath(bucketName, namespace, viewName)
var metadata tableMetadataInternal
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
entry, err := h.lookupEntry(r.Context(), client, viewPath)
if err != nil {
return err
}
if EntryType(entry.Extended) != EntryTypeView {
return filer_pb.ErrNotFound
}
data, ok := entry.Extended[ExtendedKeyMetadata]
if !ok {
return fmt.Errorf("%w: %s", ErrAttributeNotFound, ExtendedKeyMetadata)
}
return json.Unmarshal(data, &metadata)
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchView, fmt.Sprintf("view %s not found", viewName))
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get view: %v", err))
}
return err
}
accountID := h.getAccountID(r)
viewPolicy, bucketPolicy, bucketTags, bucketMetadata, err := h.loadAuthContext(r, filerClient, viewPath, GetTableBucketPath(bucketName))
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to fetch policies: %v", err))
return err
}
viewARN := h.generateViewARN(metadata.OwnerAccountID, bucketName, namespace+"/"+viewName)
if !h.authorizeViewOp(r, "GetView", accountID, metadata.OwnerAccountID, bucketMetadata.OwnerAccountID, viewPolicy, bucketPolicy, viewARN, bucketName, namespace, viewName, bucketTags) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchView, fmt.Sprintf("view %s not found", viewName))
return ErrAccessDenied
}
h.writeJSON(w, http.StatusOK, &GetViewResponse{
Name: metadata.Name,
ViewARN: viewARN,
Namespace: expandNamespace(metadata.Namespace),
CreatedAt: metadata.CreatedAt,
ModifiedAt: metadata.ModifiedAt,
OwnerAccountID: metadata.OwnerAccountID,
MetadataLocation: metadata.MetadataLocation,
MetadataVersion: metadata.MetadataVersion,
VersionToken: metadata.VersionToken,
Metadata: metadata.Metadata,
})
return nil
}
// handleListViews lists views in a namespace, excluding table entries.
func (h *S3TablesHandler) handleListViews(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req ListViewsRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if req.TableBucketARN == "" {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
return fmt.Errorf("tableBucketARN is required")
}
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
maxViews := req.MaxViews
if maxViews <= 0 {
maxViews = 100
}
const maxViewsLimit = 1000
if maxViews > maxViewsLimit {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "MaxViews exceeds maximum allowed value")
return fmt.Errorf("invalid maxViews value: %d", maxViews)
}
namespaceName, err := validateNamespace(req.Namespace)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
var views []ViewSummary
var paginationToken string
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
namespacePath := GetNamespacePath(bucketName, namespaceName)
bucketPath := GetTableBucketPath(bucketName)
var nsMeta namespaceMetadata
data, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &nsMeta); err != nil {
return err
}
namespacePolicy, bucketPolicy, bucketTags, bucketMeta, err := h.loadAuthContext(r, &singleClient{client}, namespacePath, bucketPath)
if err != nil {
return err
}
bucketARN := h.generateTableBucketARN(bucketMeta.OwnerAccountID, bucketName)
if !h.authorizeViewOp(r, "ListViews", h.getAccountID(r), nsMeta.OwnerAccountID, bucketMeta.OwnerAccountID, namespacePolicy, bucketPolicy, bucketARN, bucketName, namespaceName, "", bucketTags) {
return ErrAccessDenied
}
views, paginationToken, err = h.listViewsInNamespace(r, client, bucketName, namespaceName, req.Prefix, req.ContinuationToken, maxViews)
return err
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
views = []ViewSummary{}
paginationToken = ""
} else if isAuthError(err) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "Access Denied")
return err
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list views: %v", err))
return err
}
}
h.writeJSON(w, http.StatusOK, &ListViewsResponse{
Views: views,
ContinuationToken: paginationToken,
})
return nil
}
func (h *S3TablesHandler) listViewsInNamespace(r *http.Request, client filer_pb.SeaweedFilerClient, bucketName, namespaceName, prefix, continuationToken string, maxViews int) ([]ViewSummary, string, error) {
namespacePath := GetNamespacePath(bucketName, namespaceName)
var views []ViewSummary
lastFileName := continuationToken
ctx := r.Context()
for len(views) < maxViews {
resp, err := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
Directory: namespacePath,
Limit: uint32(maxViews * 2),
StartFromFileName: lastFileName,
InclusiveStartFrom: lastFileName == "" || lastFileName == continuationToken,
})
if err != nil {
return nil, "", err
}
hasMore := false
for {
entry, respErr := resp.Recv()
if respErr != nil {
if respErr == io.EOF {
break
}
return nil, "", respErr
}
if entry.Entry == nil {
continue
}
if len(views) == 0 && continuationToken != "" && entry.Entry.Name == continuationToken {
continue
}
hasMore = true
lastFileName = entry.Entry.Name
if !entry.Entry.IsDirectory || strings.HasPrefix(entry.Entry.Name, ".") {
continue
}
if prefix != "" && !strings.HasPrefix(entry.Entry.Name, prefix) {
continue
}
// Only include view entries; skip tables and untagged entries.
if EntryType(entry.Entry.Extended) != EntryTypeView {
continue
}
data, ok := entry.Entry.Extended[ExtendedKeyMetadata]
if !ok {
continue
}
var metadata tableMetadataInternal
if err := json.Unmarshal(data, &metadata); err != nil {
continue
}
views = append(views, ViewSummary{
Name: entry.Entry.Name,
ViewARN: h.generateViewARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+entry.Entry.Name),
Namespace: expandNamespace(namespaceName),
CreatedAt: metadata.CreatedAt,
ModifiedAt: metadata.ModifiedAt,
})
if len(views) >= maxViews {
return views, lastFileName, nil
}
}
if !hasMore {
break
}
}
if len(views) < maxViews {
lastFileName = ""
}
return views, lastFileName, nil
}
// handleUpdateView replaces the metadata pointer for a view, mirroring UpdateTable.
func (h *S3TablesHandler) handleUpdateView(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req UpdateViewRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketName, namespaceName, viewName, err := h.parseViewRef(req.TableBucketARN, req.Namespace, req.Name)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
if err := ValidateMetadataLocation(req.MetadataLocation, bucketName); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
viewPath := GetTablePath(bucketName, namespaceName, viewName)
var metadata tableMetadataInternal
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
entry, err := h.lookupEntry(r.Context(), client, viewPath)
if err != nil {
return err
}
if EntryType(entry.Extended) != EntryTypeView {
return filer_pb.ErrNotFound
}
data, ok := entry.Extended[ExtendedKeyMetadata]
if !ok {
return fmt.Errorf("%w: %s", ErrAttributeNotFound, ExtendedKeyMetadata)
}
return json.Unmarshal(data, &metadata)
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchView, "view not found")
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, err.Error())
}
return err
}
accountID := h.getAccountID(r)
viewPolicy, bucketPolicy, bucketTags, bucketMetadata, err := h.loadAuthContext(r, filerClient, viewPath, GetTableBucketPath(bucketName))
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, err.Error())
return err
}
viewARN := h.generateViewARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+viewName)
if !h.authorizeViewOp(r, "UpdateView", accountID, metadata.OwnerAccountID, bucketMetadata.OwnerAccountID, viewPolicy, bucketPolicy, viewARN, bucketName, namespaceName, viewName, bucketTags) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to update view")
return NewAuthError("UpdateView", accountID, "not authorized to update view")
}
if req.VersionToken != "" && req.VersionToken != metadata.VersionToken {
h.writeError(w, http.StatusConflict, ErrCodeConflict, "Version token mismatch")
return ErrVersionTokenMismatch
}
if req.Metadata != nil {
if metadata.Metadata == nil {
metadata.Metadata = &TableMetadata{}
}
if req.Metadata.Iceberg != nil {
if metadata.Metadata.Iceberg == nil {
metadata.Metadata.Iceberg = &IcebergMetadata{}
}
if req.Metadata.Iceberg.TableUUID != "" {
metadata.Metadata.Iceberg.TableUUID = req.Metadata.Iceberg.TableUUID
}
}
if len(req.Metadata.FullMetadata) > 0 {
metadata.Metadata.FullMetadata = req.Metadata.FullMetadata
}
}
if req.MetadataLocation != "" {
metadata.MetadataLocation = req.MetadataLocation
}
if req.MetadataVersion > 0 {
metadata.MetadataVersion = req.MetadataVersion
} else if metadata.MetadataVersion == 0 {
metadata.MetadataVersion = 1
}
metadata.ModifiedAt = time.Now()
metadata.VersionToken = generateVersionToken()
metadataBytes, err := json.Marshal(metadata)
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal metadata")
return err
}
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return h.setExtendedAttribute(r.Context(), client, viewPath, ExtendedKeyMetadata, metadataBytes)
})
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to update metadata")
return err
}
h.writeJSON(w, http.StatusOK, &UpdateViewResponse{
ViewARN: viewARN,
MetadataLocation: metadata.MetadataLocation,
VersionToken: metadata.VersionToken,
})
return nil
}
// handleDeleteView deletes a view.
func (h *S3TablesHandler) handleDeleteView(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
var req DeleteViewRequest
if err := h.readRequestBody(r, &req); err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
bucketName, namespaceName, viewName, err := h.parseViewRef(req.TableBucketARN, req.Namespace, req.Name)
if err != nil {
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
return err
}
viewPath := GetTablePath(bucketName, namespaceName, viewName)
var metadata tableMetadataInternal
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
entry, err := h.lookupEntry(r.Context(), client, viewPath)
if err != nil {
return err
}
if EntryType(entry.Extended) != EntryTypeView {
return filer_pb.ErrNotFound
}
data, ok := entry.Extended[ExtendedKeyMetadata]
if !ok {
return fmt.Errorf("%w: %s", ErrAttributeNotFound, ExtendedKeyMetadata)
}
if err := json.Unmarshal(data, &metadata); err != nil {
return err
}
if req.VersionToken != "" && metadata.VersionToken != req.VersionToken {
return ErrVersionTokenMismatch
}
return nil
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchView, fmt.Sprintf("view %s not found", viewName))
} else if errors.Is(err, ErrVersionTokenMismatch) {
h.writeError(w, http.StatusConflict, ErrCodeConflict, "version token mismatch")
} else {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check view: %v", err))
}
return err
}
accountID := h.getAccountID(r)
viewPolicy, bucketPolicy, bucketTags, bucketMetadata, err := h.loadAuthContext(r, filerClient, viewPath, GetTableBucketPath(bucketName))
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, err.Error())
return err
}
viewARN := h.generateViewARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+viewName)
if !h.authorizeViewOp(r, "DeleteView", accountID, metadata.OwnerAccountID, bucketMetadata.OwnerAccountID, viewPolicy, bucketPolicy, viewARN, bucketName, namespaceName, viewName, bucketTags) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete view")
return NewAuthError("DeleteView", accountID, "not authorized to delete view")
}
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return h.deleteDirectory(r.Context(), client, viewPath)
})
if err != nil {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete view")
return err
}
h.writeJSON(w, http.StatusOK, nil)
return nil
}
// parseViewRef validates and resolves a (bucketARN, namespace, name) triple.
func (h *S3TablesHandler) parseViewRef(bucketARN string, namespace []string, name string) (bucketName, namespaceName, viewName string, err error) {
if bucketARN == "" || len(namespace) == 0 || name == "" {
return "", "", "", fmt.Errorf("tableBucketARN, namespace, and name are required")
}
if bucketName, err = parseBucketNameFromARN(bucketARN); err != nil {
return "", "", "", err
}
if namespaceName, err = validateNamespace(namespace); err != nil {
return "", "", "", err
}
if viewName, err = validateTableName(name); err != nil {
return "", "", "", err
}
return bucketName, namespaceName, viewName, nil
}
// loadAuthContext fetches the resource policy, bucket policy, bucket tags and
// bucket metadata used to authorize a view operation.
func (h *S3TablesHandler) loadAuthContext(r *http.Request, filerClient FilerClient, resourcePath, bucketPath string) (resourcePolicy, bucketPolicy string, bucketTags map[string]string, bucketMetadata tableBucketMetadata, err error) {
bucketTags = map[string]string{}
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, e := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
if e == nil {
if e := json.Unmarshal(data, &bucketMetadata); e != nil {
return fmt.Errorf("failed to unmarshal bucket metadata: %w", e)
}
} else if !errors.Is(e, ErrAttributeNotFound) {
return fmt.Errorf("failed to fetch bucket metadata: %w", e)
}
policyData, e := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyPolicy)
if e == nil {
resourcePolicy = string(policyData)
} else if !errors.Is(e, ErrAttributeNotFound) {
return fmt.Errorf("failed to fetch resource policy: %w", e)
}
policyData, e = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
if e == nil {
bucketPolicy = string(policyData)
} else if !errors.Is(e, ErrAttributeNotFound) {
return fmt.Errorf("failed to fetch bucket policy: %w", e)
}
if tags, e := h.readTags(r.Context(), client, bucketPath); e != nil {
return e
} else if tags != nil {
bucketTags = tags
}
return nil
})
return resourcePolicy, bucketPolicy, bucketTags, bucketMetadata, err
}
// authorizeViewOp evaluates the resource and bucket policies for a view operation.
func (h *S3TablesHandler) authorizeViewOp(r *http.Request, operation, principal, resourceOwner, bucketOwner, resourcePolicy, bucketPolicy, resourceARN, bucketName, namespace, viewName string, bucketTags map[string]string) bool {
identityActions := getIdentityActions(r)
ctx := &PolicyContext{
TableBucketName: bucketName,
Namespace: namespace,
TableName: viewName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllowFor(r),
}
resourceAllowed := CheckPermissionWithContext(operation, principal, resourceOwner, resourcePolicy, resourceARN, ctx)
bucketARN := h.generateTableBucketARN(bucketOwner, bucketName)
bucketAllowed := CheckPermissionWithContext(operation, principal, bucketOwner, bucketPolicy, bucketARN, ctx)
return resourceAllowed || bucketAllowed
}
// singleClient adapts a bound filer client to the FilerClient interface so the
// shared helpers can be reused inside an existing WithFilerClient closure.
type singleClient struct {
client filer_pb.SeaweedFilerClient
}
func (s *singleClient) WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
return fn(s.client)
}

View File

@@ -0,0 +1,136 @@
package s3tables
import (
"context"
"encoding/json"
"path"
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// seedView adds a view alongside the table the rename harness already creates.
func seedView(t *testing.T, fs *s3tablestest.MemFiler, name string) {
t.Helper()
viewMeta, err := json.Marshal(tableMetadataInternal{
Name: name,
Namespace: "ns",
Format: "ICEBERG",
OwnerAccountID: DefaultAccountID,
MetadataVersion: 1,
MetadataLocation: "s3://" + renameTestBucket + "/ns/" + name + "/metadata/v1.metadata.json",
})
require.NoError(t, err)
fs.Put(GetNamespacePath(renameTestBucket, "ns"), name, map[string][]byte{
ExtendedKeyMetadata: viewMeta,
ExtendedKeyMetadataVersion: []byte("1"),
ExtendedKeyEntryType: []byte(EntryTypeView),
})
viewPath := GetTablePath(renameTestBucket, "ns", name)
fs.Put(viewPath, "metadata", nil)
fs.Put(path.Join(viewPath, "metadata"), "v1.metadata.json", nil)
}
func runRenameView(t *testing.T, m *Manager, fs *s3tablestest.MemFiler, sourceName, destName string) error {
t.Helper()
return m.Execute(context.Background(), NewManagerClient(fs.Client), "RenameView", &RenameTableRequest{
TableBucketARN: mustBucketARN(t),
SourceNamespace: []string{"ns"},
SourceName: sourceName,
DestNamespace: []string{"ns"},
DestName: destName,
}, nil, "")
}
func TestRenameViewMovesCatalogPointer(t *testing.T) {
fs, m := startRenameManager(t)
seedView(t, fs, "v")
require.NoError(t, runRenameView(t, m, fs, "v", "v2"))
dest := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "v2")
require.NotNil(t, dest)
assert.Equal(t, EntryTypeView, EntryType(dest.Extended), "destination must stay a view")
var moved tableMetadataInternal
require.NoError(t, json.Unmarshal(dest.Extended[ExtendedKeyMetadata], &moved))
assert.Equal(t, "v2", moved.Name)
assert.Equal(t, "s3://"+renameTestBucket+"/ns/v/metadata/v1.metadata.json", moved.MetadataLocation,
"rename is catalog-only, the metadata stays where it was written")
src := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "v")
require.NotNil(t, src)
_, stillListed := src.Extended[ExtendedKeyMetadata]
assert.False(t, stillListed, "source name must stop resolving")
assert.NotNil(t, fs.Get(path.Join(GetTablePath(renameTestBucket, "ns", "v"), "metadata"), "v1.metadata.json"),
"the view's metadata file must survive")
}
// Tables and views share one namespace directory, so each rename must refuse
// the other kind rather than moving it.
func TestRenameViewRejectsTable(t *testing.T) {
fs, m := startRenameManager(t)
err := runRenameView(t, m, fs, "t", "t2")
require.Error(t, err)
var s3Err *S3TablesError
require.ErrorAs(t, err, &s3Err)
assert.Equal(t, ErrCodeNoSuchView, s3Err.Type)
}
func TestRenameTableRejectsView(t *testing.T) {
fs, m := startRenameManager(t)
seedView(t, fs, "v")
err := runRename(t, m, fs, &RenameTableRequest{
TableBucketARN: mustBucketARN(t),
SourceNamespace: []string{"ns"},
SourceName: "v",
DestNamespace: []string{"ns"},
DestName: "v2",
})
require.Error(t, err)
var s3Err *S3TablesError
require.ErrorAs(t, err, &s3Err)
assert.Equal(t, ErrCodeNoSuchTable, s3Err.Type)
}
// A policy scoped to the view ARN must authorize renaming that view. Checking
// the source against a table ARN would silently ignore it.
func TestRenameViewAuthorizesAgainstTheViewARN(t *testing.T) {
fs, m := startRenameManager(t)
m.SetTrusted(false)
m.SetDefaultAllow(false)
seedView(t, fs, "v")
const principal = "analyst"
viewARN := "arn:aws:s3tables:" + DefaultRegion + ":" + DefaultAccountID + ":bucket/" + renameTestBucket + "/view/ns/v"
viewPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"` + principal +
`","Action":"s3tables:RenameView","Resource":"` + viewARN + `"}]}`
view := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "v")
require.NotNil(t, view)
view.Extended[ExtendedKeyPolicy] = []byte(viewPolicy)
// Landing in the namespace needs create permission there as well.
namespacePolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"` + principal +
`","Action":"s3tables:CreateView","Resource":"` + mustBucketARN(t) + `"}]}`
namespace := fs.Get(GetTableBucketPath(renameTestBucket), "ns")
require.NotNil(t, namespace)
namespace.Extended[ExtendedKeyPolicy] = []byte(namespacePolicy)
err := m.Execute(context.Background(), NewManagerClient(fs.Client), "RenameView", &RenameTableRequest{
TableBucketARN: mustBucketARN(t),
SourceNamespace: []string{"ns"},
SourceName: "v",
DestNamespace: []string{"ns"},
DestName: "v2",
}, nil, principal)
require.NoError(t, err)
assert.NotNil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "v2"))
}

View File

@@ -0,0 +1,33 @@
package s3tables
import "testing"
func TestEntryType(t *testing.T) {
cases := []struct {
name string
extended map[string][]byte
want string
}{
{"nil extended is table", nil, EntryTypeTable},
{"absent marker is table", map[string][]byte{ExtendedKeyMetadata: []byte("{}")}, EntryTypeTable},
{"empty marker is table", map[string][]byte{ExtendedKeyEntryType: {}}, EntryTypeTable},
{"table marker", map[string][]byte{ExtendedKeyEntryType: []byte(EntryTypeTable)}, EntryTypeTable},
{"view marker", map[string][]byte{ExtendedKeyEntryType: []byte(EntryTypeView)}, EntryTypeView},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := EntryType(c.extended); got != c.want {
t.Fatalf("EntryType() = %q, want %q", got, c.want)
}
})
}
}
func TestGenerateViewARN(t *testing.T) {
h := NewS3TablesHandler()
got := h.generateViewARN(DefaultAccountID, "warehouse", "ns/v")
want := "arn:aws:s3tables:us-east-1:000000000000:bucket/warehouse/view/ns/v"
if got != want {
t.Fatalf("generateViewARN() = %q, want %q", got, want)
}
}

284
weed/s3api/s3tables/iam.go Normal file
View File

@@ -0,0 +1,284 @@
package s3tables
import (
"context"
"fmt"
"net/http"
"reflect"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/iam/integration"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
// IAMAuthorizer allows s3tables handlers to evaluate IAM policies without importing s3api.
type IAMAuthorizer interface {
IsActionAllowed(ctx context.Context, request *integration.ActionRequest) (bool, error)
}
// SetIAMAuthorizer injects the IAM authorizer for policy-based access checks.
func (h *S3TablesHandler) SetIAMAuthorizer(authorizer IAMAuthorizer) {
h.iamAuthorizer = authorizer
}
func (h *S3TablesHandler) shouldUseIAM(r *http.Request, identityActions, identityPolicyNames []string) bool {
if h.iamAuthorizer == nil || r == nil {
return false
}
if s3_constants.GetIdentityFromContext(r) == nil {
return false
}
// When default-allow is enabled, keep anonymous requests on the legacy path
// to preserve zero-config behavior (IAM policies are not available for anonymous).
if h.defaultAllow && isAnonymousIdentity(r) {
return false
}
// An empty inline `identityActions` slice doesn't mean the identity has no
// permissions—it just means authorization lives in IAM policies or session
// tokens instead of static action lists. We therefore prefer the IAM path
// whenever inline actions are absent and fall back to default policy names
// or session tokens.
if hasSessionToken(r) {
return true
}
if len(identityActions) == 0 {
return true
}
return len(identityPolicyNames) > 0
}
// defaultAllowFor reports whether the open-by-default fallback applies: only for
// trusted tooling or unauthenticated/anonymous access. An authenticated principal
// must pass an explicit check.
func (h *S3TablesHandler) defaultAllowFor(r *http.Request) bool {
if h.trusted {
return true
}
if !h.defaultAllow {
return false
}
// The Manager path forwards only the identity name, so a name alone (no
// identity object) still counts as an authenticated principal.
if s3_constants.GetIdentityFromContext(r) == nil && s3_constants.GetIdentityNameFromContext(r) == "" {
return true
}
return isAnonymousIdentity(r)
}
func isAnonymousIdentity(r *http.Request) bool {
val, ok := getIdentityStructValue(r)
if !ok {
return false
}
if nameField := val.FieldByName("Name"); nameField.IsValid() && nameField.Kind() == reflect.String {
if nameField.String() == s3_constants.AccountAnonymousId {
return true
}
}
accountField := val.FieldByName("Account")
if accountField.IsValid() && !accountField.IsNil() {
if accountField.Kind() == reflect.Ptr {
accountField = accountField.Elem()
}
if accountField.Kind() == reflect.Struct {
if idField := accountField.FieldByName("Id"); idField.IsValid() && idField.Kind() == reflect.String {
if idField.String() == s3_constants.AccountAnonymousId {
return true
}
}
}
}
return false
}
func hasSessionToken(r *http.Request) bool {
return extractSessionToken(r) != ""
}
func extractSessionToken(r *http.Request) string {
if token := r.Header.Get(s3_constants.SeaweedFSSessionTokenHeader); token != "" {
return token
}
if token := r.Header.Get("X-Amz-Security-Token"); token != "" {
return token
}
return r.URL.Query().Get("X-Amz-Security-Token")
}
func (h *S3TablesHandler) authorizeIAMAction(r *http.Request, identityPolicyNames []string, action string, resources ...string) (bool, error) {
if h.iamAuthorizer == nil {
err := fmt.Errorf("nil iamAuthorizer in authorizeIAMAction")
glog.V(2).Infof("S3Tables: %v", err)
return false, err
}
principal := r.Header.Get(s3_constants.SeaweedFSPrincipalHeader)
if principal == "" {
principal = getIdentityPrincipalArn(r)
}
if principal == "" {
return false, fmt.Errorf("missing principal for IAM authorization")
}
if !strings.Contains(action, ":") {
action = "s3tables:" + action
}
sessionToken := extractSessionToken(r)
requestContext := buildIAMRequestContext(r, getIdentityClaims(r))
policyNames := identityPolicyNames
if len(policyNames) == 0 {
policyNames = getIdentityPolicyNames(r)
}
if len(resources) == 0 {
return false, fmt.Errorf("no resources provided to authorizeIAMAction")
}
checkedResource := false
for _, resource := range resources {
if resource == "" {
continue
}
checkedResource = true
allowed, err := h.iamAuthorizer.IsActionAllowed(r.Context(), &integration.ActionRequest{
Principal: principal,
Action: action,
Resource: resource,
SessionToken: sessionToken,
RequestContext: requestContext,
PolicyNames: policyNames,
})
if err != nil {
glog.V(2).Infof("S3Tables: IAM authorization error action=%s resource=%s principal=%s: %v", action, resource, principal, err)
return false, err
}
if !allowed {
err := fmt.Errorf("access denied by IAM for resource %s", resource)
return false, err
}
}
if !checkedResource {
return false, fmt.Errorf("no non-empty resources provided to authorizeIAMAction")
}
return true, nil
}
func getIdentityPrincipalArn(r *http.Request) string {
val, ok := getIdentityStructValue(r)
if !ok {
return ""
}
field := val.FieldByName("PrincipalArn")
if field.IsValid() && field.Kind() == reflect.String {
return field.String()
}
return ""
}
func getIdentityPolicyNames(r *http.Request) []string {
val, ok := getIdentityStructValue(r)
if !ok {
return nil
}
field := val.FieldByName("PolicyNames")
if !field.IsValid() || field.Kind() != reflect.Slice {
return nil
}
policies := make([]string, 0, field.Len())
for i := 0; i < field.Len(); i++ {
item := field.Index(i)
if item.Kind() == reflect.String {
policies = append(policies, item.String())
} else if item.CanInterface() {
policies = append(policies, fmt.Sprint(item.Interface()))
}
}
if len(policies) == 0 {
return nil
}
return policies
}
func getIdentityClaims(r *http.Request) map[string]interface{} {
val, ok := getIdentityStructValue(r)
if !ok {
return nil
}
field := val.FieldByName("Claims")
if !field.IsValid() || field.Kind() != reflect.Map || field.IsNil() {
return nil
}
if field.Type().Key().Kind() != reflect.String {
return nil
}
claims := make(map[string]interface{}, field.Len())
for _, key := range field.MapKeys() {
if key.Kind() != reflect.String {
continue
}
val := field.MapIndex(key)
if !val.IsValid() {
continue
}
claims[key.String()] = val.Interface()
}
if len(claims) == 0 {
return nil
}
return claims
}
func buildIAMRequestContext(r *http.Request, claims map[string]interface{}) map[string]interface{} {
ctx := make(map[string]interface{})
if ua := r.Header.Get("User-Agent"); ua != "" {
ctx["userAgent"] = ua
}
if referer := r.Header.Get("Referer"); referer != "" {
ctx["referer"] = referer
}
for k, v := range claims {
if strings.HasPrefix(k, "jwt:") {
if _, exists := ctx[k]; !exists {
ctx[k] = v
}
}
}
for k, v := range claims {
if strings.HasPrefix(k, "jwt:") {
continue
}
if _, exists := ctx[k]; !exists {
ctx[k] = v
}
jwtKey := "jwt:" + k
if _, exists := ctx[jwtKey]; !exists {
ctx[jwtKey] = v
}
}
if len(ctx) == 0 {
return nil
}
return ctx
}
// getIdentityStructValue fetches the identity struct held in the request context.
// The identity is expected to be a pointer to a struct with the fields used by
// the reflection helpers (PrincipalArn string, PolicyNames []string,
// Claims map[string]interface{}).
// This helper centralizes the nil-check and ptr-deref logic so callers focus on
// reading the specific fields they need.
func getIdentityStructValue(r *http.Request) (reflect.Value, bool) {
identityRaw := s3_constants.GetIdentityFromContext(r)
if identityRaw == nil {
return reflect.Value{}, false
}
val := reflect.ValueOf(identityRaw)
if val.Kind() == reflect.Ptr {
val = val.Elem()
}
if val.Kind() != reflect.Struct {
return reflect.Value{}, false
}
return val, true
}

View File

@@ -0,0 +1,365 @@
package s3tables
import (
pathpkg "path"
"regexp"
"strings"
)
// Iceberg file layout validation
// Apache Iceberg tables follow a specific file layout structure:
// - metadata/ directory containing metadata files (*.json, *.avro)
// - data/ directory containing data files (*.parquet, *.orc, *.avro)
//
// Valid file patterns include:
// - metadata/v*.metadata.json (table metadata)
// - metadata/snap-*.avro (snapshot manifest lists)
// - metadata/*.avro (manifest files)
// - data/*.parquet, data/*.orc, data/*.avro (data files)
const uuidPattern = `[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}`
var (
// Allowed directories in a table. The set is the union of what every
// supported format writes, because this runs on the S3 door where the
// table's format is not in hand: metadata/ and data/ for Iceberg,
// _versions/ and _indices/ for Lance, which also puts its fragments in
// data/.
tableAllowedDirs = map[string]bool{
"metadata": true,
"data": true,
}
// A format's own bookkeeping lives in underscore-prefixed directories that
// the catalog does not interpret: Lance writes _versions, _transactions,
// _indices and _deletions, and enumerating them here would mean guessing at
// the next one. Iceberg writes none, so admitting them costs it nothing.
formatOwnedDirPattern = regexp.MustCompile(`^_[a-z][a-z0-9_]*$`)
// Marker files a Lance table keeps at its root to record that it is
// reserved but not yet written, or deregistered but still on storage.
lanceMarkerFiles = map[string]bool{
".lance-reserved": true,
".lance-deregistered": true,
}
// Patterns for valid metadata files.
//
// Note: Iceberg engines (Flink/Spark/Trino, plus different Iceberg versions)
// use a variety of manifest/snapshot naming schemes that the strict patterns
// below don't cover - e.g. Flink emits manifests like
// "{flink-job-id}-{checkpoint}-{operator-id}-{n}.avro". The Iceberg spec
// itself doesn't mandate a specific filename; engines just need a stable
// unique name. So in addition to the strict patterns we keep below for
// documentation, the catch-all entries at the bottom accept any
// *.avro and *.metadata.json filename composed of safe characters
// ([A-Za-z0-9._-]). The catch-all deliberately does NOT cover arbitrary
// *.json — only the *.metadata.json suffix — to avoid letting random
// JSON drop into the metadata/ directory.
metadataFilePatterns = []*regexp.Regexp{
regexp.MustCompile(`^v\d+\.metadata\.json$`), // Table metadata: v1.metadata.json, v2.metadata.json
regexp.MustCompile(`^snap-\d+-\d+-` + uuidPattern + `\.avro$`), // Snapshot manifests: snap-123-1-uuid.avro
regexp.MustCompile(`^` + uuidPattern + `-m\d+\.avro$`), // Manifest files: uuid-m0.avro
regexp.MustCompile(`^` + uuidPattern + `\.avro$`), // General manifest files
regexp.MustCompile(`^version-hint\.text$`), // Version hint file
regexp.MustCompile(`^` + uuidPattern + `\.metadata\.json$`), // UUID-named metadata
regexp.MustCompile(`^[^/]+\.stats$`), // Trino/Iceberg stats files
// Catch-all for Iceberg writer-generated manifest / metadata files
// whose naming we can't anticipate across engines and versions.
regexp.MustCompile(`^[A-Za-z0-9._-]+\.avro$`),
regexp.MustCompile(`^[A-Za-z0-9._-]+\.metadata\.json$`),
}
// Patterns for valid data files
dataFilePatterns = []*regexp.Regexp{
regexp.MustCompile(`^[^/]+\.parquet$`), // Parquet files
regexp.MustCompile(`^[^/]+\.orc$`), // ORC files
regexp.MustCompile(`^[^/]+\.avro$`), // Avro files
regexp.MustCompile(`^[^/]+\.lance$`), // Lance fragments
}
// Data file partition path pattern (e.g., year=2024/month=01/)
partitionPathPattern = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*=[^/]+$`)
// Pattern for valid subdirectory names (alphanumeric, underscore, hyphen, and UUID-style directories)
validSubdirectoryPattern = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`)
)
// IcebergLayoutValidator validates that files conform to Iceberg table layout
type IcebergLayoutValidator struct{}
// NewIcebergLayoutValidator creates a new Iceberg layout validator
func NewIcebergLayoutValidator() *IcebergLayoutValidator {
return &IcebergLayoutValidator{}
}
// ValidateFilePath validates that a file path conforms to Iceberg layout
// The path should be relative to the table root (e.g., "metadata/v1.metadata.json" or "data/file.parquet")
func (v *IcebergLayoutValidator) ValidateFilePath(relativePath string) error {
// Normalize path separators
relativePath = strings.TrimPrefix(relativePath, "/")
if relativePath == "" {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "empty file path",
}
}
parts := strings.SplitN(relativePath, "/", 2)
topDir := parts[0]
// A Lance table's marker files sit at its root rather than in a directory.
if len(parts) == 1 && lanceMarkerFiles[topDir] {
return nil
}
// Check if top-level directory is allowed
if !tableAllowedDirs[topDir] && !formatOwnedDirPattern.MatchString(topDir) {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "files must be placed in 'metadata/', 'data/' or a format-owned '_' directory",
}
}
// If it's just a bare top-level key (no trailing slash and no subpath), reject it
if len(parts) == 1 {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "must be a directory (use trailing slash) or contain a subpath",
}
}
remainingPath := parts[1]
if remainingPath == "" {
return nil // allow paths like "data/" or "metadata/"
}
switch topDir {
case "metadata":
return v.validateMetadataFile(remainingPath)
case "data":
return v.validateDataFile(remainingPath)
}
// Everything else is a format-owned directory: the only guard is that the
// path stays inside the table.
return validateFileSegments(remainingPath)
}
// IsTableMarkerFile reports whether a name is one of the marker files a table
// keeps at its root.
func IsTableMarkerFile(name string) bool {
return lanceMarkerFiles[name]
}
// validateFileSegments rejects traversal and empty segments in a path whose
// contents the catalog does not otherwise interpret.
func validateFileSegments(path string) error {
for _, segment := range strings.Split(path, "/") {
if segment == "" || segment == "." || segment == ".." {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "invalid path segment in: " + path,
}
}
}
return nil
}
// validateDirectoryPath validates intermediate subdirectories in a path
// isMetadata indicates if we're in the metadata directory (true) or data directory (false)
func validateDirectoryPath(normalizedPath string, isMetadata bool) error {
if isMetadata {
// For metadata, reject any subdirectories (enforce flat structure under metadata/)
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "metadata directory does not support subdirectories",
}
}
// For data, validate each partition or subdirectory segment
subdirs := strings.Split(normalizedPath, "/")
for _, subdir := range subdirs {
if subdir == "" {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "invalid partition or subdirectory format in data path: empty segment",
}
}
// For data, allow both partitions and valid subdirectories
if !partitionPathPattern.MatchString(subdir) && !isValidSubdirectory(subdir) {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "invalid partition or subdirectory format in data path",
}
}
}
return nil
}
// validateFilePatterns validates a filename against allowed patterns
// isMetadata indicates if we're validating metadata files (true) or data files (false)
func validateFilePatterns(filename string, isMetadata bool) error {
var patterns []*regexp.Regexp
var errorMsg string
if isMetadata {
patterns = metadataFilePatterns
errorMsg = "invalid metadata file format: must be a valid Iceberg metadata, manifest, or snapshot file"
} else {
patterns = dataFilePatterns
errorMsg = "invalid data file format: must be .parquet, .orc, or .avro"
}
for _, pattern := range patterns {
if pattern.MatchString(filename) {
return nil
}
}
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: errorMsg,
}
}
// validateFile validates files with a unified logic for metadata and data directories
// isMetadata indicates whether we're validating metadata files (true) or data files (false)
// The logic is:
// 1. If path ends with "/", it's a directory - validate all parts and return nil
// 2. Otherwise, validate intermediate parts, then check the filename against patterns
func (v *IcebergLayoutValidator) validateFile(path string, isMetadata bool) error {
// Detect if it's a directory (path ends with "/")
if strings.HasSuffix(path, "/") {
// Normalize by removing trailing slash
normalizedPath := strings.TrimSuffix(path, "/")
return validateDirectoryPath(normalizedPath, isMetadata)
}
filename := pathpkg.Base(path)
// Validate intermediate subdirectories if present
// Find if there are intermediate directories by looking for the last slash
lastSlash := strings.LastIndex(path, "/")
if lastSlash != -1 {
dir := path[:lastSlash]
if err := validateDirectoryPath(dir, isMetadata); err != nil {
return err
}
}
// Check against allowed file patterns
err := validateFilePatterns(filename, isMetadata)
if err == nil {
return nil
}
// Path could be for a directory without a trailing slash, e.g., "data/year=2024"
if !isMetadata {
if partitionPathPattern.MatchString(filename) || isValidSubdirectory(filename) {
return nil
}
}
return err
}
// validateMetadataFile validates files in the metadata/ directory
// This is a thin wrapper that calls validateFile with isMetadata=true
func (v *IcebergLayoutValidator) validateMetadataFile(path string) error {
return v.validateFile(path, true)
}
// validateDataFile validates files in the data/ directory
// This is a thin wrapper that calls validateFile with isMetadata=false
func (v *IcebergLayoutValidator) validateDataFile(path string) error {
return v.validateFile(path, false)
}
// isValidSubdirectory checks if a path component is a valid subdirectory name
func isValidSubdirectory(name string) bool {
// Allow alphanumeric, underscore, hyphen, and UUID-style directories
return validSubdirectoryPattern.MatchString(name)
}
// IcebergLayoutError represents an Iceberg layout validation error
type IcebergLayoutError struct {
Code string
Message string
}
func (e *IcebergLayoutError) Error() string {
return e.Message
}
// Error code for Iceberg layout violations
const (
ErrCodeInvalidIcebergLayout = "InvalidIcebergLayout"
)
// TableBucketFileValidator validates file uploads to table buckets
type TableBucketFileValidator struct {
layoutValidator *IcebergLayoutValidator
}
// NewTableBucketFileValidator creates a new table bucket file validator
func NewTableBucketFileValidator() *TableBucketFileValidator {
return &TableBucketFileValidator{
layoutValidator: NewIcebergLayoutValidator(),
}
}
// ValidateTableBucketUpload checks if a file upload to a table bucket conforms to Iceberg layout
// fullPath is the complete filer path (e.g., /buckets/mybucket/mynamespace/mytable/data/file.parquet)
// Returns nil if the path is not a table bucket path or if validation passes
// Returns an error if the file doesn't conform to Iceberg layout
func (v *TableBucketFileValidator) ValidateTableBucketUpload(fullPath string) error {
// Check if this is a table bucket path
if !strings.HasPrefix(fullPath, TablesPath+"/") {
return nil // Not a table bucket, no validation needed
}
// Extract the path relative to table bucket root
// Format: /buckets/{bucket}/{namespace}/{table}/{relative-path}
relativePath := strings.TrimPrefix(fullPath, TablesPath+"/")
parts := strings.SplitN(relativePath, "/", 4)
// Need at least bucket/namespace/table/file
if len(parts) < 4 {
// Creating bucket, namespace, or table directories - allow only if preceding parts are non-empty
for i := 0; i < len(parts); i++ {
if parts[i] == "" {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "bucket, namespace, and table segments cannot be empty",
}
}
}
return nil
}
// For full paths, also verify bucket, namespace, and table segments are non-empty
if parts[0] == "" || parts[1] == "" || parts[2] == "" {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "bucket, namespace, and table segments cannot be empty",
}
}
// The last part is the path within the table (data/file.parquet or metadata/v1.json)
tableRelativePath := parts[3]
if tableRelativePath == "" {
return nil
}
// Reject paths with empty segments (double slashes) within the table path
if strings.HasPrefix(tableRelativePath, "/") || strings.Contains(tableRelativePath, "//") {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "bucket, namespace, and table segments cannot be empty",
}
}
return v.layoutValidator.ValidateFilePath(tableRelativePath)
}

View File

@@ -0,0 +1,136 @@
package s3tables
import "testing"
// TestIcebergLayoutValidator_AcceptsRealWorldManifestNames pins down the
// filename patterns we must accept across Iceberg engines. The original strict
// regex only covered Iceberg-internal naming (`{uuid}-m{n}.avro`,
// `snap-{n}-{n}-{uuid}.avro`) and rejected real manifests written by Flink and
// other writers, causing 403s during INSERT commits. The catch-all entries
// added alongside this test must keep these names valid.
func TestIcebergLayoutValidator_AcceptsRealWorldManifestNames(t *testing.T) {
cases := []struct {
name string
path string
}{
{
"flink-style manifest (job-id + checkpoint + operator-id + counter)",
"metadata/02678a59b3d6b460ba392851d77155fc-1-cbc357ccb763df2852fee8c4fc7d55f2-00001.avro",
},
{
"spark-style manifest with two uuids and -m suffix",
"metadata/00000000-0000-0000-0000-000000000000-m0.avro",
},
{
"snapshot manifest list with iceberg-internal naming",
"metadata/snap-7234891234567890123-1-82e3eec4-3aee-414f-a444-94c03c641d20.avro",
},
{
"versioned table metadata",
"metadata/v1.metadata.json",
},
{
"uuid-named metadata json (newer iceberg)",
"metadata/82e3eec4-3aee-414f-a444-94c03c641d20.metadata.json",
},
{
"flink-style metadata json with dashes and digits",
"metadata/02678a59-1-cbc357cc.metadata.json",
},
{
"version hint",
"metadata/version-hint.text",
},
{
"trino/iceberg stats file",
"metadata/table.stats",
},
}
v := NewIcebergLayoutValidator()
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if err := v.ValidateFilePath(tc.path); err != nil {
t.Errorf("expected %q to be accepted, got error: %v", tc.path, err)
}
})
}
}
// TestIcebergLayoutValidator_RejectsClearlyBadMetadataNames guards the
// catch-all from being too permissive — paths that look like attempts to
// escape the metadata layout or have forbidden file types must still fail.
func TestIcebergLayoutValidator_RejectsClearlyBadMetadataNames(t *testing.T) {
cases := []struct {
name string
path string
}{
{"random extension", "metadata/random-file.txt"},
{"executable masquerading as avro path", "metadata/evil.sh"},
{"subdirectory under metadata is not allowed", "metadata/sub/file.avro"},
{"top-level dir other than metadata or data", "garbage/file.avro"},
{"catch-all anchors must reject trailing extension", "metadata/file.avro.txt"},
}
v := NewIcebergLayoutValidator()
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if err := v.ValidateFilePath(tc.path); err == nil {
t.Errorf("expected %q to be rejected, but it passed validation", tc.path)
}
})
}
}
// TestIcebergLayoutValidator_AcceptsRealWorldDataFiles is a sanity check that
// the patterns most engines actually emit for data files still pass.
func TestIcebergLayoutValidator_AcceptsRealWorldDataFiles(t *testing.T) {
cases := []string{
"data/00000-0-ede83b82-08e1-40cd-af8a-6d83680a5194-00001.parquet",
"data/part-00000.parquet",
"data/some-file.orc",
"data/year=2026/month=05/00000-0-uuid.parquet",
}
v := NewIcebergLayoutValidator()
for _, p := range cases {
t.Run(p, func(t *testing.T) {
if err := v.ValidateFilePath(p); err != nil {
t.Errorf("expected %q to be accepted, got error: %v", p, err)
}
})
}
}
// A Lance dataset writes fragments into data/ and keeps its own bookkeeping in
// underscore-prefixed directories. The validator runs on the S3 door, where the
// table's format is not in hand, so it has to admit both formats' layouts.
func TestValidateFilePathAcceptsLanceLayout(t *testing.T) {
v := NewIcebergLayoutValidator()
allowed := []string{
"data/01111110101110001101011164cadc43919eace9c608107bd9.lance",
"_versions/1.manifest",
"_versions/9223372036854775806.manifest",
"_transactions/0-ddb27ab7-2e5c-42c7-b4bf-265d8a3ff636.txn",
"_indices/85814508-ed9a-41f2-b939-2050bb7a0ed5-fts/index.idx",
"_deletions/_deletions-1.arrow",
".lance-reserved",
".lance-deregistered",
}
for _, path := range allowed {
if err := v.ValidateFilePath(path); err != nil {
t.Errorf("ValidateFilePath(%q) = %v, want nil", path, err)
}
}
rejected := []string{
"_versions/../../escape",
"_versions//empty",
"notadir/file.lance",
"random.txt",
}
for _, path := range rejected {
if err := v.ValidateFilePath(path); err == nil {
t.Errorf("ValidateFilePath(%q) = nil, want an error", path)
}
}
}

View File

@@ -0,0 +1,193 @@
package s3tables
import (
"bytes"
"time"
"github.com/apache/iceberg-go"
"github.com/google/uuid"
)
const secondsPerDay = int64(24 * 60 * 60)
// ReadManifest parses an Iceberg manifest and converts its partition values
// while the manifest's own schema still describes them.
//
// iceberg-go converts what the Avro decoder returned for a logical type - a
// time.Time for a date, a time.Duration for a time - to an Iceberg value
// lazily, on the first Partition() call, using the logical types it read from
// the manifest being parsed. ManifestWriter.addEntry rebinds those to the
// logical types of the manifest it is about to write before it makes that
// call, so an entry that is read and written again without anyone looking at
// its partition converts against the wrong schema. A day partition is where
// the two disagree: iceberg-go's day transform reports an int32 result type,
// so the manifest it writes has no date logical type for that field, nothing
// converts, and the time.Time reaches the encoder as "cannot use time.Time
// with Avro type int".
//
// What survives that is a value iceberg-go never recognized on read either: it
// takes a partition field's logical type from the last branch of its Avro
// union, so a writer that spells an optional partition [<type>, null] rather
// than [null, <type>] hides it, and a time partition is then worse than a
// failed write - time.Duration converts to int64 nanoseconds and silently
// records the wrong value. normalizePartitionValue puts those back.
//
// specs maps partition spec ID to spec the way the table metadata records them.
func ReadManifest(m iceberg.ManifestFile, manifest []byte, discardDeleted bool, specs map[int]iceberg.PartitionSpec, schema *iceberg.Schema) ([]iceberg.ManifestEntry, error) {
entries, err := iceberg.ReadManifest(m, bytes.NewReader(manifest), discardDeleted)
if err != nil {
return nil, err
}
var partitionFields []iceberg.NestedField
var spec iceberg.PartitionSpec
var specFound bool
if schema != nil {
spec, specFound = specs[int(m.PartitionSpecID())]
if specFound {
if partitionType := spec.PartitionType(schema); partitionType != nil {
partitionFields = partitionType.FieldList
}
}
}
for i, entry := range entries {
partition := entry.DataFile().Partition()
changed := false
for _, field := range partitionFields {
value, ok := partition[field.ID]
if !ok {
continue
}
if normalized, ok := normalizePartitionValue(value, field.Type); ok {
partition[field.ID] = normalized
changed = true
}
}
if changed {
// iceberg-go returns a defensive copy from Partition(), so the
// in-place edits above never reach the underlying DataFile.
// Rebuild the entry with a DataFile that carries the normalized
// partition so a subsequent write encodes the Iceberg values.
entries[i] = rebuildManifestEntry(entry, spec, partition)
}
}
return entries, nil
}
// rebuildManifestEntry returns a copy of entry whose DataFile carries the
// given partition map. It is used to persist partition values normalized
// after a read, since iceberg-go's Partition() getter hands back a clone
// rather than the live map. Only the partition changes; every other DataFile
// field is copied through the builder so manifest round-trips are preserved.
func rebuildManifestEntry(entry iceberg.ManifestEntry, spec iceberg.PartitionSpec, partition map[int]any) iceberg.ManifestEntry {
df := entry.DataFile()
builder, err := iceberg.NewDataFileBuilder(
spec,
df.ContentType(),
df.FilePath(),
df.FileFormat(),
partition,
nil, // logical types are derived from the spec at write time
nil,
df.Count(),
df.FileSizeBytes(),
)
if err != nil {
// The original DataFile already validated these fields, so the only
// ways to get here are a nil spec (which rebuildManifestEntry's caller
// guards against) or a zero-record / zero-byte file, which the builder
// rejects with recordCount <= 0 or fileSize <= 0. Fall back to the
// original entry rather than panic; the unnormalized partition survives
// but an empty file is unlikely to carry a logical partition value.
return entry
}
builder.BlockSizeInBytes(64 * 1024 * 1024) // v1 schema default; deprecated and omitted in v2+
if sizes := df.ColumnSizes(); sizes != nil {
builder.ColumnSizes(sizes)
}
if counts := df.ValueCounts(); counts != nil {
builder.ValueCounts(counts)
}
if counts := df.NullValueCounts(); counts != nil {
builder.NullValueCounts(counts)
}
if counts := df.NaNValueCounts(); counts != nil {
builder.NaNValueCounts(counts)
}
if counts := df.DistinctValueCounts(); counts != nil {
builder.DistinctValueCounts(counts)
}
if bounds := df.LowerBoundValues(); bounds != nil {
builder.LowerBoundValues(bounds)
}
if bounds := df.UpperBoundValues(); bounds != nil {
builder.UpperBoundValues(bounds)
}
if key := df.KeyMetadata(); key != nil {
builder.KeyMetadata(key)
}
if offsets := df.SplitOffsets(); offsets != nil {
builder.SplitOffsets(offsets)
}
if ids := df.EqualityFieldIDs(); ids != nil {
builder.EqualityFieldIDs(ids)
}
if id := df.SortOrderID(); id != nil {
builder.SortOrderID(*id)
}
if id := df.FirstRowID(); id != nil {
builder.FirstRowID(*id)
}
if ref := df.ReferencedDataFile(); ref != nil {
builder.ReferencedDataFile(*ref)
}
if off := df.ContentOffset(); off != nil {
builder.ContentOffset(*off)
}
if size := df.ContentSizeInBytes(); size != nil {
builder.ContentSizeInBytes(*size)
}
newEntry := iceberg.NewManifestEntryBuilder(entry.Status(), ptrInt64(entry.SnapshotID()), builder.Build())
if seq := entry.SequenceNum(); seq != -1 {
newEntry.SequenceNum(seq)
}
if fileSeq := entry.FileSequenceNum(); fileSeq != nil {
newEntry.FileSequenceNum(*fileSeq)
}
return newEntry.Build()
}
func ptrInt64(v int64) *int64 { return &v }
// normalizePartitionValue converts one value the Avro decoder returned for a
// logical type to the Iceberg representation the manifest writer expects,
// mirroring what iceberg-go does itself for the unions it does recognize.
func normalizePartitionValue(value any, fieldType iceberg.Type) (any, bool) {
switch v := value.(type) {
case time.Time:
utc := v.UTC()
switch fieldType.(type) {
case iceberg.DateType, iceberg.Int32Type:
// A day transform reports an int32 result type and an identity
// transform on a date column reports date; both hold days.
midnight := time.Date(utc.Year(), utc.Month(), utc.Day(), 0, 0, 0, 0, time.UTC)
return iceberg.Date(midnight.Unix() / secondsPerDay), true
case iceberg.TimestampType, iceberg.TimestampTzType:
return iceberg.Timestamp(utc.UnixMicro()), true
case iceberg.TimestampNsType, iceberg.TimestampTzNsType:
return iceberg.TimestampNano(utc.UnixNano()), true
}
case time.Duration:
if _, ok := fieldType.(iceberg.TimeType); ok {
return iceberg.Time(v.Microseconds()), true
}
case [16]byte:
if _, ok := fieldType.(iceberg.UUIDType); ok {
return uuid.UUID(v), true
}
}
return nil, false
}

View File

@@ -0,0 +1,168 @@
package s3tables
import (
"bytes"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"strconv"
"github.com/apache/iceberg-go"
)
// avroFileMagic prefixes every Avro object container file.
var avroFileMagic = []byte{'O', 'b', 'j', 1}
// formatVersionKey is the Avro header entry naming the Iceberg format version.
const formatVersionKey = "format-version"
// ReadManifestList parses an Iceberg manifest list, tolerating writers that
// leave the format version out of the Avro header.
//
// The Iceberg spec pins the header metadata of manifest *files* but says
// nothing about manifest *lists*, so writers disagree: Java and PyIceberg
// record "format-version", DuckDB writes no header metadata at all. iceberg-go
// reads a missing entry as v1, which makes every v2 manifest reachable from
// such a list fail with "manifest file's 'format-version' metadata indicates
// version 2, but entry from manifest list indicates version 1", and silently
// decodes delete manifests as data manifests because v1 has no "content"
// field. Both leave DuckDB-written tables unmaintainable.
//
// When the entry is missing we derive the version from the record schema the
// writer embedded — the fields it carries say which version wrote it — and add
// it to the header before handing the bytes to iceberg-go.
func ReadManifestList(manifestList []byte) ([]iceberg.ManifestFile, error) {
if patched, ok := withFormatVersion(manifestList); ok {
manifestList = patched
}
return iceberg.ReadManifestList(bytes.NewReader(manifestList))
}
// withFormatVersion returns manifestList with a "format-version" header entry
// spliced in. It reports false when the header already carries one, or when it
// cannot be parsed — the untouched bytes then go to iceberg-go, which reports
// the underlying problem.
func withFormatVersion(manifestList []byte) ([]byte, bool) {
metadata, terminator, err := readAvroFileMetadata(manifestList)
if err != nil {
return nil, false
}
if _, ok := metadata[formatVersionKey]; ok {
return nil, false
}
// The metadata map is a sequence of blocks closed by a zero count, so a
// new single-entry block can be spliced in ahead of that closing count
// without re-encoding what the writer already wrote.
block := appendAvroLong(nil, 1)
block = appendAvroBytes(block, []byte(formatVersionKey))
block = appendAvroBytes(block, []byte(strconv.Itoa(manifestListFormatVersion(metadata["avro.schema"]))))
patched := make([]byte, 0, len(manifestList)+len(block))
patched = append(patched, manifestList[:terminator]...)
patched = append(patched, block...)
return append(patched, manifestList[terminator:]...), true
}
// manifestListFormatVersion infers the format version a manifest list was
// written at from the fields of its manifest_file record: v2 added "content",
// "sequence_number" and "min_sequence_number", v3 added "first_row_id".
func manifestListFormatVersion(schema []byte) int {
var record struct {
Fields []struct {
Name string `json:"name"`
} `json:"fields"`
}
if err := json.Unmarshal(schema, &record); err != nil {
return 1
}
version := 1
for _, field := range record.Fields {
switch field.Name {
case "content", "sequence_number", "min_sequence_number":
if version < 2 {
version = 2
}
case "first_row_id":
version = 3
}
}
return version
}
// readAvroFileMetadata decodes the header metadata map of an Avro object
// container file, returning the map alongside the offset of the zero block
// count that closes it.
func readAvroFileMetadata(data []byte) (map[string][]byte, int, error) {
if !bytes.HasPrefix(data, avroFileMagic) {
return nil, 0, errors.New("not an avro object container file")
}
metadata := make(map[string][]byte)
pos := len(avroFileMagic)
for {
blockStart := pos
count, next, err := readAvroLong(data, pos)
if err != nil {
return nil, 0, err
}
pos = next
if count == 0 {
return metadata, blockStart, nil
}
if count < 0 {
// A negative count is followed by the block size in bytes.
count = -count
if _, pos, err = readAvroLong(data, pos); err != nil {
return nil, 0, err
}
}
for i := int64(0); i < count; i++ {
key, next, err := readAvroBytes(data, pos)
if err != nil {
return nil, 0, err
}
value, next, err := readAvroBytes(data, next)
if err != nil {
return nil, 0, err
}
metadata[string(key)] = value
pos = next
}
}
}
// readAvroLong decodes the zig-zag varint at pos, returning it with the offset
// just past it.
func readAvroLong(data []byte, pos int) (int64, int, error) {
if pos < 0 || pos >= len(data) {
return 0, 0, fmt.Errorf("avro long at offset %d is out of range", pos)
}
value, n := binary.Varint(data[pos:])
if n <= 0 {
return 0, 0, fmt.Errorf("avro long at offset %d is truncated", pos)
}
return value, pos + n, nil
}
// readAvroBytes decodes the length-prefixed byte sequence at pos, returning it
// with the offset just past it. The result aliases data.
func readAvroBytes(data []byte, pos int) ([]byte, int, error) {
length, pos, err := readAvroLong(data, pos)
if err != nil {
return nil, 0, err
}
if length < 0 || int64(len(data)-pos) < length {
return nil, 0, fmt.Errorf("avro bytes at offset %d are truncated", pos)
}
return data[pos : pos+int(length)], pos + int(length), nil
}
func appendAvroLong(dst []byte, value int64) []byte {
var scratch [binary.MaxVarintLen64]byte
return append(dst, scratch[:binary.PutVarint(scratch[:], value)]...)
}
func appendAvroBytes(dst []byte, value []byte) []byte {
return append(appendAvroLong(dst, int64(len(value))), value...)
}

View File

@@ -0,0 +1,176 @@
package s3tables
import (
"bytes"
"testing"
"github.com/apache/iceberg-go"
)
// writeManifestList builds a manifest list holding one data and one delete
// manifest, the shape DuckDB produces for a v2 table that has been updated.
func writeManifestList(t *testing.T, version int) []byte {
t.Helper()
dataManifest := iceberg.NewManifestFile(version, "s3://bucket/ns/tbl/metadata/data-m0.avro", 1024, 0, 7).
SequenceNum(3, 3).
Content(iceberg.ManifestContentData).
AddedFiles(1).
AddedRows(10).
Build()
files := []iceberg.ManifestFile{dataManifest}
if version > 1 {
// Delete manifests only exist from v2 onwards.
files = append(files, iceberg.NewManifestFile(version, "s3://bucket/ns/tbl/metadata/deletes-m0.avro", 512, 0, 7).
SequenceNum(3, 3).
Content(iceberg.ManifestContentDeletes).
AddedFiles(1).
AddedRows(1).
Build())
}
var buf bytes.Buffer
seqNum := int64(3)
if err := iceberg.WriteManifestList(version, &buf, 7, nil, &seqNum, 0, files); err != nil {
t.Fatalf("write v%d manifest list: %v", version, err)
}
return buf.Bytes()
}
// stripFormatVersion rewrites an Avro header without its "format-version"
// entry, reproducing what DuckDB writes: a manifest list carrying no Iceberg
// header metadata at all.
func stripFormatVersion(t *testing.T, data []byte) []byte {
t.Helper()
metadata, terminator, err := readAvroFileMetadata(data)
if err != nil {
t.Fatalf("read avro header: %v", err)
}
if _, ok := metadata[formatVersionKey]; !ok {
t.Fatal("fixture has no format-version to strip")
}
var entries []byte
count := 0
for key, value := range metadata {
if key == formatVersionKey {
continue
}
entries = appendAvroBytes(appendAvroBytes(entries, []byte(key)), value)
count++
}
stripped := append([]byte{}, avroFileMagic...)
stripped = appendAvroLong(stripped, int64(count))
stripped = append(stripped, entries...)
// Everything from the closing zero count on — sync marker and data
// blocks included — is unaffected by the header rewrite.
return append(stripped, data[terminator:]...)
}
// DuckDB writes manifest lists with no Avro header metadata. iceberg-go now
// infers the format version from the embedded writer schema on its own, so a
// stripped v2 list is read as v2 even without the header entry. ReadManifestList
// still patches the header for older iceberg-go releases and for writers whose
// schema carries none of the version-distinguishing fields.
func TestReadManifestListWithoutFormatVersion(t *testing.T) {
stripped := stripFormatVersion(t, writeManifestList(t, 2))
// Baseline: iceberg-go reads the list as v2 by inferring the version from
// the record schema's fields (content / sequence_number were added in v2).
unpatched, err := iceberg.ReadManifestList(bytes.NewReader(stripped))
if err != nil {
t.Fatalf("iceberg.ReadManifestList: %v", err)
}
if got := unpatched[0].Version(); got != 2 {
t.Fatalf("expected iceberg-go to infer v2, got v%d", got)
}
manifests, err := ReadManifestList(stripped)
if err != nil {
t.Fatalf("ReadManifestList: %v", err)
}
if len(manifests) != 2 {
t.Fatalf("expected 2 manifests, got %d", len(manifests))
}
for _, mf := range manifests {
if mf.Version() != 2 {
t.Errorf("manifest %s: version = %d, want 2", mf.FilePath(), mf.Version())
}
if mf.SequenceNum() != 3 || mf.MinSequenceNum() != 3 {
t.Errorf("manifest %s: sequence numbers = %d/%d, want 3/3", mf.FilePath(), mf.SequenceNum(), mf.MinSequenceNum())
}
}
if got := manifests[0].ManifestContent(); got != iceberg.ManifestContentData {
t.Errorf("first manifest content = %s, want data", got)
}
if got := manifests[1].ManifestContent(); got != iceberg.ManifestContentDeletes {
t.Errorf("second manifest content = %s, want deletes", got)
}
}
func TestReadManifestListKeepsWrittenFormatVersion(t *testing.T) {
for _, version := range []int{1, 2} {
manifests, err := ReadManifestList(writeManifestList(t, version))
if err != nil {
t.Fatalf("v%d: ReadManifestList: %v", version, err)
}
if got := manifests[0].Version(); got != version {
t.Errorf("v%d: version = %d", version, got)
}
}
}
// A v1 manifest list without the header entry must stay v1: its record schema
// has none of the fields v2 added.
func TestReadManifestListWithoutFormatVersionStaysV1(t *testing.T) {
manifests, err := ReadManifestList(stripFormatVersion(t, writeManifestList(t, 1)))
if err != nil {
t.Fatalf("ReadManifestList: %v", err)
}
if got := manifests[0].Version(); got != 1 {
t.Errorf("version = %d, want 1", got)
}
}
// Unparseable input is handed to iceberg-go untouched so it keeps reporting
// the underlying problem rather than a header-rewriting one.
func TestReadManifestListRejectsNonAvro(t *testing.T) {
for name, data := range map[string][]byte{
"empty": nil,
"not avro": []byte("this is not an avro file"),
"truncated magic": avroFileMagic[:3],
"truncated header": func() []byte {
list := writeManifestList(t, 2)
return list[:len(avroFileMagic)+2]
}(),
} {
if _, err := ReadManifestList(data); err == nil {
t.Errorf("%s: expected an error", name)
}
}
}
func TestManifestListFormatVersion(t *testing.T) {
tests := []struct {
name string
schema string
want int
}{
{"v1 fields", `{"fields":[{"name":"manifest_path"},{"name":"added_snapshot_id"}]}`, 1},
{"v2 adds content", `{"fields":[{"name":"manifest_path"},{"name":"content"}]}`, 2},
{"v2 adds sequence numbers", `{"fields":[{"name":"sequence_number"},{"name":"min_sequence_number"}]}`, 2},
{"v3 adds first_row_id", `{"fields":[{"name":"content"},{"name":"first_row_id"}]}`, 3},
{"v3 field ordering", `{"fields":[{"name":"first_row_id"},{"name":"content"}]}`, 3},
{"unparseable schema", `not json`, 1},
{"missing schema", ``, 1},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if got := manifestListFormatVersion([]byte(tc.schema)); got != tc.want {
t.Errorf("manifestListFormatVersion() = %d, want %d", got, tc.want)
}
})
}
}

View File

@@ -0,0 +1,275 @@
package s3tables
import (
"bytes"
"testing"
"time"
"github.com/apache/iceberg-go"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
"github.com/twmb/avro/ocf"
)
// A manifest whose partition union puts the value branch first hides the
// logical type from iceberg-go, which then hands back whatever the Avro decoder
// produced even once the partition is read. Rewriting such an entry fails when
// the writer builds its partition summaries, or encodes the wrong number for a
// time partition, so every logical type a partition field can carry has to come
// back as an Iceberg value.
func TestReadManifestNormalizesForeignPartitions(t *testing.T) {
day := time.Date(2026, time.October, 11, 0, 0, 0, 0, time.UTC)
cases := []struct {
name string
sourceType iceberg.PrimitiveType
transform iceberg.Transform
logicalType string
// decoded is the value the foreign manifest carries, as the Avro
// decoder hands it back.
decoded any
want any
// nullFirst spells the union the way Java and iceberg-rust do, which
// iceberg-go converts on its own once the partition is read.
nullFirst bool
// rawIsNormalized marks transforms iceberg-go now converts itself on
// read (day transforms), so the raw value already matches want.
rawIsNormalized bool
}{
{
name: "day transform written null-first converts on read",
sourceType: iceberg.PrimitiveTypes.Timestamp,
transform: iceberg.DayTransform{},
logicalType: "date",
decoded: day,
want: iceberg.Date(20737),
nullFirst: true,
},
{
name: "day transform on a timestamp column",
sourceType: iceberg.PrimitiveTypes.Timestamp,
transform: iceberg.DayTransform{},
logicalType: "date",
decoded: day,
want: iceberg.Date(20737),
// iceberg-go now converts day-transform partitions to iceberg.Date
// itself (applyDayTransformDates), so the raw read already yields
// the Iceberg value regardless of union branch ordering.
rawIsNormalized: true,
},
{
name: "identity transform on a date column",
sourceType: iceberg.PrimitiveTypes.Date,
transform: iceberg.IdentityTransform{},
logicalType: "date",
decoded: day,
want: iceberg.Date(20737),
},
{
name: "identity transform on a timestamp column",
sourceType: iceberg.PrimitiveTypes.Timestamp,
transform: iceberg.IdentityTransform{},
logicalType: "timestamp-micros",
decoded: day.Add(3 * time.Hour),
want: iceberg.Timestamp(day.Add(3 * time.Hour).UnixMicro()),
},
{
name: "identity transform on a time column",
sourceType: iceberg.PrimitiveTypes.Time,
transform: iceberg.IdentityTransform{},
logicalType: "time-micros",
decoded: 3 * time.Hour,
want: iceberg.Time((3 * time.Hour).Microseconds()),
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
schema := iceberg.NewSchema(0,
iceberg.NestedField{ID: 1, Name: "source", Type: c.sourceType, Required: true},
)
spec := iceberg.NewPartitionSpec(iceberg.PartitionField{
SourceIDs: []int{1}, FieldID: 1000, Name: "part", Transform: c.transform,
})
specs := map[int]iceberg.PartitionSpec{spec.ID(): spec}
dfBuilder, err := iceberg.NewDataFileBuilder(spec, iceberg.EntryContentData, "data/file.parquet",
iceberg.ParquetFile, map[int]any{1000: c.want}, nil, nil, 1, 1)
if err != nil {
t.Fatalf("build data file: %v", err)
}
snapshotID := int64(1)
entry := iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapshotID, nil, nil, dfBuilder.Build())
foreignBytes, foreignManifest := s3tablestest.ForeignPartitionManifest(t, schema, spec, entry,
"metadata/foreign-manifest.avro", c.logicalType, c.decoded, !c.nullFirst)
// iceberg-go reads the logical type off the union's last branch, so
// the ordering decides whether the entry arrives as an Iceberg value
// or as whatever the Avro decoder produced.
rawWant := any(c.decoded)
if c.nullFirst || c.rawIsNormalized {
rawWant = c.want
}
raw, err := iceberg.ReadManifest(foreignManifest, bytes.NewReader(foreignBytes), true)
if err != nil {
t.Fatalf("read foreign manifest: %v", err)
}
if got := raw[0].DataFile().Partition()[1000]; got != rawWant {
t.Fatalf("unnormalized partition value = %#v (%T), want %#v (%T)", got, got, rawWant, rawWant)
}
entries, err := ReadManifest(foreignManifest, foreignBytes, true, specs, schema)
if err != nil {
t.Fatalf("read foreign manifest: %v", err)
}
if got := entries[0].DataFile().Partition()[1000]; got != c.want {
t.Fatalf("normalized partition value = %#v (%T), want %#v (%T)", got, got, c.want, c.want)
}
// The point of normalizing: the entry can be written out again.
var buf bytes.Buffer
if _, err := iceberg.WriteManifest("metadata/manifest.avro", &buf, 2, spec, schema, snapshotID, entries); err != nil {
t.Fatalf("write normalized manifest: %v", err)
}
})
}
}
// A manifest written under a spec the table metadata no longer records is
// returned as read rather than refused: the caller decides what to do with it.
func TestReadManifestUnknownSpec(t *testing.T) {
schema := iceberg.NewSchema(0,
iceberg.NestedField{ID: 1, Name: "source", Type: iceberg.PrimitiveTypes.Date, Required: true},
)
spec := iceberg.NewPartitionSpec(iceberg.PartitionField{
SourceIDs: []int{1}, FieldID: 1000, Name: "part", Transform: iceberg.IdentityTransform{},
})
dfBuilder, err := iceberg.NewDataFileBuilder(spec, iceberg.EntryContentData, "data/file.parquet",
iceberg.ParquetFile, map[int]any{1000: iceberg.Date(20737)}, nil, nil, 1, 1)
if err != nil {
t.Fatalf("build data file: %v", err)
}
snapshotID := int64(1)
entry := iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapshotID, nil, nil, dfBuilder.Build())
foreignBytes, foreignManifest := s3tablestest.ForeignPartitionManifest(t, schema, spec, entry,
"metadata/foreign-manifest.avro", "date", time.Date(2026, time.October, 11, 0, 0, 0, 0, time.UTC), true)
entries, err := ReadManifest(foreignManifest, foreignBytes, true, nil, schema)
if err != nil {
t.Fatalf("read foreign manifest: %v", err)
}
if _, ok := entries[0].DataFile().Partition()[1000].(time.Time); !ok {
t.Fatalf("partition value = %T, want it returned as read", entries[0].DataFile().Partition()[1000])
}
}
// iceberg-go now converts day-transform partitions to iceberg.Date during read
// (applyDayTransformDates), so an entry nobody looked at between reading and
// writing still carries an Iceberg value and writes cleanly. This used to fail
// because conversion was lazy and ManifestWriter.addEntry rebound the logical
// types to the output manifest — whose day transform carries no date logical
// type — before the first Partition() call, leaving a time.Time that the
// encoder rejected. The shim keeps the round-trip working for the identity
// transforms iceberg-go still does not convert (see TestReadManifestNormalizesForeignPartitions).
func TestReadManifestConvertsBeforeTheWriterRebindsLogicalTypes(t *testing.T) {
schema := iceberg.NewSchema(0,
iceberg.NestedField{ID: 1, Name: "event_time", Type: iceberg.PrimitiveTypes.Timestamp, Required: true},
)
spec := iceberg.NewPartitionSpec(iceberg.PartitionField{
SourceIDs: []int{1}, FieldID: 1000, Name: "event_time_day", Transform: iceberg.DayTransform{},
})
specs := map[int]iceberg.PartitionSpec{spec.ID(): spec}
dfBuilder, err := iceberg.NewDataFileBuilder(spec, iceberg.EntryContentData, "data/file.parquet",
iceberg.ParquetFile, map[int]any{1000: iceberg.Date(20737)}, nil, nil, 1, 1)
if err != nil {
t.Fatalf("build data file: %v", err)
}
snapshotID := int64(1)
entry := iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapshotID, nil, nil, dfBuilder.Build())
foreignBytes, foreignManifest := s3tablestest.ForeignPartitionManifest(t, schema, spec, entry,
"metadata/foreign-manifest.avro", "date", time.Date(2026, time.October, 11, 0, 0, 0, 0, time.UTC), false)
untouched, err := iceberg.ReadManifest(foreignManifest, bytes.NewReader(foreignBytes), true)
if err != nil {
t.Fatalf("read foreign manifest: %v", err)
}
got := untouched[0].DataFile().Partition()[1000]
if _, ok := got.(iceberg.Date); !ok {
t.Fatalf("iceberg-go now converts day partitions on read; got %T, want iceberg.Date", got)
}
var buf bytes.Buffer
if _, err := iceberg.WriteManifest("metadata/manifest.avro", &buf, 2, spec, schema, snapshotID, untouched); err != nil {
t.Fatalf("write manifest read directly through iceberg-go: %v", err)
}
entries, err := ReadManifest(foreignManifest, foreignBytes, true, specs, schema)
if err != nil {
t.Fatalf("read foreign manifest: %v", err)
}
buf.Reset()
if _, err := iceberg.WriteManifest("metadata/manifest.avro", &buf, 2, spec, schema, snapshotID, entries); err != nil {
t.Fatalf("write manifest read through the shim: %v", err)
}
}
// rebuildManifestEntry rebuilds the DataFile through NewDataFileBuilder, which
// has no setter for the original block_size_in_bytes (the field is not exposed
// on DataFile). The v1 manifest schema includes block_size_in_bytes with a
// default of 64 MiB and the spec says "Always write default in v1", so a
// rebuilt entry written as v1 must carry that default — not 0.
func TestReadManifestRebuiltEntryPreservesV1BlockSizeDefault(t *testing.T) {
schema := iceberg.NewSchema(0,
iceberg.NestedField{ID: 1, Name: "event_date", Type: iceberg.PrimitiveTypes.Date, Required: true},
)
spec := iceberg.NewPartitionSpec(iceberg.PartitionField{
SourceIDs: []int{1}, FieldID: 1000, Name: "part", Transform: iceberg.IdentityTransform{},
})
specs := map[int]iceberg.PartitionSpec{spec.ID(): spec}
dfBuilder, err := iceberg.NewDataFileBuilder(spec, iceberg.EntryContentData, "data/file.parquet",
iceberg.ParquetFile, map[int]any{1000: iceberg.Date(20737)}, nil, nil, 1, 1)
if err != nil {
t.Fatalf("build data file: %v", err)
}
snapshotID := int64(1)
entry := iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapshotID, nil, nil, dfBuilder.Build())
// valueFirst=true spells the union [<type>, null], hiding the logical type
// from iceberg-go so the partition arrives as a time.Time. That triggers
// normalizePartitionValue → rebuildManifestEntry, which is the path that
// sets block_size_in_bytes.
foreignBytes, foreignManifest := s3tablestest.ForeignPartitionManifest(t, schema, spec, entry,
"metadata/foreign-manifest.avro", "date", time.Date(2026, time.October, 11, 0, 0, 0, 0, time.UTC), true)
entries, err := ReadManifest(foreignManifest, foreignBytes, true, specs, schema)
if err != nil {
t.Fatalf("read foreign manifest: %v", err)
}
var buf bytes.Buffer
if _, err := iceberg.WriteManifest("metadata/manifest.avro", &buf, 1, spec, schema, snapshotID, entries); err != nil {
t.Fatalf("write v1 manifest: %v", err)
}
reader, err := ocf.NewReader(bytes.NewReader(buf.Bytes()))
if err != nil {
t.Fatalf("open v1 manifest: %v", err)
}
var record map[string]any
if err := reader.Decode(&record); err != nil {
t.Fatalf("decode v1 manifest: %v", err)
}
if err := reader.Close(); err != nil {
t.Fatalf("close v1 manifest: %v", err)
}
dataFile, ok := record["data_file"].(map[string]any)
if !ok {
t.Fatalf("data_file = %T, want record", record["data_file"])
}
got, ok := dataFile["block_size_in_bytes"]
if !ok {
t.Fatal("block_size_in_bytes missing from v1 manifest")
}
const want = int64(64 * 1024 * 1024)
if got != want {
t.Fatalf("block_size_in_bytes = %v, want %d", got, want)
}
}

View File

@@ -0,0 +1,139 @@
package s3tables
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
// Manager provides reusable S3 Tables operations for shell/admin without HTTP routing.
type Manager struct {
handler *S3TablesHandler
}
// NewManager creates a new Manager.
func NewManager() *Manager {
m := &Manager{handler: NewS3TablesHandler()}
// Default to allowing access when IAM is not configured
m.handler.SetDefaultAllow(true)
return m
}
// SetRegion sets the AWS region for ARN generation.
func (m *Manager) SetRegion(region string) {
m.handler.SetRegion(region)
}
// SetAccountID sets the AWS account ID for ARN generation.
func (m *Manager) SetAccountID(accountID string) {
m.handler.SetAccountID(accountID)
}
// SetDefaultAllow sets whether to allow access by default.
func (m *Manager) SetDefaultAllow(allow bool) {
m.handler.SetDefaultAllow(allow)
}
// SetTrusted lets trusted local tooling (shell, admin console) bypass authorization.
func (m *Manager) SetTrusted(trusted bool) {
m.handler.SetTrusted(trusted)
}
// Execute runs an S3 Tables operation and decodes the response into resp (if provided).
func (m *Manager) Execute(ctx context.Context, filerClient FilerClient, operation string, req interface{}, resp interface{}, identity string) error {
body, err := json.Marshal(req)
if err != nil {
return err
}
httpReq, err := newManagerRequest(ctx, operation, body, identity)
if err != nil {
return err
}
recorder := httptest.NewRecorder()
m.handler.HandleRequest(recorder, httpReq, filerClient)
return decodeS3TablesHTTPResponse(recorder, resp)
}
// AuthorizeCreateTable checks that identity may create the table the request
// describes, without creating anything. A deferred create (Iceberg
// stage-create) writes into the table bucket long before it registers the
// table, so it passes this gate first.
func (m *Manager) AuthorizeCreateTable(ctx context.Context, filerClient FilerClient, req *CreateTableRequest, identity string) error {
httpReq, err := newManagerRequest(ctx, "CreateTable", nil, identity)
if err != nil {
return err
}
recorder := httptest.NewRecorder()
_, authErr := m.handler.authorizeCreateTable(recorder, httpReq, filerClient, req.TableBucketARN, req.Namespace, req.Name, req.Tags)
if authErr == nil {
return nil
}
if decoded := decodeS3TablesHTTPResponse(recorder, nil); decoded != nil {
return decoded
}
return authErr
}
func newManagerRequest(ctx context.Context, operation string, body []byte, identity string) (*http.Request, error) {
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, "/", bytes.NewReader(body))
if err != nil {
return nil, err
}
httpReq.Header.Set("Content-Type", "application/x-amz-json-1.1")
httpReq.Header.Set("X-Amz-Target", "S3Tables."+operation)
if identity != "" {
httpReq.Header.Set(s3_constants.AmzAccountId, identity)
httpReq = httpReq.WithContext(s3_constants.SetIdentityNameInContext(httpReq.Context(), identity))
}
return httpReq, nil
}
func decodeS3TablesHTTPResponse(recorder *httptest.ResponseRecorder, resp interface{}) error {
result := recorder.Result()
defer result.Body.Close()
data, err := io.ReadAll(result.Body)
if err != nil {
return err
}
if result.StatusCode >= http.StatusBadRequest {
var errResp S3TablesError
if len(data) > 0 {
if jsonErr := json.Unmarshal(data, &errResp); jsonErr == nil && (errResp.Type != "" || errResp.Message != "") {
return &errResp
}
}
return &S3TablesError{Type: ErrCodeInternalError, Message: string(bytes.TrimSpace(data))}
}
if resp == nil || len(data) == 0 {
return nil
}
if err := json.Unmarshal(data, resp); err != nil {
return err
}
return nil
}
// ManagerClient adapts a SeaweedFilerClient to the FilerClient interface.
type ManagerClient struct {
client filer_pb.SeaweedFilerClient
}
// NewManagerClient wraps a filer client.
func NewManagerClient(client filer_pb.SeaweedFilerClient) *ManagerClient {
return &ManagerClient{client: client}
}
// WithFilerClient implements FilerClient.
func (m *ManagerClient) WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
if m.client == nil {
return errors.New("nil filer client")
}
return fn(m.client)
}

View File

@@ -0,0 +1,99 @@
package s3tables
import (
"context"
"errors"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var errFilerReached = errors.New("filer reached")
// recordingFilerClient reports whether the handler reached the filer. The
// CreateTableBucket handler authorizes before touching the filer, so "filer
// reached" is a proxy for "authorization passed".
type recordingFilerClient struct {
called bool
}
func (c *recordingFilerClient) WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
c.called = true
return errFilerReached
}
// The Manager path (used by the Iceberg catalog) enforces authorization for
// authenticated callers and only falls open for trusted/zero-config access.
func TestManagerCreateTableBucketAuthorization(t *testing.T) {
lowPriv := &testIdentity{
Name: "alice",
Account: &testIdentityAccount{Id: s3_constants.AccountAdminId},
Actions: []string{"Read"},
}
cases := []struct {
name string
defaultAllow bool
trusted bool
ctx context.Context
identity string
wantFiler bool // true => authorization passed (filer reached)
}{
{
name: "authenticated identity struct is enforced",
defaultAllow: true,
ctx: s3_constants.SetIdentityInContext(context.Background(), lowPriv),
identity: "alice",
wantFiler: false,
},
{
name: "secured manager denies a name without struct",
defaultAllow: false,
ctx: context.Background(),
identity: "alice",
wantFiler: false,
},
{
name: "untrusted name without struct is enforced",
defaultAllow: true,
ctx: context.Background(),
identity: "alice",
wantFiler: false,
},
{
name: "trusted manager allows a name without struct",
defaultAllow: true,
trusted: true,
ctx: context.Background(),
identity: "alice",
wantFiler: true,
},
{
name: "admin principal is allowed",
defaultAllow: false,
ctx: context.Background(),
identity: s3_constants.AccountAdminId,
wantFiler: true,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
m := NewManager()
m.SetDefaultAllow(tc.defaultAllow)
m.SetTrusted(tc.trusted)
fc := &recordingFilerClient{}
err := m.Execute(tc.ctx, fc, "CreateTableBucket", &CreateTableBucketRequest{Name: "testbucket"}, nil, tc.identity)
require.Error(t, err) // denied (403) or the filer sentinel (reached); never nil here
assert.Equal(t, tc.wantFiler, fc.called, "filer reached should equal authorization passed")
if !tc.wantFiler {
var s3Err *S3TablesError
require.ErrorAs(t, err, &s3Err)
assert.Equal(t, ErrCodeAccessDenied, s3Err.Type)
}
})
}
}

View File

@@ -0,0 +1,435 @@
package s3tables
import (
"encoding/json"
"fmt"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
)
// Permission represents a specific action permission
type Permission string
// IAM Policy structures for evaluation
type PolicyDocument struct {
Version string `json:"Version"`
Statement []Statement `json:"Statement"`
}
// UnmarshalJSON handles both single statement object and array of statements
// AWS allows {"Statement": {...}} or {"Statement": [{...}]}
func (pd *PolicyDocument) UnmarshalJSON(data []byte) error {
type Alias PolicyDocument
aux := &struct {
Statement interface{} `json:"Statement"`
*Alias
}{
Alias: (*Alias)(pd),
}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
// Handle Statement as either a single object or array
switch s := aux.Statement.(type) {
case map[string]interface{}:
// Single statement object - unmarshal to one Statement
stmtData, err := json.Marshal(s)
if err != nil {
return fmt.Errorf("failed to marshal single statement: %w", err)
}
var stmt Statement
if err := json.Unmarshal(stmtData, &stmt); err != nil {
return fmt.Errorf("failed to unmarshal single statement: %w", err)
}
pd.Statement = []Statement{stmt}
case []interface{}:
// Array of statements - normal handling
stmtData, err := json.Marshal(s)
if err != nil {
return fmt.Errorf("failed to marshal statement array: %w", err)
}
if err := json.Unmarshal(stmtData, &pd.Statement); err != nil {
return fmt.Errorf("failed to unmarshal statement array: %w", err)
}
case nil:
// No statements
pd.Statement = []Statement{}
default:
return fmt.Errorf("Statement must be an object or array, got %T", aux.Statement)
}
return nil
}
type Statement struct {
Effect string `json:"Effect"` // "Allow" or "Deny"
Principal interface{} `json:"Principal"` // Can be string, []string, or map
Action interface{} `json:"Action"` // Can be string or []string
Resource interface{} `json:"Resource"` // Can be string or []string
Condition map[string]map[string]interface{} `json:"Condition,omitempty"`
}
type PolicyContext struct {
Namespace string
TableName string
TableBucketName string
IdentityActions []string
RequestTags map[string]string
ResourceTags map[string]string
TableBucketTags map[string]string
TagKeys []string
SSEAlgorithm string
KMSKeyArn string
StorageClass string
DefaultAllow bool
}
// CheckPermissionWithContext checks permission with optional resource and condition context.
func CheckPermissionWithContext(operation, principal, owner, resourcePolicy, resourceARN string, ctx *PolicyContext) bool {
// Deny access if identities are empty
if principal == "" || owner == "" {
return false
}
// Admin always has permission.
if principal == s3_constants.AccountAdminId {
return true
}
glog.V(2).Infof("S3Tables: CheckPermission operation=%s principal=%s owner=%s", operation, principal, owner)
return checkPermission(operation, principal, owner, resourcePolicy, resourceARN, ctx)
}
func checkPermission(operation, principal, owner, resourcePolicy, resourceARN string, ctx *PolicyContext) bool {
// Owner always has permission
if principal == owner {
return true
}
if hasIdentityPermission(operation, ctx) {
return true
}
// If no policy is provided, use default allow if enabled
if resourcePolicy == "" {
if ctx != nil && ctx.DefaultAllow {
return true
}
return false
}
// Normalize operation to full IAM-style action name (e.g., "s3tables:CreateTableBucket")
// if not already prefixed
fullAction := operation
if !strings.Contains(operation, ":") {
fullAction = "s3tables:" + operation
}
// Parse and evaluate policy
var policy PolicyDocument
if err := json.Unmarshal([]byte(resourcePolicy), &policy); err != nil {
return false
}
// Evaluate policy statements
// Default is deny, so we need an explicit allow
hasAllow := false
for _, stmt := range policy.Statement {
// Check if principal matches
if !matchesPrincipal(stmt.Principal, principal) {
continue
}
// Check if action matches (using normalized full action name)
if !matchesAction(stmt.Action, fullAction) {
continue
}
// Check if resource matches (if resourceARN specified and Resource field exists)
if resourceARN != "" && !matchesResource(stmt.Resource, resourceARN) {
continue
}
if !matchesConditions(stmt.Condition, ctx) {
continue
}
// Statement matches - check effect
switch stmt.Effect {
case "Allow":
hasAllow = true
case "Deny":
// Explicit deny always wins
return false
}
}
if hasAllow {
return true
}
// If no statement matched, use default allow if enabled
if ctx != nil && ctx.DefaultAllow {
return true
}
return false
}
func hasIdentityPermission(operation string, ctx *PolicyContext) bool {
if ctx == nil || len(ctx.IdentityActions) == 0 {
return false
}
fullAction := operation
if !strings.Contains(operation, ":") {
fullAction = "s3tables:" + operation
}
if hasAdminAction(ctx.IdentityActions) {
return true
}
candidates := []string{operation, fullAction}
if ctx.TableBucketName != "" {
candidates = append(candidates, operation+":"+ctx.TableBucketName, fullAction+":"+ctx.TableBucketName)
}
for _, action := range ctx.IdentityActions {
for _, candidate := range candidates {
if action == candidate {
return true
}
if strings.ContainsAny(action, "*?") && wildcard.MatchesWildcard(action, candidate) {
return true
}
}
}
return false
}
// hasAdminAction reports whether the action list grants blanket admin access.
// Legacy static identities may use broad markers or s3 wildcards; s3:* is treated
// as s3tables:* so shared admin policies still permit table access.
func hasAdminAction(actions []string) bool {
for _, action := range actions {
if action == "*" || action == s3_constants.ACTION_ADMIN || action == "s3:*" || action == "s3tables:*" {
return true
}
}
return false
}
// matchesPrincipal checks if the principal matches the statement's principal
func matchesPrincipal(principalSpec interface{}, principal string) bool {
if principalSpec == nil {
return false
}
switch p := principalSpec.(type) {
case string:
// Direct string match or wildcard
if p == "*" || p == principal {
return true
}
// Support wildcard matching for principals (e.g., "arn:aws:iam::*:user/admin")
return wildcard.MatchesWildcard(p, principal)
case []interface{}:
// Array of principals
for _, item := range p {
if str, ok := item.(string); ok {
if str == "*" || str == principal {
return true
}
// Support wildcard matching
if wildcard.MatchesWildcard(str, principal) {
return true
}
}
}
case map[string]interface{}:
// AWS-style principal with service prefix, e.g., {"AWS": "arn:aws:iam::..."}
// For S3 Tables, we primarily care about the AWS key
if aws, ok := p["AWS"]; ok {
return matchesPrincipal(aws, principal)
}
}
return false
}
// matchesAction checks if the action matches the statement's action
func matchesAction(actionSpec interface{}, action string) bool {
if actionSpec == nil {
return false
}
switch a := actionSpec.(type) {
case string:
// Direct match or wildcard
return matchesActionPattern(a, action)
case []interface{}:
// Array of actions
for _, item := range a {
if str, ok := item.(string); ok {
if matchesActionPattern(str, action) {
return true
}
}
}
}
return false
}
// matchesActionPattern checks if an action matches a pattern (supports wildcards)
// This uses the policy_engine.MatchesWildcard function for full wildcard support,
// including middle wildcards (e.g., "s3tables:Get*Table") for complete IAM compatibility.
func matchesActionPattern(pattern, action string) bool {
if pattern == "*" {
return true
}
// Exact match
if pattern == action {
return true
}
// Wildcard match using policy engine's wildcard matcher
// Supports both * (any sequence) and ? (single character) anywhere in the pattern
return wildcard.MatchesWildcard(pattern, action)
}
func matchesConditions(conditions map[string]map[string]interface{}, ctx *PolicyContext) bool {
if len(conditions) == 0 {
return true
}
if ctx == nil {
return false
}
for operator, conditionValues := range conditions {
if !matchesConditionOperator(operator, conditionValues, ctx) {
return false
}
}
return true
}
func matchesConditionOperator(operator string, conditionValues map[string]interface{}, ctx *PolicyContext) bool {
evaluator, err := policy_engine.GetConditionEvaluator(operator)
if err != nil {
return false
}
for key, value := range conditionValues {
contextVals := getConditionContextValues(key, ctx)
if !evaluator.Evaluate(value, contextVals) {
return false
}
}
return true
}
func getConditionContextValues(key string, ctx *PolicyContext) []string {
switch key {
case "s3tables:namespace":
return []string{ctx.Namespace}
case "s3tables:tableName":
return []string{ctx.TableName}
case "s3tables:tableBucketName":
return []string{ctx.TableBucketName}
case "s3tables:SSEAlgorithm":
return []string{ctx.SSEAlgorithm}
case "s3tables:KMSKeyArn":
return []string{ctx.KMSKeyArn}
case "s3tables:StorageClass":
return []string{ctx.StorageClass}
case "aws:TagKeys":
return ctx.TagKeys
}
if strings.HasPrefix(key, "aws:RequestTag/") {
tagKey := strings.TrimPrefix(key, "aws:RequestTag/")
if val, ok := ctx.RequestTags[tagKey]; ok {
return []string{val}
}
}
if strings.HasPrefix(key, "aws:ResourceTag/") {
tagKey := strings.TrimPrefix(key, "aws:ResourceTag/")
if val, ok := ctx.ResourceTags[tagKey]; ok {
return []string{val}
}
}
if strings.HasPrefix(key, "s3tables:TableBucketTag/") {
tagKey := strings.TrimPrefix(key, "s3tables:TableBucketTag/")
if val, ok := ctx.TableBucketTags[tagKey]; ok {
return []string{val}
}
}
return nil
}
// matchesResource checks if the resource ARN matches the statement's resource specification
// Returns true if resource matches or if Resource is not specified (implicit match)
func matchesResource(resourceSpec interface{}, resourceARN string) bool {
// If no Resource is specified, match all resources (implicit *)
if resourceSpec == nil {
return true
}
switch r := resourceSpec.(type) {
case string:
// Direct match or wildcard
return matchesResourcePattern(r, resourceARN)
case []interface{}:
// Array of resources - match if any matches
for _, item := range r {
if str, ok := item.(string); ok {
if matchesResourcePattern(str, resourceARN) {
return true
}
}
}
}
return false
}
// matchesResourcePattern checks if a resource ARN matches a pattern (supports wildcards)
func matchesResourcePattern(pattern, resourceARN string) bool {
if pattern == "*" {
return true
}
// Exact match
if pattern == resourceARN {
return true
}
// Wildcard match using policy engine's wildcard matcher
return wildcard.MatchesWildcard(pattern, resourceARN)
}
// AuthError represents an authorization error
type AuthError struct {
Operation string
Principal string
Message string
}
func (e *AuthError) Error() string {
return "unauthorized: " + e.Principal + " is not permitted to perform " + e.Operation + ": " + e.Message
}
// NewAuthError creates a new authorization error
func NewAuthError(operation, principal, message string) *AuthError {
return &AuthError{
Operation: operation,
Principal: principal,
Message: message,
}
}

View File

@@ -0,0 +1,253 @@
package s3tables
import (
"encoding/json"
"testing"
)
func TestMatchesActionPattern(t *testing.T) {
tests := []struct {
name string
pattern string
action string
expected bool
}{
// Exact matches
{"exact match", "GetTable", "GetTable", true},
{"no match", "GetTable", "DeleteTable", false},
// Universal wildcard
{"universal wildcard", "*", "anything", true},
// Suffix wildcards
{"suffix wildcard match", "s3tables:*", "s3tables:GetTable", true},
{"suffix wildcard no match", "s3tables:*", "iam:GetUser", false},
// Middle wildcards (new capability from policy_engine)
{"middle wildcard Get*Table", "s3tables:Get*Table", "s3tables:GetTable", true},
{"middle wildcard Get*Table no match GetTableBucket", "s3tables:Get*Table", "s3tables:GetTableBucket", false},
{"middle wildcard Get*Table no match DeleteTable", "s3tables:Get*Table", "s3tables:DeleteTable", false},
{"middle wildcard *Table*", "s3tables:*Table*", "s3tables:GetTableBucket", true},
{"middle wildcard *Table* match CreateTable", "s3tables:*Table*", "s3tables:CreateTable", true},
// Question mark wildcards
{"question mark single char", "GetTable?", "GetTableX", true},
{"question mark no match", "GetTable?", "GetTableXY", false},
// Combined wildcards
{"combined * and ? singular", "s3tables:Get?able*", "s3tables:GetTable", true},
{"combined * and ? plural", "s3tables:Get?able*", "s3tables:GetTables", true},
{"combined no match - ? needs 1 char", "s3tables:Get?able*", "s3tables:Getable", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := matchesActionPattern(tt.pattern, tt.action)
if result != tt.expected {
t.Errorf("matchesActionPattern(%q, %q) = %v, want %v", tt.pattern, tt.action, result, tt.expected)
}
})
}
}
func TestMatchesPrincipal(t *testing.T) {
tests := []struct {
name string
principalSpec interface{}
principal string
expected bool
}{
// String principals
{"exact match", "user123", "user123", true},
{"no match", "user123", "user456", false},
{"universal wildcard", "*", "anyone", true},
// Wildcard principals
{"prefix wildcard", "arn:aws:iam::123456789012:user/*", "arn:aws:iam::123456789012:user/admin", true},
{"prefix wildcard no match", "arn:aws:iam::123456789012:user/*", "arn:aws:iam::987654321098:user/admin", false},
{"middle wildcard", "arn:aws:iam::*:user/admin", "arn:aws:iam::123456789012:user/admin", true},
// Array of principals
{"array match first", []interface{}{"user1", "user2"}, "user1", true},
{"array match second", []interface{}{"user1", "user2"}, "user2", true},
{"array no match", []interface{}{"user1", "user2"}, "user3", false},
{"array wildcard", []interface{}{"user1", "arn:aws:iam::*:user/admin"}, "arn:aws:iam::123:user/admin", true},
// Map-style AWS principals
{"AWS map exact", map[string]interface{}{"AWS": "user123"}, "user123", true},
{"AWS map wildcard", map[string]interface{}{"AWS": "arn:aws:iam::*:user/admin"}, "arn:aws:iam::123:user/admin", true},
{"AWS map array", map[string]interface{}{"AWS": []interface{}{"user1", "user2"}}, "user1", true},
// Nil/empty cases
{"nil principal", nil, "user123", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := matchesPrincipal(tt.principalSpec, tt.principal)
if result != tt.expected {
t.Errorf("matchesPrincipal(%v, %q) = %v, want %v", tt.principalSpec, tt.principal, result, tt.expected)
}
})
}
}
func TestEvaluatePolicyWithConditions(t *testing.T) {
policy := &PolicyDocument{
Statement: []Statement{
{
Effect: "Allow",
Principal: "*",
Action: "s3tables:GetTable",
Condition: map[string]map[string]interface{}{
"StringEquals": {
"s3tables:namespace": "default",
},
"StringLike": {
"s3tables:tableName": "test_*",
},
"NumericGreaterThan": {
"aws:RequestTag/priority": "10",
},
"Bool": {
"aws:ResourceTag/is_public": "true",
},
},
},
},
}
policyBytes, _ := json.Marshal(policy)
policyStr := string(policyBytes)
tests := []struct {
name string
ctx *PolicyContext
expected bool
}{
{
"all conditions match",
&PolicyContext{
Namespace: "default",
TableName: "test_table",
RequestTags: map[string]string{
"priority": "15",
},
ResourceTags: map[string]string{
"is_public": "true",
},
},
true,
},
{
"namespace mismatch",
&PolicyContext{
Namespace: "other",
TableName: "test_table",
RequestTags: map[string]string{
"priority": "15",
},
ResourceTags: map[string]string{
"is_public": "true",
},
},
false,
},
{
"table name mismatch",
&PolicyContext{
Namespace: "default",
TableName: "other_table",
RequestTags: map[string]string{
"priority": "15",
},
ResourceTags: map[string]string{
"is_public": "true",
},
},
false,
},
{
"numeric condition failure",
&PolicyContext{
Namespace: "default",
TableName: "test_table",
RequestTags: map[string]string{
"priority": "5",
},
ResourceTags: map[string]string{
"is_public": "true",
},
},
false,
},
{
"bool condition failure",
&PolicyContext{
Namespace: "default",
TableName: "test_table",
RequestTags: map[string]string{
"priority": "15",
},
ResourceTags: map[string]string{
"is_public": "false",
},
},
false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// principal="user123", owner="owner123"
result := CheckPermissionWithContext("s3tables:GetTable", "user123", "owner123", policyStr, "", tt.ctx)
if result != tt.expected {
t.Errorf("CheckPermissionWithContext() = %v, want %v", result, tt.expected)
}
})
}
}
func TestCheckPermissionWithDefaultAllow(t *testing.T) {
tests := []struct {
name string
defaultAllow bool
policy string
expected bool
}{
{
"default deny (no policy, DefaultAllow=false)",
false,
"",
false,
},
{
"default allow (no policy, DefaultAllow=true)",
true,
"",
true,
},
{
"explicit deny overrides DefaultAllow=true",
true,
`{"Statement":[{"Effect":"Deny","Principal":"*","Action":"s3tables:GetTable"}]}`,
false,
},
{
"explicit allow works with DefaultAllow=false",
false,
`{"Statement":[{"Effect":"Allow","Principal":"*","Action":"s3tables:GetTable"}]}`,
true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := &PolicyContext{
DefaultAllow: tt.defaultAllow,
}
result := CheckPermissionWithContext("s3tables:GetTable", "user123", "owner123", tt.policy, "", ctx)
if result != tt.expected {
t.Errorf("CheckPermissionWithContext() = %v, want %v (DefaultAllow=%v, Policy=%s)", result, tt.expected, tt.defaultAllow, tt.policy)
}
})
}
}

View File

@@ -0,0 +1,150 @@
package s3tablestest
import (
"bytes"
"encoding/json"
"testing"
"github.com/apache/iceberg-go"
"github.com/twmb/avro"
"github.com/twmb/avro/ocf"
)
// ForeignPartitionManifest rewrites a manifest iceberg-go wrote so its
// partition field carries the Avro union a foreign writer emits: [<type>, null]
// rather than [null, <type>]. Both are valid Avro, but iceberg-go reads a
// partition field's logical type from the union's last branch, so the ordering
// decides whether it hands back an Iceberg value or the rich Go value the Avro
// decoder produced.
//
// logicalType is stamped onto the value branch, since iceberg-go writes a day
// partition as a bare int, and decoded replaces the partition value in the
// record, standing in for what the foreign writer encoded. valueFirst orders
// the union [<type>, null] rather than the [null, <type>] Java and iceberg-rust
// emit.
func ForeignPartitionManifest(
t *testing.T,
schema *iceberg.Schema,
spec iceberg.PartitionSpec,
entry iceberg.ManifestEntry,
manifestPath, logicalType string,
decoded any,
valueFirst bool,
) ([]byte, iceberg.ManifestFile) {
t.Helper()
var original bytes.Buffer
manifest, err := iceberg.WriteManifest(manifestPath, &original, 2, spec, schema, 1, []iceberg.ManifestEntry{entry})
if err != nil {
t.Fatalf("write base manifest: %v", err)
}
reader, err := ocf.NewReader(bytes.NewReader(original.Bytes()))
if err != nil {
t.Fatalf("open base manifest: %v", err)
}
metadata := reader.Metadata()
var record map[string]any
if err := reader.Decode(&record); err != nil {
t.Fatalf("decode base manifest: %v", err)
}
if err := reader.Close(); err != nil {
t.Fatalf("close base manifest: %v", err)
}
fieldName := spec.Field(0).Name
var schemaDoc map[string]any
if err := json.Unmarshal(metadata["avro.schema"], &schemaDoc); err != nil {
t.Fatalf("decode manifest schema: %v", err)
}
dataFile := findAvroField(t, schemaDoc, "data_file")
partition := findAvroField(t, dataFile, "partition")
partitionFields, ok := partition["fields"].([]any)
if !ok {
t.Fatalf("partition schema fields have type %T", partition["fields"])
}
for _, rawField := range partitionFields {
field, ok := rawField.(map[string]any)
if !ok || field["name"] != fieldName {
continue
}
fieldType, ok := field["type"].([]any)
if !ok || len(fieldType) != 2 {
t.Fatalf("%s schema type = %#v, want nullable union", fieldName, field["type"])
}
valueType, ok := fieldType[1].(map[string]any)
if !ok {
primitive, primitiveOK := fieldType[1].(string)
if !primitiveOK {
t.Fatalf("%s value branch = %T, want Avro type", fieldName, fieldType[1])
}
valueType = map[string]any{"type": primitive}
}
valueType["logicalType"] = logicalType
if valueFirst {
field["type"] = []any{valueType, "null"}
} else {
field["type"] = []any{"null", valueType}
}
break
}
dataFileRecord, ok := record["data_file"].(map[string]any)
if !ok {
t.Fatalf("decoded data_file = %T, want record", record["data_file"])
}
partitionRecord, ok := dataFileRecord["partition"].(map[string]any)
if !ok {
t.Fatalf("decoded partition = %T, want record", dataFileRecord["partition"])
}
partitionRecord[fieldName] = decoded
foreignSchemaJSON, err := json.Marshal(schemaDoc)
if err != nil {
t.Fatalf("encode foreign manifest schema: %v", err)
}
foreignSchema, err := avro.Parse(string(foreignSchemaJSON))
if err != nil {
t.Fatalf("parse foreign manifest schema: %v", err)
}
userMetadata := make(map[string][]byte)
for key, value := range metadata {
if key != "avro.schema" && key != "avro.codec" {
userMetadata[key] = value
}
}
var foreign bytes.Buffer
writer, err := ocf.NewWriter(&foreign, foreignSchema, ocf.WithSchema(string(foreignSchemaJSON)), ocf.WithMetadata(userMetadata))
if err != nil {
t.Fatalf("create foreign manifest writer: %v", err)
}
if err := writer.Encode(record); err != nil {
t.Fatalf("encode foreign manifest: %v", err)
}
if err := writer.Close(); err != nil {
t.Fatalf("close foreign manifest writer: %v", err)
}
return foreign.Bytes(), iceberg.NewManifestFile(2, manifest.FilePath(), int64(foreign.Len()), manifest.PartitionSpecID(), 1).AddedFiles(1).SequenceNum(1, 1).Build()
}
func findAvroField(t *testing.T, record map[string]any, name string) map[string]any {
t.Helper()
fields, ok := record["fields"].([]any)
if !ok {
t.Fatalf("Avro record fields have type %T", record["fields"])
}
for _, rawField := range fields {
field, ok := rawField.(map[string]any)
if ok && field["name"] == name {
fieldType, ok := field["type"].(map[string]any)
if ok {
return fieldType
}
t.Fatalf("Avro field %q type has type %T", name, field["type"])
}
}
t.Fatalf("Avro field %q not found", name)
return nil
}

View File

@@ -0,0 +1,242 @@
// Package s3tablestest provides an in-memory filer for driving S3 Tables and
// Lance namespace operations end-to-end without a live cluster.
package s3tablestest
import (
"bytes"
"context"
"net"
"path"
"sort"
"strings"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
)
// MemFiler is an in-memory filer used to drive Manager operations
// end-to-end without a live cluster.
type MemFiler struct {
filer_pb.UnimplementedSeaweedFilerServer
// mu guards entries. The real filer serves concurrent RPCs, and a test that
// races writers against each other - the point of an exclusive create - hits
// this map from several goroutines at once.
mu sync.RWMutex
entries map[string]map[string]*filer_pb.Entry // dir -> name -> entry
Client filer_pb.SeaweedFilerClient
// BeforeUpdate runs once, at the start of the next UpdateEntry, so a test
// can land a competing write in a handler's read-to-write window.
BeforeUpdate func()
// RejectDelete returns the reason DeleteEntry refuses a path, answered the
// way the filer answers one: no transport error, the reason in the response.
RejectDelete func(dir, name string) string
}
func newMemFiler() *MemFiler {
return &MemFiler{entries: make(map[string]map[string]*filer_pb.Entry)}
}
func (f *MemFiler) Get(dir, name string) *filer_pb.Entry {
f.mu.RLock()
defer f.mu.RUnlock()
if d, ok := f.entries[dir]; ok {
return d[name]
}
return nil
}
func (f *MemFiler) Put(dir, name string, extended map[string][]byte) {
f.mu.Lock()
defer f.mu.Unlock()
if _, ok := f.entries[dir]; !ok {
f.entries[dir] = make(map[string]*filer_pb.Entry)
}
f.entries[dir][name] = &filer_pb.Entry{Name: name, IsDirectory: true, Extended: extended}
}
// PutFile adds a file entry with an explicit modification time, which callers
// that age entries out (orphan cleanup, expiry) need in order to see them.
func (f *MemFiler) PutFile(dir, name string, mtime time.Time) {
f.mu.Lock()
defer f.mu.Unlock()
if _, ok := f.entries[dir]; !ok {
f.entries[dir] = make(map[string]*filer_pb.Entry)
}
f.entries[dir][name] = &filer_pb.Entry{
Name: name,
Attributes: &filer_pb.FuseAttributes{Mtime: mtime.Unix(), Crtime: mtime.Unix()},
}
}
func (f *MemFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
if e := f.Get(req.Directory, req.Name); e != nil {
return &filer_pb.LookupDirectoryEntryResponse{Entry: e}, nil
}
// Carry the sentinel text so filer_pb.LookupEntry maps it to ErrNotFound.
return nil, status.Errorf(codes.NotFound, "%s: %s/%s", filer_pb.ErrNotFound.Error(), req.Directory, req.Name)
}
// ListEntries honours prefix, start-from and limit the way the real filer does.
// A harness that ignores them makes a paginating caller re-read the first page
// forever, which looks like duplicated entries rather than a broken listing.
func (f *MemFiler) ListEntries(req *filer_pb.ListEntriesRequest, stream grpc.ServerStreamingServer[filer_pb.ListEntriesResponse]) error {
f.mu.RLock()
d, ok := f.entries[req.Directory]
names := make([]string, 0, len(d))
snapshot := make(map[string]*filer_pb.Entry, len(d))
for name, entry := range d {
names = append(names, name)
snapshot[name] = entry
}
f.mu.RUnlock()
if !ok {
return nil
}
sort.Strings(names)
sent := uint32(0)
for _, name := range names {
if req.Prefix != "" && !strings.HasPrefix(name, req.Prefix) {
continue
}
if req.StartFromFileName != "" {
if name < req.StartFromFileName {
continue
}
if name == req.StartFromFileName && !req.InclusiveStartFrom {
continue
}
}
if err := stream.Send(&filer_pb.ListEntriesResponse{Entry: snapshot[name]}); err != nil {
return err
}
sent++
if req.Limit > 0 && sent >= req.Limit {
return nil
}
}
return nil
}
func (f *MemFiler) CreateEntry(_ context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
f.mu.Lock()
defer f.mu.Unlock()
if _, ok := f.entries[req.Directory]; !ok {
f.entries[req.Directory] = make(map[string]*filer_pb.Entry)
}
// O_EXCL is the filer's put-if-not-exists. Ignoring it here would let a test
// that races two writers see both of them win.
if _, exists := f.entries[req.Directory][req.Entry.Name]; exists && req.OExcl {
return &filer_pb.CreateEntryResponse{ErrorCode: filer_pb.FilerError_ENTRY_ALREADY_EXISTS}, nil
}
f.entries[req.Directory][req.Entry.Name] = req.Entry
return &filer_pb.CreateEntryResponse{}, nil
}
func (f *MemFiler) UpdateEntry(_ context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
// The hook runs before the lock is taken: its whole purpose is to land a
// competing write in the read-to-write window, and that write needs the lock.
if hook := f.BeforeUpdate; hook != nil {
f.BeforeUpdate = nil
hook()
}
f.mu.Lock()
defer f.mu.Unlock()
// The real filer validates ExpectedExtended under the per-path lock; without
// it here a lost update would look like a success.
for key, expected := range req.ExpectedExtended {
var actual []byte
if d, ok := f.entries[req.Directory]; ok {
if existing := d[req.Entry.Name]; existing != nil {
actual = existing.Extended[key]
}
}
if !bytes.Equal(actual, expected) {
return nil, status.Errorf(codes.FailedPrecondition, "extended attribute %q changed", key)
}
}
if _, ok := f.entries[req.Directory]; !ok {
f.entries[req.Directory] = make(map[string]*filer_pb.Entry)
}
f.entries[req.Directory][req.Entry.Name] = req.Entry
return &filer_pb.UpdateEntryResponse{}, nil
}
func (f *MemFiler) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) {
if reject := f.RejectDelete; reject != nil {
if reason := reject(req.Directory, req.Name); reason != "" {
return &filer_pb.DeleteEntryResponse{Error: reason}, nil
}
}
f.mu.Lock()
defer f.mu.Unlock()
if d, ok := f.entries[req.Directory]; ok {
delete(d, req.Name)
}
// Honor recursive data deletion so a regression that wipes the table directory
// also drops its metadata/ and data/ children (the data-loss this guards against).
if req.IsRecursive && req.IsDeleteData {
child := path.Join(req.Directory, req.Name)
for dir := range f.entries {
if dir == child || strings.HasPrefix(dir, child+"/") {
delete(f.entries, dir)
}
}
}
return &filer_pb.DeleteEntryResponse{}, nil
}
// GetFilerConfiguration answers with the defaults, so operations that resolve
// the buckets directory before touching an entry work against this filer.
func (f *MemFiler) GetFilerConfiguration(_ context.Context, _ *filer_pb.GetFilerConfigurationRequest) (*filer_pb.GetFilerConfigurationResponse, error) {
return &filer_pb.GetFilerConfigurationResponse{DirBuckets: s3_constants.DefaultBucketsPath}, nil
}
func (f *MemFiler) Ping(_ context.Context, _ *filer_pb.PingRequest) (*filer_pb.PingResponse, error) {
now := time.Now().UnixNano()
return &filer_pb.PingResponse{StartTimeNs: now, RemoteTimeNs: now, StopTimeNs: now}, nil
}
func Start(t *testing.T) *MemFiler {
t.Helper()
fs := newMemFiler()
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("start filer: %v", err)
}
server := grpc.NewServer()
filer_pb.RegisterSeaweedFilerServer(server, fs)
go func() { _ = server.Serve(listener) }()
t.Cleanup(server.GracefulStop)
conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("start filer: %v", err)
}
t.Cleanup(func() { _ = conn.Close() })
fs.Client = filer_pb.NewSeaweedFilerClient(conn)
deadline := time.Now().Add(5 * time.Second)
for {
pingCtx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
_, err := fs.Client.Ping(pingCtx, &filer_pb.PingRequest{})
cancel()
if err == nil {
break
}
if time.Now().After(deadline) {
t.Fatalf("filer not ready: %v", err)
}
time.Sleep(10 * time.Millisecond)
}
return fs
}

View File

@@ -0,0 +1,37 @@
package s3tables
import (
"path"
"testing"
)
func TestTableDataDirFromMetadataLocation(t *testing.T) {
cases := []struct {
loc string
want string
}{
{"s3://warehouse/sales/orders/metadata/v1.metadata.json", path.Join(TablesPath, "warehouse/sales/orders")},
{"s3://warehouse/sales/orders/metadata/00003-9f1c.metadata.json", path.Join(TablesPath, "warehouse/sales/orders")},
{"s3://warehouse/ns/tbl", path.Join(TablesPath, "warehouse/ns/tbl")},
{"", ""},
}
for _, c := range cases {
if got := TableDataDirFromMetadataLocation(c.loc); got != c.want {
t.Errorf("TableDataDirFromMetadataLocation(%q) = %q, want %q", c.loc, got, c.want)
}
}
}
func TestMetadataVersionFromLocationUniqueSuffix(t *testing.T) {
cases := map[string]int{
"s3://bkt/ns/t/metadata/v4.metadata.json": 4,
"s3://bkt/ns/t/metadata/v4-0a1b2c3d-4e5f-6789-abcd-ef0123456789.metadata.json": 4,
"s3://bkt/ns/t/metadata/00007-0a1b2c3d.metadata.json": 7,
"s3://bkt/ns/t/metadata/whatever.metadata.json": 1,
}
for location, want := range cases {
if got := metadataVersionFromLocation(location); got != want {
t.Errorf("metadataVersionFromLocation(%q) = %d, want %d", location, got, want)
}
}
}

View File

@@ -0,0 +1,626 @@
package s3tables
import (
"encoding/json"
"strings"
"time"
)
// Table bucket types
type TableBucket struct {
ARN string `json:"arn"`
Name string `json:"name"`
OwnerAccountID string `json:"ownerAccountId"`
CreatedAt time.Time `json:"createdAt"`
Format string `json:"format,omitempty"`
}
type CreateTableBucketRequest struct {
Name string `json:"name"`
Tags map[string]string `json:"tags,omitempty"`
// Format is the table format this bucket holds. A bucket is a catalog and a
// catalog serves one protocol, so declaring it here is what lets a caller be
// told where to connect. Empty means ICEBERG, which is what AWS S3 Tables
// serves and therefore what an SDK that has never heard of this field means.
Format string `json:"format,omitempty"`
}
type CreateTableBucketResponse struct {
ARN string `json:"arn"`
}
type GetTableBucketRequest struct {
TableBucketARN string `json:"tableBucketARN"`
}
type GetTableBucketResponse struct {
ARN string `json:"arn"`
Name string `json:"name"`
OwnerAccountID string `json:"ownerAccountId"`
CreatedAt time.Time `json:"createdAt"`
// Format is empty for a bucket created before formats were declared. Such a
// bucket accepts any format, which is what it did when it was made.
Format string `json:"format,omitempty"`
}
type ListTableBucketsRequest struct {
Prefix string `json:"prefix,omitempty"`
ContinuationToken string `json:"continuationToken,omitempty"`
MaxBuckets int `json:"maxBuckets,omitempty"`
}
type TableBucketSummary struct {
ARN string `json:"arn"`
Name string `json:"name"`
CreatedAt time.Time `json:"createdAt"`
Format string `json:"format,omitempty"`
}
type ListTableBucketsResponse struct {
TableBuckets []TableBucketSummary `json:"tableBuckets"`
ContinuationToken string `json:"continuationToken,omitempty"`
}
type DeleteTableBucketRequest struct {
TableBucketARN string `json:"tableBucketARN"`
}
// Table bucket policy types
type PutTableBucketPolicyRequest struct {
TableBucketARN string `json:"tableBucketARN"`
ResourcePolicy string `json:"resourcePolicy"`
}
type GetTableBucketPolicyRequest struct {
TableBucketARN string `json:"tableBucketARN"`
}
type GetTableBucketPolicyResponse struct {
ResourcePolicy string `json:"resourcePolicy"`
}
type DeleteTableBucketPolicyRequest struct {
TableBucketARN string `json:"tableBucketARN"`
}
// Namespace types
type Namespace struct {
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Properties map[string]string `json:"properties,omitempty"`
}
type CreateNamespaceRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Properties map[string]string `json:"properties,omitempty"`
}
type CreateNamespaceResponse struct {
Namespace []string `json:"namespace"`
TableBucketARN string `json:"tableBucketARN"`
Properties map[string]string `json:"properties,omitempty"`
}
type GetNamespaceRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
}
type UpdateNamespaceRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Properties map[string]string `json:"properties,omitempty"`
}
type UpdateNamespaceResponse struct {
Namespace []string `json:"namespace"`
TableBucketARN string `json:"tableBucketARN"`
Properties map[string]string `json:"properties,omitempty"`
}
type GetNamespaceResponse struct {
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Properties map[string]string `json:"properties,omitempty"`
}
type ListNamespacesRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Prefix string `json:"prefix,omitempty"`
ContinuationToken string `json:"continuationToken,omitempty"`
MaxNamespaces int `json:"maxNamespaces,omitempty"`
}
type NamespaceSummary struct {
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
}
type ListNamespacesResponse struct {
Namespaces []NamespaceSummary `json:"namespaces"`
ContinuationToken string `json:"continuationToken,omitempty"`
}
type DeleteNamespaceRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
}
// Table types
type IcebergSchemaField struct {
Name string `json:"name"`
Type string `json:"type"`
Required bool `json:"required,omitempty"`
}
type IcebergSchema struct {
Fields []IcebergSchemaField `json:"fields"`
}
type IcebergMetadata struct {
Schema IcebergSchema `json:"schema"`
TableUUID string `json:"tableUuid,omitempty"`
}
type TableMetadata struct {
Iceberg *IcebergMetadata `json:"iceberg,omitempty"`
FullMetadata json.RawMessage `json:"fullMetadata,omitempty"`
}
type Table struct {
Name string `json:"name"`
TableARN string `json:"tableARN"`
Namespace []string `json:"namespace"`
Format string `json:"format"`
CreatedAt time.Time `json:"createdAt"`
ModifiedAt time.Time `json:"modifiedAt"`
OwnerAccountID string `json:"ownerAccountId"`
MetadataLocation string `json:"metadataLocation,omitempty"`
Metadata *TableMetadata `json:"metadata,omitempty"`
}
type CreateTableRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
Format string `json:"format"`
Metadata *TableMetadata `json:"metadata,omitempty"`
MetadataVersion int `json:"metadataVersion,omitempty"`
MetadataLocation string `json:"metadataLocation,omitempty"`
Tags map[string]string `json:"tags,omitempty"`
}
type CreateTableResponse struct {
TableARN string `json:"tableARN"`
VersionToken string `json:"versionToken"`
MetadataLocation string `json:"metadataLocation,omitempty"`
}
type RegisterTableRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
MetadataLocation string `json:"metadataLocation"`
}
type RegisterTableResponse struct {
TableARN string `json:"tableARN"`
VersionToken string `json:"versionToken"`
MetadataLocation string `json:"metadataLocation,omitempty"`
}
type GetTableRequest struct {
TableBucketARN string `json:"tableBucketARN,omitempty"`
Namespace []string `json:"namespace,omitempty"`
Name string `json:"name,omitempty"`
TableARN string `json:"tableARN,omitempty"`
}
type GetTableResponse struct {
Name string `json:"name"`
TableARN string `json:"tableARN"`
Namespace []string `json:"namespace"`
Format string `json:"format"`
CreatedAt time.Time `json:"createdAt"`
ModifiedAt time.Time `json:"modifiedAt"`
OwnerAccountID string `json:"ownerAccountId"`
MetadataLocation string `json:"metadataLocation,omitempty"`
VersionToken string `json:"versionToken"`
MetadataVersion int `json:"metadataVersion"`
Metadata *TableMetadata `json:"metadata,omitempty"`
}
type ListTablesRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace,omitempty"`
Prefix string `json:"prefix,omitempty"`
ContinuationToken string `json:"continuationToken,omitempty"`
MaxTables int `json:"maxTables,omitempty"`
}
type TableSummary struct {
Name string `json:"name"`
TableARN string `json:"tableARN"`
Namespace []string `json:"namespace"`
// Format lets a caller tell an Iceberg table from a catalog-only one without
// a GetTable per row. AWS omits it; listing a mixed catalog needs it.
Format string `json:"format,omitempty"`
CreatedAt time.Time `json:"createdAt"`
ModifiedAt time.Time `json:"modifiedAt"`
MetadataLocation string `json:"metadataLocation,omitempty"`
}
type ListTablesResponse struct {
Tables []TableSummary `json:"tables"`
ContinuationToken string `json:"continuationToken,omitempty"`
}
type DeleteTableRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
VersionToken string `json:"versionToken,omitempty"`
}
type UpdateTableRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
VersionToken string `json:"versionToken,omitempty"`
Metadata *TableMetadata `json:"metadata,omitempty"`
MetadataVersion int `json:"metadataVersion,omitempty"`
MetadataLocation string `json:"metadataLocation,omitempty"`
}
type UpdateTableResponse struct {
TableARN string `json:"tableARN"`
VersionToken string `json:"versionToken"`
MetadataLocation string `json:"metadataLocation,omitempty"`
}
type RenameTableRequest struct {
TableBucketARN string `json:"tableBucketARN"`
SourceNamespace []string `json:"sourceNamespace"`
SourceName string `json:"sourceName"`
DestNamespace []string `json:"destNamespace"`
DestName string `json:"destName"`
}
type RenameTableResponse struct {
TableARN string `json:"tableARN"`
MetadataLocation string `json:"metadataLocation,omitempty"`
}
// View types
//
// Views are stored exactly like tables (a filer directory carrying a metadata
// pointer xattr) but tagged with ExtendedKeyEntryType="view". They reuse
// TableMetadata for the metadata pointer.
type CreateViewRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
Metadata *TableMetadata `json:"metadata,omitempty"`
MetadataVersion int `json:"metadataVersion,omitempty"`
MetadataLocation string `json:"metadataLocation,omitempty"`
}
type CreateViewResponse struct {
ViewARN string `json:"viewARN"`
VersionToken string `json:"versionToken"`
MetadataLocation string `json:"metadataLocation,omitempty"`
}
type GetViewRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
}
type GetViewResponse struct {
Name string `json:"name"`
ViewARN string `json:"viewARN"`
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
ModifiedAt time.Time `json:"modifiedAt"`
OwnerAccountID string `json:"ownerAccountId"`
MetadataLocation string `json:"metadataLocation,omitempty"`
VersionToken string `json:"versionToken"`
MetadataVersion int `json:"metadataVersion"`
Metadata *TableMetadata `json:"metadata,omitempty"`
}
type ListViewsRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace,omitempty"`
Prefix string `json:"prefix,omitempty"`
ContinuationToken string `json:"continuationToken,omitempty"`
MaxViews int `json:"maxViews,omitempty"`
}
type ViewSummary struct {
Name string `json:"name"`
ViewARN string `json:"viewARN"`
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
ModifiedAt time.Time `json:"modifiedAt"`
MetadataLocation string `json:"metadataLocation,omitempty"`
}
type ListViewsResponse struct {
Views []ViewSummary `json:"views"`
ContinuationToken string `json:"continuationToken,omitempty"`
}
type UpdateViewRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
VersionToken string `json:"versionToken,omitempty"`
Metadata *TableMetadata `json:"metadata,omitempty"`
MetadataVersion int `json:"metadataVersion,omitempty"`
MetadataLocation string `json:"metadataLocation,omitempty"`
}
type UpdateViewResponse struct {
ViewARN string `json:"viewARN"`
VersionToken string `json:"versionToken"`
MetadataLocation string `json:"metadataLocation,omitempty"`
}
type DeleteViewRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
VersionToken string `json:"versionToken,omitempty"`
}
// Table policy types
type PutTablePolicyRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
ResourcePolicy string `json:"resourcePolicy"`
}
type GetTablePolicyRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
}
type GetTablePolicyResponse struct {
ResourcePolicy string `json:"resourcePolicy"`
}
type DeleteTablePolicyRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
}
// Maintenance configuration types
const (
MaintenanceTypeIcebergCompaction = "icebergCompaction"
MaintenanceTypeIcebergSnapshotManagement = "icebergSnapshotManagement"
MaintenanceTypeIcebergUnreferencedFileRemoval = "icebergUnreferencedFileRemoval"
MaintenanceStatusEnabled = "enabled"
MaintenanceStatusDisabled = "disabled"
)
type IcebergCompactionSettings struct {
Strategy string `json:"strategy,omitempty"`
TargetFileSizeMB *int64 `json:"targetFileSizeMB,omitempty"`
}
// Compaction strategies. AWS also defines z-order, which the maintenance
// worker cannot produce, so it is rejected rather than silently binpacked.
const (
CompactionStrategyAuto = "auto"
CompactionStrategyBinpack = "binpack"
CompactionStrategySort = "sort"
CompactionStrategyZOrder = "z-order"
)
type IcebergSnapshotManagementSettings struct {
MinSnapshotsToKeep *int64 `json:"minSnapshotsToKeep,omitempty"`
MaxSnapshotAgeHours *int64 `json:"maxSnapshotAgeHours,omitempty"`
}
type IcebergUnreferencedFileRemovalSettings struct {
UnreferencedDays *int64 `json:"unreferencedDays,omitempty"`
NonCurrentDays *int64 `json:"nonCurrentDays,omitempty"`
}
type MaintenanceSettings struct {
IcebergCompaction *IcebergCompactionSettings `json:"icebergCompaction,omitempty"`
IcebergSnapshotManagement *IcebergSnapshotManagementSettings `json:"icebergSnapshotManagement,omitempty"`
IcebergUnreferencedFileRemoval *IcebergUnreferencedFileRemovalSettings `json:"icebergUnreferencedFileRemoval,omitempty"`
}
type MaintenanceConfigurationValue struct {
Status string `json:"status,omitempty"`
Settings *MaintenanceSettings `json:"settings,omitempty"`
}
// MaintenanceConfiguration maps a maintenance type to its configuration. It is
// stored verbatim as the maintenance extended attribute so Get is a passthrough.
type MaintenanceConfiguration map[string]*MaintenanceConfigurationValue
// MergeMaintenanceConfiguration overlays a table's configuration on its
// bucket's. Unreferenced file removal is only configurable on the bucket, so
// anything reading a table's effective configuration has to consult both.
func MergeMaintenanceConfiguration(bucket, table MaintenanceConfiguration) MaintenanceConfiguration {
if len(bucket) == 0 {
return table
}
if len(table) == 0 {
return bucket
}
merged := make(MaintenanceConfiguration, len(bucket)+len(table))
for k, v := range bucket {
merged[k] = v
}
for k, v := range table {
merged[k] = v
}
return merged
}
type PutTableBucketMaintenanceConfigurationRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Type string `json:"type"`
Value *MaintenanceConfigurationValue `json:"value"`
}
type GetTableBucketMaintenanceConfigurationRequest struct {
TableBucketARN string `json:"tableBucketARN"`
}
type GetTableBucketMaintenanceConfigurationResponse struct {
TableBucketARN string `json:"tableBucketARN"`
Configuration MaintenanceConfiguration `json:"configuration"`
}
type PutTableMaintenanceConfigurationRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
Type string `json:"type"`
Value *MaintenanceConfigurationValue `json:"value"`
}
type GetTableMaintenanceConfigurationRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
}
type GetTableMaintenanceConfigurationResponse struct {
TableARN string `json:"tableARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
Configuration MaintenanceConfiguration `json:"configuration"`
}
// Maintenance job status types
const (
MaintenanceJobStatusNotYetRun = "Not_Yet_Run"
MaintenanceJobStatusSuccessful = "Successful"
MaintenanceJobStatusFailed = "Failed"
MaintenanceJobStatusDisabled = "Disabled"
)
type MaintenanceJobStatusValue struct {
Status string `json:"status"`
LastRunTimestamp *time.Time `json:"lastRunTimestamp,omitempty"`
FailureMessage string `json:"failureMessage,omitempty"`
}
// MaintenanceJobStatus maps a maintenance type to the outcome of its last run.
// The worker writes it; the control plane only reads it.
type MaintenanceJobStatus map[string]*MaintenanceJobStatusValue
type GetTableMaintenanceJobStatusRequest struct {
TableBucketARN string `json:"tableBucketARN"`
Namespace []string `json:"namespace"`
Name string `json:"name"`
}
type GetTableMaintenanceJobStatusResponse struct {
TableARN string `json:"tableARN"`
Status MaintenanceJobStatus `json:"status"`
}
// Tagging types
type TagResourceRequest struct {
ResourceARN string `json:"resourceArn"`
Tags map[string]string `json:"tags"`
}
type ListTagsForResourceRequest struct {
ResourceARN string `json:"resourceArn"`
}
type ListTagsForResourceResponse struct {
Tags map[string]string `json:"tags"`
}
type UntagResourceRequest struct {
ResourceARN string `json:"resourceArn"`
TagKeys []string `json:"tagKeys"`
}
// Error types
type S3TablesError struct {
Type string `json:"__type"`
Message string `json:"message"`
}
func (e *S3TablesError) Error() string {
return e.Message
}
// Table formats a catalog entry may declare.
//
// ICEBERG tables carry metadata the catalog maintains and the maintenance
// worker rewrites. LANCE is catalog-only: the entry records a name and the
// dataset root in MetadataLocation, and the Lance client owns every byte under
// it. Nothing in this package interprets a catalog-only table's files.
const (
FormatIceberg = "ICEBERG"
FormatLance = "LANCE"
)
// IsCatalogOnlyFormat reports whether the catalog only records where a table of
// this format lives, without understanding its files.
func IsCatalogOnlyFormat(format string) bool {
return format == FormatLance
}
// NormalizeFormat folds a caller's spelling onto the canonical one and reports
// whether it names a format this catalog serves.
func NormalizeFormat(format string) (string, bool) {
switch strings.ToUpper(strings.TrimSpace(format)) {
case FormatIceberg:
return FormatIceberg, true
case FormatLance:
return FormatLance, true
default:
return "", false
}
}
// Error codes
const (
ErrCodeBucketAlreadyExists = "BucketAlreadyExists"
ErrCodeBucketNotEmpty = "BucketNotEmpty"
ErrCodeNoSuchBucket = "NoSuchBucket"
ErrCodeNoSuchNamespace = "NoSuchNamespace"
ErrCodeNoSuchTable = "NoSuchTable"
ErrCodeNamespaceAlreadyExists = "NamespaceAlreadyExists"
ErrCodeNamespaceNotEmpty = "NamespaceNotEmpty"
ErrCodeTableAlreadyExists = "TableAlreadyExists"
ErrCodeNoSuchView = "NoSuchView"
ErrCodeViewAlreadyExists = "ViewAlreadyExists"
ErrCodeAccessDenied = "AccessDenied"
ErrCodeInvalidRequest = "InvalidRequest"
ErrCodeInternalError = "InternalError"
ErrCodeNoSuchPolicy = "NoSuchPolicy"
ErrCodeConflict = "Conflict"
)

View File

@@ -0,0 +1,535 @@
package s3tables
import (
"crypto/rand"
"encoding/hex"
"fmt"
"net/url"
"path"
"regexp"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
const (
bucketNamePatternStr = `[a-z0-9-]+`
tableNamespacePatternStr = `[a-z0-9_.-]+`
tableNamePatternStr = `[a-z0-9_-]+`
)
const (
tableObjectRootDirName = ".objects"
)
// ARNPartitionPatternStr matches any AWS partition, not just the commercial
// one: aws-cn and aws-us-gov ARNs are valid and must reach the handlers.
const ARNPartitionPatternStr = `aws[-a-z0-9]*`
// ARNPrefixPatternStr is the leading, partition-tolerant part of every S3
// Tables ARN. The HTTP router shares it so routes and parsing agree.
const ARNPrefixPatternStr = `arn:` + ARNPartitionPatternStr + `:s3tables`
var (
bucketARNPattern = regexp.MustCompile(`^` + ARNPrefixPatternStr + `:[^:]*:[^:]*:bucket/(` + bucketNamePatternStr + `)$`)
tableARNPattern = regexp.MustCompile(`^` + ARNPrefixPatternStr + `:[^:]*:[^:]*:bucket/(` + bucketNamePatternStr + `)/table/(` + tableNamespacePatternStr + `)/(` + tableNamePatternStr + `)$`)
tagPattern = regexp.MustCompile(`^([\p{L}\p{Z}\p{N}_.:/=+\-@]*)$`)
)
// ARN parsing functions
// parseBucketNameFromARN extracts bucket name from table bucket ARN
// ARN format: arn:aws:s3tables:{region}:{account}:bucket/{bucket-name}
func parseBucketNameFromARN(arn string) (string, error) {
matches := bucketARNPattern.FindStringSubmatch(arn)
if len(matches) != 2 {
return "", fmt.Errorf("invalid bucket ARN: %s", arn)
}
bucketName := matches[1]
if !isValidBucketName(bucketName) {
return "", fmt.Errorf("invalid bucket name in ARN: %s", bucketName)
}
return bucketName, nil
}
// ParseBucketNameFromARN is a wrapper to validate bucket ARN for other packages.
func ParseBucketNameFromARN(arn string) (string, error) {
return parseBucketNameFromARN(arn)
}
// IsValidBucketName is a wrapper to validate a table bucket name for other packages.
func IsValidBucketName(name string) bool {
return isValidBucketName(name)
}
// parseTableFromARN extracts bucket name, namespace, and table name from ARN
// ARN format: arn:aws:s3tables:{region}:{account}:bucket/{bucket-name}/table/{namespace}/{table-name}
func parseTableFromARN(arn string) (bucketName, namespace, tableName string, err error) {
matches := tableARNPattern.FindStringSubmatch(arn)
if len(matches) != 4 {
return "", "", "", fmt.Errorf("invalid table ARN: %s", arn)
}
// Validate bucket name
bucketName = matches[1]
if err := validateBucketName(bucketName); err != nil {
return "", "", "", fmt.Errorf("invalid bucket name in ARN: %v", err)
}
namespace, err = validateNamespace([]string{matches[2]})
if err != nil {
return "", "", "", fmt.Errorf("invalid namespace in ARN: %v", err)
}
// URL decode and validate the table name from the ARN path component
tableNameUnescaped, err := url.PathUnescape(matches[3])
if err != nil {
return "", "", "", fmt.Errorf("invalid table name encoding in ARN: %v", err)
}
if _, err := validateTableName(tableNameUnescaped); err != nil {
return "", "", "", fmt.Errorf("invalid table name in ARN: %v", err)
}
return bucketName, namespace, tableNameUnescaped, nil
}
// Path helpers
// GetTableBucketPath returns the filer path for a table bucket
func GetTableBucketPath(bucketName string) string {
return path.Join(TablesPath, bucketName)
}
// GetNamespacePath returns the filer path for a namespace
func GetNamespacePath(bucketName, namespace string) string {
return path.Join(TablesPath, bucketName, namespace)
}
// GetTablePath returns the filer path for a table
func GetTablePath(bucketName, namespace, tableName string) string {
return path.Join(TablesPath, bucketName, namespace, tableName)
}
// TableDataDirFromMetadataLocation maps a table's s3:// metadata location to the
// filer directory holding its data. A renamed table is catalog-only, so its data
// stays at the original location while its catalog entry moves; this lets a drop
// purge the real data instead of the now-empty catalog path.
func TableDataDirFromMetadataLocation(metadataLocation string) string {
loc := strings.TrimSuffix(metadataLocation, "/")
if idx := strings.LastIndex(loc, "/metadata/"); idx != -1 {
loc = loc[:idx]
}
loc = strings.TrimPrefix(loc, "s3://")
if loc == "" {
return ""
}
return path.Join(TablesPath, loc)
}
// ValidateMetadataLocation checks that an s3:// metadata location stays within
// the authorized table bucket and rejects traversal segments that path.Join
// would collapse to escape the bucket directory. Empty locations are allowed
// (the catalog derives one). A non-empty location must include a table path so
// its metadata directory is table-specific, not shared at the bucket level.
func ValidateMetadataLocation(metadataLocation, bucketName string) error {
if metadataLocation == "" {
return nil
}
bucket, tablePath, err := parseS3Location(metadataLocation)
if err != nil {
return err
}
if bucket != bucketName {
return fmt.Errorf("metadata location must be within bucket %s", bucketName)
}
hasSegment := false
for _, segment := range strings.Split(tablePath, "/") {
if segment == "" {
continue
}
if segment == "." || segment == ".." || strings.ContainsAny(segment, "\\\x00") {
return fmt.Errorf("invalid metadata location path")
}
hasSegment = true
}
if !hasSegment {
return fmt.Errorf("metadata location must include a table path")
}
return nil
}
func parseS3Location(location string) (bucket, tablePath string, err error) {
if !strings.HasPrefix(location, "s3://") {
return "", "", fmt.Errorf("unsupported location: %s", location)
}
trimmed := strings.TrimPrefix(location, "s3://")
trimmed = strings.TrimSuffix(trimmed, "/")
if trimmed == "" {
return "", "", fmt.Errorf("invalid location: %s", location)
}
parts := strings.SplitN(trimmed, "/", 2)
bucket = parts[0]
if bucket == "" {
return "", "", fmt.Errorf("invalid location bucket: %s", location)
}
if len(parts) == 2 {
tablePath = parts[1]
}
return bucket, tablePath, nil
}
// GetTableObjectRootDir returns the root path for table bucket object storage
func GetTableObjectRootDir() string {
return path.Join(TablesPath, tableObjectRootDirName)
}
// GetTableObjectBucketPath returns the filer path for table bucket object storage
func GetTableObjectBucketPath(bucketName string) string {
return path.Join(GetTableObjectRootDir(), bucketName)
}
// Metadata structures
type tableBucketMetadata struct {
Name string `json:"name"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Format string `json:"format,omitempty"`
}
// namespaceMetadata stores metadata for a namespace
type namespaceMetadata struct {
Namespace []string `json:"namespace"`
CreatedAt time.Time `json:"createdAt"`
OwnerAccountID string `json:"ownerAccountId"`
Properties map[string]string `json:"properties,omitempty"`
}
// tableMetadataInternal stores metadata for a table
type tableMetadataInternal struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
Format string `json:"format"`
CreatedAt time.Time `json:"createdAt"`
ModifiedAt time.Time `json:"modifiedAt"`
OwnerAccountID string `json:"ownerAccountId"`
VersionToken string `json:"versionToken"`
MetadataVersion int `json:"metadataVersion"`
MetadataLocation string `json:"metadataLocation,omitempty"`
Metadata *TableMetadata `json:"metadata,omitempty"`
}
// IsTableBucketEntry returns true when the entry is marked as a table bucket.
func IsTableBucketEntry(entry *filer_pb.Entry) bool {
if entry == nil || entry.Extended == nil {
return false
}
_, ok := entry.Extended[ExtendedKeyTableBucket]
return ok
}
// EntryType returns the entry-type marker for a catalog entry. Tables and views
// share the same on-disk layout; the marker distinguishes them. An absent marker
// means table for back-compat.
func EntryType(extended map[string][]byte) string {
if extended == nil {
return EntryTypeTable
}
if v, ok := extended[ExtendedKeyEntryType]; ok && len(v) > 0 {
return string(v)
}
return EntryTypeTable
}
// Utility functions
// validateBucketName validates bucket name and returns an error if invalid.
// Bucket names must contain only lowercase letters, numbers, and hyphens.
// Length must be between 3 and 63 characters.
// Must start and end with a letter or digit.
// Reserved prefixes/suffixes are rejected.
func validateBucketName(name string) error {
if name == "" {
return fmt.Errorf("bucket name is required")
}
if len(name) < 3 || len(name) > 63 {
return fmt.Errorf("bucket name must be between 3 and 63 characters")
}
// Must start and end with a letter or digit
start := name[0]
end := name[len(name)-1]
if !((start >= 'a' && start <= 'z') || (start >= '0' && start <= '9')) {
return fmt.Errorf("bucket name must start with a letter or digit")
}
if !((end >= 'a' && end <= 'z') || (end >= '0' && end <= '9')) {
return fmt.Errorf("bucket name must end with a letter or digit")
}
// Allowed characters: a-z, 0-9, -
for i := 0; i < len(name); i++ {
ch := name[i]
if (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '-' {
continue
}
return fmt.Errorf("bucket name can only contain lowercase letters, numbers, and hyphens")
}
// Reserved prefixes
reservedPrefixes := []string{"xn--", "sthree-", "amzn-s3-demo-", "aws"}
for _, p := range reservedPrefixes {
if strings.HasPrefix(name, p) {
return fmt.Errorf("bucket name cannot start with reserved prefix: %s", p)
}
}
// Reserved suffixes
reservedSuffixes := []string{"-s3alias", "--ol-s3", "--x-s3", "--table-s3"}
for _, s := range reservedSuffixes {
if strings.HasSuffix(name, s) {
return fmt.Errorf("bucket name cannot end with reserved suffix: %s", s)
}
}
return nil
}
// BuildBucketARN builds a bucket ARN with the provided region and account ID.
// If region is empty, the ARN will omit the region field.
func BuildBucketARN(region, accountID, bucketName string) (string, error) {
if bucketName == "" {
return "", fmt.Errorf("bucket name is required")
}
if err := validateBucketName(bucketName); err != nil {
return "", err
}
if accountID == "" {
accountID = DefaultAccountID
}
return buildARN(region, accountID, fmt.Sprintf("bucket/%s", bucketName)), nil
}
// BuildTableARN builds a table ARN with the provided region and account ID.
func BuildTableARN(region, accountID, bucketName, namespace, tableName string) (string, error) {
if bucketName == "" {
return "", fmt.Errorf("bucket name is required")
}
if err := validateBucketName(bucketName); err != nil {
return "", err
}
if namespace == "" {
return "", fmt.Errorf("namespace is required")
}
normalizedNamespace, err := validateNamespace([]string{namespace})
if err != nil {
return "", err
}
if tableName == "" {
return "", fmt.Errorf("table name is required")
}
normalizedTable, err := validateTableName(tableName)
if err != nil {
return "", err
}
if accountID == "" {
accountID = DefaultAccountID
}
return buildARN(region, accountID, fmt.Sprintf("bucket/%s/table/%s/%s", bucketName, normalizedNamespace, normalizedTable)), nil
}
func buildARN(region, accountID, resourcePath string) string {
return fmt.Sprintf("arn:%s:s3tables:%s:%s:%s", arnPartitionForRegion(region), region, accountID, resourcePath)
}
// arnPartitionForRegion returns the ARN partition a region belongs to, so an
// ARN this handler emits round-trips through a client in that partition.
func arnPartitionForRegion(region string) string {
switch {
case strings.HasPrefix(region, "cn-"):
return "aws-cn"
case strings.HasPrefix(region, "us-gov-"):
return "aws-us-gov"
case strings.HasPrefix(region, "us-iso-"):
return "aws-iso"
case strings.HasPrefix(region, "us-isob-"):
return "aws-iso-b"
case strings.HasPrefix(region, "eu-isoe-"):
return "aws-iso-e"
case strings.HasPrefix(region, "us-isof-"):
return "aws-iso-f"
case strings.HasPrefix(region, "eusc-"):
return "aws-eusc"
default:
return "aws"
}
}
// ValidateTags validates tags for S3 Tables.
func ValidateTags(tags map[string]string) error {
if len(tags) > 10 {
return fmt.Errorf("validate tags: %d tags more than 10", len(tags))
}
for k, v := range tags {
if len(k) > 128 {
return fmt.Errorf("validate tags: tag key longer than 128")
}
if !tagPattern.MatchString(k) {
return fmt.Errorf("validate tags key %s error, incorrect key", k)
}
if len(v) > 256 {
return fmt.Errorf("validate tags: tag value longer than 256")
}
if !tagPattern.MatchString(v) {
return fmt.Errorf("validate tags value %s error, incorrect value", v)
}
}
return nil
}
// isValidBucketName validates bucket name characters (kept for compatibility)
// Deprecated: use validateBucketName instead
func isValidBucketName(name string) bool {
return validateBucketName(name) == nil
}
// generateVersionToken generates a unique, unpredictable version token
func generateVersionToken() string {
b := make([]byte, 16)
if _, err := rand.Read(b); err != nil {
// Fallback to timestamp if crypto/rand fails
return fmt.Sprintf("%x", time.Now().UnixNano())
}
return hex.EncodeToString(b)
}
// splitPath splits a path into directory and name components using stdlib
func splitPath(p string) (dir, name string) {
dir = path.Dir(p)
name = path.Base(p)
return
}
func validateNamespacePart(name string) error {
if len(name) < 1 || len(name) > 255 {
return fmt.Errorf("namespace name must be between 1 and 255 characters")
}
// Prevent path traversal and multi-segment paths
if name == "." || name == ".." {
return fmt.Errorf("namespace name cannot be '.' or '..'")
}
if strings.Contains(name, "/") {
return fmt.Errorf("namespace name cannot contain '/'")
}
// Must start and end with a letter or digit
start := name[0]
end := name[len(name)-1]
if !((start >= 'a' && start <= 'z') || (start >= '0' && start <= '9')) {
return fmt.Errorf("namespace name must start with a letter or digit")
}
if !((end >= 'a' && end <= 'z') || (end >= '0' && end <= '9')) {
return fmt.Errorf("namespace name must end with a letter or digit")
}
// Allowed characters: a-z, 0-9, _, - (hyphen interior; start/end checked above)
for _, ch := range name {
if (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '-' {
continue
}
return fmt.Errorf("invalid namespace name: only 'a-z', '0-9', '_', and '-' are allowed")
}
// Reserved prefix
if strings.HasPrefix(name, "aws") {
return fmt.Errorf("namespace name cannot start with reserved prefix 'aws'")
}
return nil
}
func normalizeNamespace(namespace []string) ([]string, error) {
if len(namespace) == 0 {
return nil, fmt.Errorf("namespace is required")
}
parts := namespace
if len(namespace) == 1 {
parts = strings.Split(namespace[0], ".")
}
normalized := make([]string, 0, len(parts))
for _, part := range parts {
if err := validateNamespacePart(part); err != nil {
return nil, err
}
normalized = append(normalized, part)
}
return normalized, nil
}
// validateNamespace validates namespace identifiers and returns an internal namespace key.
// A single dotted namespace value is interpreted as multi-level namespace for compatibility
// with path-style APIs, for example "analytics.daily" => ["analytics", "daily"].
func validateNamespace(namespace []string) (string, error) {
parts, err := normalizeNamespace(namespace)
if err != nil {
return "", err
}
return flattenNamespace(parts), nil
}
// ParseNamespace parses a namespace string into namespace parts.
func ParseNamespace(namespace string) ([]string, error) {
return normalizeNamespace([]string{namespace})
}
// validateTableName validates a table name
func validateTableName(name string) (string, error) {
if len(name) < 1 || len(name) > 255 {
return "", fmt.Errorf("table name must be between 1 and 255 characters")
}
if name == "." || name == ".." || strings.Contains(name, "/") {
return "", fmt.Errorf("invalid table name: cannot be '.', '..' or contain '/'")
}
// First character must be a letter or digit
start := name[0]
if !((start >= 'a' && start <= 'z') || (start >= '0' && start <= '9')) {
return "", fmt.Errorf("table name must start with a letter or digit")
}
// Allowed characters: a-z, 0-9, _, - (start checked above)
for _, ch := range name {
if (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '-' {
continue
}
return "", fmt.Errorf("invalid table name: only 'a-z', '0-9', '_', and '-' are allowed")
}
return name, nil
}
// ValidateTableName is a wrapper to validate table name for other packages.
func ValidateTableName(name string) (string, error) {
return validateTableName(name)
}
// flattenNamespace joins namespace elements into a single string (using dots as per AWS S3 Tables)
func flattenNamespace(namespace []string) string {
if len(namespace) == 0 {
return ""
}
return strings.Join(namespace, ".")
}
func expandNamespace(namespace string) []string {
if namespace == "" {
return nil
}
parts, err := ParseNamespace(namespace)
if err != nil {
return []string{namespace}
}
return parts
}

View File

@@ -0,0 +1,33 @@
package s3tables
import "testing"
func TestValidateMetadataLocation(t *testing.T) {
tests := []struct {
name string
loc string
bucket string
wantErr bool
}{
{"empty allowed", "", "bkt", false},
{"same bucket", "s3://bkt/ns/t/metadata/v1.metadata.json", "bkt", false},
{"cross bucket rejected", "s3://other/ns/t/metadata/v1.metadata.json", "bkt", true},
{"bucket only rejected", "s3://bkt", "bkt", true},
{"bucket with trailing slash rejected", "s3://bkt/", "bkt", true},
{"slash-only path rejected", "s3://bkt///", "bkt", true},
{"dotdot in path rejected", "s3://bkt/../victim/metadata/v1.metadata.json", "bkt", true},
{"dotdot mid path rejected", "s3://bkt/ns/../../victim/metadata/v1.metadata.json", "bkt", true},
{"dot segment rejected", "s3://bkt/ns/./t/metadata/v1.metadata.json", "bkt", true},
{"backslash segment rejected", "s3://bkt/ns/\\t/metadata/v1.metadata.json", "bkt", true},
{"non-s3 scheme rejected", "file:///bkt/ns/t", "bkt", true},
{"empty location string", "s3://", "bkt", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := ValidateMetadataLocation(tt.loc, tt.bucket)
if (err != nil) != tt.wantErr {
t.Fatalf("ValidateMetadataLocation(%q, %q) err = %v, wantErr = %v", tt.loc, tt.bucket, err, tt.wantErr)
}
})
}
}

View File

@@ -0,0 +1,186 @@
package s3tables
import (
"encoding/json"
"reflect"
"strings"
"testing"
)
func TestValidateNamespaceSupportsMultiLevel(t *testing.T) {
got, err := validateNamespace([]string{"analytics", "daily"})
if err != nil {
t.Fatalf("validateNamespace returned error: %v", err)
}
if got != "analytics.daily" {
t.Fatalf("validateNamespace = %q, want %q", got, "analytics.daily")
}
}
func TestValidateNamespaceSupportsDottedInput(t *testing.T) {
got, err := validateNamespace([]string{"analytics.daily"})
if err != nil {
t.Fatalf("validateNamespace returned error: %v", err)
}
if got != "analytics.daily" {
t.Fatalf("validateNamespace = %q, want %q", got, "analytics.daily")
}
}
func TestValidateNamespaceRejectsEmptyDottedSegment(t *testing.T) {
_, err := validateNamespace([]string{"analytics..daily"})
if err == nil {
t.Fatalf("expected validateNamespace to fail for empty dotted segment")
}
}
func TestParseNamespace(t *testing.T) {
tests := []struct {
name string
namespace string
want []string
wantErr bool
}{
{
name: "single level",
namespace: "analytics",
want: []string{"analytics"},
},
{
name: "multi level dotted",
namespace: "analytics.daily",
want: []string{"analytics", "daily"},
},
{
name: "invalid reserved prefix",
namespace: "analytics.awsprod",
wantErr: true,
},
{
name: "invalid empty segment",
namespace: "analytics..daily",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ParseNamespace(tt.namespace)
if tt.wantErr {
if err == nil {
t.Fatalf("ParseNamespace(%q) expected error", tt.namespace)
}
return
}
if err != nil {
t.Fatalf("ParseNamespace(%q) unexpected error: %v", tt.namespace, err)
}
if len(got) != len(tt.want) {
t.Fatalf("ParseNamespace(%q) = %v, want %v", tt.namespace, got, tt.want)
}
for i := range got {
if got[i] != tt.want[i] {
t.Fatalf("ParseNamespace(%q) = %v, want %v", tt.namespace, got, tt.want)
}
}
})
}
}
func TestParseTableFromARNWithMultiLevelNamespace(t *testing.T) {
arn := "arn:aws:s3tables:us-east-1:123456789012:bucket/testbucket/table/analytics.daily/events"
bucket, namespace, table, err := parseTableFromARN(arn)
if err != nil {
t.Fatalf("parseTableFromARN returned error: %v", err)
}
if bucket != "testbucket" {
t.Fatalf("bucket = %q, want %q", bucket, "testbucket")
}
if namespace != "analytics.daily" {
t.Fatalf("namespace = %q, want %q", namespace, "analytics.daily")
}
if table != "events" {
t.Fatalf("table = %q, want %q", table, "events")
}
}
func TestBuildTableARNWithDottedNamespace(t *testing.T) {
arn, err := BuildTableARN("us-east-1", "123456789012", "testbucket", "analytics.daily", "events")
if err != nil {
t.Fatalf("BuildTableARN returned error: %v", err)
}
if !strings.Contains(arn, "/table/analytics.daily/events") {
t.Fatalf("BuildTableARN returned %q, missing normalized namespace/table path", arn)
}
}
func TestExpandNamespace(t *testing.T) {
got := expandNamespace("analytics.daily")
want := []string{"analytics", "daily"}
if len(got) != len(want) {
t.Fatalf("expandNamespace = %v, want %v", got, want)
}
for i := range got {
if got[i] != want[i] {
t.Fatalf("expandNamespace = %v, want %v", got, want)
}
}
}
func TestNamespaceMetadataPropertiesRoundTrip(t *testing.T) {
testCases := []struct {
name string
metadata namespaceMetadata
}{
{
name: "with properties",
metadata: namespaceMetadata{
Namespace: []string{"analytics"},
Properties: map[string]string{"owner": "finance"},
OwnerAccountID: "123456789012",
},
},
{
name: "nil properties",
metadata: namespaceMetadata{
Namespace: []string{"analytics"},
Properties: nil,
OwnerAccountID: "123456789012",
},
},
{
name: "empty properties",
metadata: namespaceMetadata{
Namespace: []string{"analytics"},
Properties: map[string]string{},
OwnerAccountID: "123456789012",
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
data, err := json.Marshal(tc.metadata)
if err != nil {
t.Fatalf("json.Marshal(metadata) returned error: %v", err)
}
var decoded namespaceMetadata
if err := json.Unmarshal(data, &decoded); err != nil {
t.Fatalf("json.Unmarshal(data) returned error: %v", err)
}
// Due to `omitempty`, nil and empty maps are unmarshaled as nil.
if len(tc.metadata.Properties) == 0 {
if decoded.Properties != nil {
t.Fatalf("expected nil properties for empty/nil input, got %v", decoded.Properties)
}
return
}
if !reflect.DeepEqual(decoded.Properties, tc.metadata.Properties) {
t.Fatalf("decoded.Properties = %v, want %v", decoded.Properties, tc.metadata.Properties)
}
})
}
}