snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
311
weed/s3api/s3tables/filer_ops.go
Normal file
311
weed/s3api/s3tables/filer_ops.go
Normal file
@@ -0,0 +1,311 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrAttributeNotFound = errors.New("attribute not found")
|
||||
ErrConcurrentUpdate = errors.New("attribute changed concurrently")
|
||||
)
|
||||
|
||||
// extendedAttributeUpdateAttempts bounds the read-modify-write retry loop in
|
||||
// updateExtendedAttribute.
|
||||
const extendedAttributeUpdateAttempts = 5
|
||||
|
||||
// Filer operations - Common functions for interacting with the filer
|
||||
|
||||
// createDirectory creates a new directory at the specified path
|
||||
func (h *S3TablesHandler) createDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
|
||||
dir, name := splitPath(path)
|
||||
now := time.Now().Unix()
|
||||
return filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
|
||||
Directory: dir,
|
||||
Entry: &filer_pb.Entry{
|
||||
Name: name,
|
||||
IsDirectory: true,
|
||||
Attributes: &filer_pb.FuseAttributes{
|
||||
Mtime: now,
|
||||
Crtime: now,
|
||||
FileMode: uint32(0755 | os.ModeDir),
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// ensureDirectory ensures a directory exists at the specified path
|
||||
func (h *S3TablesHandler) ensureDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
|
||||
dir, name := splitPath(path)
|
||||
_, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return h.createDirectory(ctx, client, path)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// deleteEntryIfExists removes an entry if it exists, ignoring missing errors
|
||||
func (h *S3TablesHandler) deleteEntryIfExists(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
|
||||
dir, name := splitPath(path)
|
||||
return filer_pb.DoRemove(ctx, client, dir, name, true, false, true, false, nil)
|
||||
}
|
||||
|
||||
// mutateEntryExtended applies mutate to an entry's extended attributes and
|
||||
// writes the entry back, asserting that nothing on it changed in between.
|
||||
// UpdateEntry rewrites the whole entry from the snapshot read here, so an
|
||||
// unconditional write would delete an attribute a concurrent writer created.
|
||||
func (h *S3TablesHandler) mutateEntryExtended(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
path string,
|
||||
mutate func(extended map[string][]byte) error,
|
||||
) error {
|
||||
dir, name := splitPath(path)
|
||||
|
||||
for attempt := 0; attempt < extendedAttributeUpdateAttempts; attempt++ {
|
||||
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
entry := resp.Entry
|
||||
expected := snapshotExtended(entry.Extended)
|
||||
if entry.Extended == nil {
|
||||
entry.Extended = make(map[string][]byte)
|
||||
}
|
||||
if err := mutate(entry.Extended); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{
|
||||
Directory: dir,
|
||||
Entry: entry,
|
||||
ExpectedExtended: expected,
|
||||
})
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if status.Code(err) != codes.FailedPrecondition {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("%w: %s", ErrConcurrentUpdate, path)
|
||||
}
|
||||
|
||||
// setExtendedAttribute sets an extended attribute on an existing entry
|
||||
func (h *S3TablesHandler) setExtendedAttribute(ctx context.Context, client filer_pb.SeaweedFilerClient, path, key string, data []byte) error {
|
||||
return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
|
||||
extended[key] = data
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// setExtendedAttributes sets multiple extended attributes on an existing entry
|
||||
// in a single UpdateEntry, so callers don't leave the entry partially tagged.
|
||||
func (h *S3TablesHandler) setExtendedAttributes(ctx context.Context, client filer_pb.SeaweedFilerClient, path string, attrs map[string][]byte) error {
|
||||
return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
|
||||
for key, data := range attrs {
|
||||
extended[key] = data
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// removeExtendedAttributes deletes the given extended attributes from an entry,
|
||||
// leaving the directory and its children intact.
|
||||
func (h *S3TablesHandler) removeExtendedAttributes(ctx context.Context, client filer_pb.SeaweedFilerClient, path string, keys ...string) error {
|
||||
return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
|
||||
for _, key := range keys {
|
||||
delete(extended, key)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// removeExtendedAttributesIf drops the given attributes only while they still
|
||||
// hold the values the caller captured. A rename copies attributes to the
|
||||
// destination and then clears the source; without this, a write that landed in
|
||||
// between is deleted here and never reaches the new name.
|
||||
func (h *S3TablesHandler) removeExtendedAttributesIf(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
path string,
|
||||
expected map[string][]byte,
|
||||
keys ...string,
|
||||
) error {
|
||||
return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
|
||||
for key, want := range expected {
|
||||
if !bytes.Equal(extended[key], want) {
|
||||
return fmt.Errorf("%w: %s changed on %s", ErrConcurrentUpdate, key, path)
|
||||
}
|
||||
}
|
||||
for _, key := range keys {
|
||||
delete(extended, key)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// updateExtendedAttribute applies mutate to the current value of key and writes
|
||||
// the result back. mutate runs again on each retry, so it always sees the
|
||||
// current value.
|
||||
func (h *S3TablesHandler) updateExtendedAttribute(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
path, key string,
|
||||
mutate func(current []byte) ([]byte, error),
|
||||
) error {
|
||||
return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
|
||||
updated, err := mutate(extended[key])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
extended[key] = updated
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// s3tablesExtendedKeys is every attribute this package stores on a bucket or
|
||||
// table entry. A whole-entry write has to assert the absent ones too: a key
|
||||
// missing from the precondition is a key a concurrent writer can create and
|
||||
// this write will then delete.
|
||||
var s3tablesExtendedKeys = []string{
|
||||
ExtendedKeyTableBucket,
|
||||
ExtendedKeyMetadata,
|
||||
ExtendedKeyMetadataVersion,
|
||||
ExtendedKeyPolicy,
|
||||
ExtendedKeyTags,
|
||||
ExtendedKeyMaintenance,
|
||||
ExtendedKeyMaintenanceStatus,
|
||||
ExtendedKeyEntryType,
|
||||
}
|
||||
|
||||
// SnapshotExtended captures an entry's attributes as an UpdateEntry
|
||||
// precondition, so a whole-entry write cannot silently revert or delete an
|
||||
// attribute a concurrent writer touched. Attributes absent at read time are
|
||||
// asserted absent, which fails the precondition if someone creates one first.
|
||||
func SnapshotExtended(extended map[string][]byte, keys ...string) map[string][]byte {
|
||||
return snapshotExtended(extended, keys...)
|
||||
}
|
||||
|
||||
func snapshotExtended(extended map[string][]byte, keys ...string) map[string][]byte {
|
||||
expected := make(map[string][]byte, len(extended)+len(s3tablesExtendedKeys))
|
||||
for k, v := range extended {
|
||||
expected[k] = v
|
||||
}
|
||||
for _, k := range s3tablesExtendedKeys {
|
||||
if _, ok := expected[k]; !ok {
|
||||
expected[k] = nil
|
||||
}
|
||||
}
|
||||
for _, k := range keys {
|
||||
if _, ok := expected[k]; !ok {
|
||||
expected[k] = nil
|
||||
}
|
||||
}
|
||||
return expected
|
||||
}
|
||||
|
||||
// getExtendedAttribute gets an extended attribute from an entry
|
||||
func (h *S3TablesHandler) getExtendedAttribute(ctx context.Context, client filer_pb.SeaweedFilerClient, path, key string) ([]byte, error) {
|
||||
dir, name := splitPath(path)
|
||||
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.Entry.Extended == nil {
|
||||
return nil, fmt.Errorf("%w: %s", ErrAttributeNotFound, key)
|
||||
}
|
||||
|
||||
data, ok := resp.Entry.Extended[key]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: %s", ErrAttributeNotFound, key)
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// loadNamespaceMetadata resolves a namespace to its metadata. A directory that
|
||||
// carries no namespace metadata is not a namespace, so a missing attribute
|
||||
// reports the same absence as a missing entry and every caller tests one
|
||||
// condition instead of forgetting the second.
|
||||
func (h *S3TablesHandler) loadNamespaceMetadata(ctx context.Context, filerClient FilerClient, bucketName, namespaceName string) (*namespaceMetadata, error) {
|
||||
var metadata namespaceMetadata
|
||||
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(ctx, client, GetNamespacePath(bucketName, namespaceName), ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
return filer_pb.ErrNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal namespace metadata: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &metadata, nil
|
||||
}
|
||||
|
||||
// lookupEntry returns the filer entry at the given path.
|
||||
func (h *S3TablesHandler) lookupEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) (*filer_pb.Entry, error) {
|
||||
dir, name := splitPath(path)
|
||||
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return resp.Entry, nil
|
||||
}
|
||||
|
||||
// deleteExtendedAttribute deletes an extended attribute from an entry
|
||||
func (h *S3TablesHandler) deleteExtendedAttribute(ctx context.Context, client filer_pb.SeaweedFilerClient, path, key string) error {
|
||||
return h.mutateEntryExtended(ctx, client, path, func(extended map[string][]byte) error {
|
||||
delete(extended, key)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// deleteDirectory deletes a directory and all its contents
|
||||
func (h *S3TablesHandler) deleteDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
|
||||
dir, name := splitPath(path)
|
||||
return filer_pb.DoRemove(ctx, client, dir, name, true, true, true, false, nil)
|
||||
}
|
||||
|
||||
// entryExists checks if an entry exists at the given path
|
||||
func (h *S3TablesHandler) entryExists(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) bool {
|
||||
dir, name := splitPath(path)
|
||||
_, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
return err == nil
|
||||
}
|
||||
102
weed/s3api/s3tables/filer_ops_test.go
Normal file
102
weed/s3api/s3tables/filer_ops_test.go
Normal file
@@ -0,0 +1,102 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// deleteEntryStub answers DeleteEntry the way the filer does: a rejected delete
|
||||
// comes back as a nil transport error with the reason in the response.
|
||||
type deleteEntryStub struct {
|
||||
filer_pb.SeaweedFilerClient
|
||||
resp *filer_pb.DeleteEntryResponse
|
||||
err error
|
||||
req *filer_pb.DeleteEntryRequest
|
||||
}
|
||||
|
||||
func (s *deleteEntryStub) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest, _ ...grpc.CallOption) (*filer_pb.DeleteEntryResponse, error) {
|
||||
s.req = req
|
||||
return s.resp, s.err
|
||||
}
|
||||
|
||||
func TestDeleteDirectoryReportsRejectedDelete(t *testing.T) {
|
||||
stub := &deleteEntryStub{resp: &filer_pb.DeleteEntryResponse{Error: "fail to delete non-empty folder"}}
|
||||
|
||||
err := (&S3TablesHandler{}).deleteDirectory(context.Background(), stub, "/buckets/b/ns/t")
|
||||
|
||||
require.Error(t, err, "a delete the filer rejected must not be reported as success")
|
||||
assert.Contains(t, err.Error(), "fail to delete non-empty folder")
|
||||
}
|
||||
|
||||
func TestDeleteDirectoryToleratesMissingEntry(t *testing.T) {
|
||||
t.Run("reported in the response", func(t *testing.T) {
|
||||
stub := &deleteEntryStub{resp: &filer_pb.DeleteEntryResponse{Error: filer_pb.ErrNotFound.Error()}}
|
||||
assert.NoError(t, (&S3TablesHandler{}).deleteDirectory(context.Background(), stub, "/buckets/b/ns/t"))
|
||||
})
|
||||
t.Run("reported as a transport error", func(t *testing.T) {
|
||||
stub := &deleteEntryStub{err: status.Error(codes.NotFound, filer_pb.ErrNotFound.Error())}
|
||||
assert.NoError(t, (&S3TablesHandler{}).deleteDirectory(context.Background(), stub, "/buckets/b/ns/t"))
|
||||
})
|
||||
}
|
||||
|
||||
func TestDeleteDirectoryPropagatesTransportError(t *testing.T) {
|
||||
stub := &deleteEntryStub{err: errors.New("filer unreachable")}
|
||||
|
||||
err := (&S3TablesHandler{}).deleteDirectory(context.Background(), stub, "/buckets/b/ns/t")
|
||||
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "filer unreachable")
|
||||
}
|
||||
|
||||
// deleteDirectory drops a whole subtree, so the delete has to stay recursive and
|
||||
// take the data with it.
|
||||
func TestDeleteDirectoryDeletesTheSubtree(t *testing.T) {
|
||||
stub := &deleteEntryStub{resp: &filer_pb.DeleteEntryResponse{}}
|
||||
|
||||
require.NoError(t, (&S3TablesHandler{}).deleteDirectory(context.Background(), stub, "/buckets/b/ns/t"))
|
||||
|
||||
require.NotNil(t, stub.req)
|
||||
assert.Equal(t, "/buckets/b/ns", stub.req.Directory)
|
||||
assert.Equal(t, "t", stub.req.Name)
|
||||
assert.True(t, stub.req.IsDeleteData)
|
||||
assert.True(t, stub.req.IsRecursive)
|
||||
assert.True(t, stub.req.IgnoreRecursiveError)
|
||||
}
|
||||
|
||||
// The bucket is the directory, so DeleteTableBucket must not answer success
|
||||
// when the filer refused to remove it.
|
||||
func TestDeleteTableBucketReportsRejectedDirectoryDelete(t *testing.T) {
|
||||
fs := s3tablestest.Start(t)
|
||||
bucketMeta, err := json.Marshal(tableBucketMetadata{Name: renameTestBucket, OwnerAccountID: DefaultAccountID})
|
||||
require.NoError(t, err)
|
||||
fs.Put(TablesPath, renameTestBucket, map[string][]byte{
|
||||
ExtendedKeyTableBucket: []byte("{}"),
|
||||
ExtendedKeyMetadata: bucketMeta,
|
||||
})
|
||||
|
||||
bucketPath := GetTableBucketPath(renameTestBucket)
|
||||
fs.RejectDelete = func(dir, name string) string {
|
||||
if dir+"/"+name == bucketPath {
|
||||
return "fail to delete non-empty folder"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
m := NewManager()
|
||||
m.SetTrusted(true)
|
||||
err = m.Execute(context.Background(), NewManagerClient(fs.Client), "DeleteTableBucket",
|
||||
&DeleteTableBucketRequest{TableBucketARN: mustBucketARN(t)}, nil, "")
|
||||
|
||||
require.Error(t, err, "a bucket the filer refused to delete must not be reported as deleted")
|
||||
assert.NotNil(t, fs.Get(TablesPath, renameTestBucket), "the bucket is still there")
|
||||
}
|
||||
444
weed/s3api/s3tables/handler.go
Normal file
444
weed/s3api/s3tables/handler.go
Normal file
@@ -0,0 +1,444 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
const (
|
||||
TablesPath = s3_constants.DefaultBucketsPath
|
||||
DefaultAccountID = "000000000000"
|
||||
DefaultRegion = "us-east-1"
|
||||
|
||||
// Extended entry attributes for metadata storage
|
||||
ExtendedKeyTableBucket = s3_constants.ExtS3TablesPrefix + "tableBucket"
|
||||
ExtendedKeyMetadata = s3_constants.ExtS3TablesPrefix + "metadata"
|
||||
ExtendedKeyMetadataVersion = s3_constants.ExtS3TablesPrefix + "metadataVersion"
|
||||
ExtendedKeyPolicy = s3_constants.ExtS3TablesPrefix + "policy"
|
||||
ExtendedKeyTags = s3_constants.ExtS3TablesPrefix + "tags"
|
||||
ExtendedKeyMaintenance = s3_constants.ExtS3TablesPrefix + "maintenance"
|
||||
// Written by the maintenance worker, read by GetTableMaintenanceJobStatus.
|
||||
// Separate from ExtendedKeyMaintenance so worker and operator writes do not
|
||||
// contend on the same attribute.
|
||||
ExtendedKeyMaintenanceStatus = s3_constants.ExtS3TablesPrefix + "maintenanceStatus"
|
||||
ExtendedKeyEntryType = s3_constants.ExtS3TablesPrefix + "entryType"
|
||||
|
||||
// Entry-type marker values for ExtendedKeyEntryType. Absent or "table" means
|
||||
// a table; views are stored like tables but tagged "view".
|
||||
EntryTypeTable = "table"
|
||||
EntryTypeView = "view"
|
||||
|
||||
// Maximum request body size (10MB)
|
||||
maxRequestBodySize = 10 * 1024 * 1024
|
||||
)
|
||||
|
||||
var (
|
||||
ErrVersionTokenMismatch = errors.New("version token mismatch")
|
||||
ErrAccessDenied = errors.New("access denied")
|
||||
ErrTableAlreadyExists = errors.New("table already exists")
|
||||
)
|
||||
|
||||
type ResourceType string
|
||||
|
||||
const (
|
||||
ResourceTypeBucket ResourceType = "bucket"
|
||||
ResourceTypeTable ResourceType = "table"
|
||||
)
|
||||
|
||||
// S3TablesHandler handles S3 Tables API requests
|
||||
type S3TablesHandler struct {
|
||||
region string
|
||||
accountID string
|
||||
defaultAllow bool // Whether to allow access by default (for zero-config IAM)
|
||||
trusted bool // Trusted local tooling (shell/admin) bypasses authorization
|
||||
iamAuthorizer IAMAuthorizer
|
||||
}
|
||||
|
||||
// NewS3TablesHandler creates a new S3 Tables handler
|
||||
func NewS3TablesHandler() *S3TablesHandler {
|
||||
return &S3TablesHandler{
|
||||
region: DefaultRegion,
|
||||
accountID: DefaultAccountID,
|
||||
defaultAllow: false,
|
||||
}
|
||||
}
|
||||
|
||||
// SetRegion sets the AWS region for ARN generation
|
||||
func (h *S3TablesHandler) SetRegion(region string) {
|
||||
if region != "" {
|
||||
h.region = region
|
||||
}
|
||||
}
|
||||
|
||||
// SetAccountID sets the AWS account ID for ARN generation
|
||||
func (h *S3TablesHandler) SetAccountID(accountID string) {
|
||||
if accountID != "" {
|
||||
h.accountID = accountID
|
||||
}
|
||||
}
|
||||
|
||||
// SetDefaultAllow sets whether to allow access by default
|
||||
func (h *S3TablesHandler) SetDefaultAllow(allow bool) {
|
||||
h.defaultAllow = allow
|
||||
}
|
||||
|
||||
// SetTrusted lets local tooling that talks to the filer directly (shell, admin
|
||||
// console) bypass authorization. HTTP-facing callers must not set it.
|
||||
func (h *S3TablesHandler) SetTrusted(trusted bool) {
|
||||
h.trusted = trusted
|
||||
}
|
||||
|
||||
// FilerClient interface for filer operations
|
||||
type FilerClient interface {
|
||||
WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error
|
||||
}
|
||||
|
||||
// HandleRequest is the main entry point for S3 Tables API requests
|
||||
func (h *S3TablesHandler) HandleRequest(w http.ResponseWriter, r *http.Request, filerClient FilerClient) {
|
||||
operation := r.Header.Get("X-Amz-Target")
|
||||
if operation != "" {
|
||||
if idx := strings.LastIndex(operation, "."); idx != -1 {
|
||||
operation = operation[idx+1:]
|
||||
}
|
||||
}
|
||||
if operation == "" {
|
||||
glog.V(1).Infof("S3Tables: missing X-Amz-Target header")
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "Missing X-Amz-Target header")
|
||||
return
|
||||
}
|
||||
|
||||
glog.V(3).Infof("S3Tables: handling operation %s", operation)
|
||||
|
||||
var err error
|
||||
switch operation {
|
||||
// Table Bucket operations
|
||||
case "CreateTableBucket":
|
||||
err = h.handleCreateTableBucket(w, r, filerClient)
|
||||
case "GetTableBucket":
|
||||
err = h.handleGetTableBucket(w, r, filerClient)
|
||||
case "ListTableBuckets":
|
||||
err = h.handleListTableBuckets(w, r, filerClient)
|
||||
case "DeleteTableBucket":
|
||||
err = h.handleDeleteTableBucket(w, r, filerClient)
|
||||
|
||||
// Table Bucket Policy operations
|
||||
case "PutTableBucketPolicy":
|
||||
err = h.handlePutTableBucketPolicy(w, r, filerClient)
|
||||
case "GetTableBucketPolicy":
|
||||
err = h.handleGetTableBucketPolicy(w, r, filerClient)
|
||||
case "DeleteTableBucketPolicy":
|
||||
err = h.handleDeleteTableBucketPolicy(w, r, filerClient)
|
||||
|
||||
// Namespace operations
|
||||
case "CreateNamespace":
|
||||
err = h.handleCreateNamespace(w, r, filerClient)
|
||||
case "GetNamespace":
|
||||
err = h.handleGetNamespace(w, r, filerClient)
|
||||
case "UpdateNamespace":
|
||||
err = h.handleUpdateNamespace(w, r, filerClient)
|
||||
case "ListNamespaces":
|
||||
err = h.handleListNamespaces(w, r, filerClient)
|
||||
case "DeleteNamespace":
|
||||
err = h.handleDeleteNamespace(w, r, filerClient)
|
||||
|
||||
// Table operations
|
||||
case "CreateTable":
|
||||
err = h.handleCreateTable(w, r, filerClient)
|
||||
case "RegisterTable":
|
||||
err = h.handleRegisterTable(w, r, filerClient)
|
||||
case "GetTable":
|
||||
err = h.handleGetTable(w, r, filerClient)
|
||||
case "ListTables":
|
||||
err = h.handleListTables(w, r, filerClient)
|
||||
case "UpdateTable":
|
||||
err = h.handleUpdateTable(w, r, filerClient)
|
||||
case "DeleteTable":
|
||||
err = h.handleDeleteTable(w, r, filerClient)
|
||||
case "RenameTable":
|
||||
err = h.handleRenameTable(w, r, filerClient)
|
||||
|
||||
// View operations
|
||||
case "CreateView":
|
||||
err = h.handleCreateView(w, r, filerClient)
|
||||
case "GetView":
|
||||
err = h.handleGetView(w, r, filerClient)
|
||||
case "ListViews":
|
||||
err = h.handleListViews(w, r, filerClient)
|
||||
case "UpdateView":
|
||||
err = h.handleUpdateView(w, r, filerClient)
|
||||
case "DeleteView":
|
||||
err = h.handleDeleteView(w, r, filerClient)
|
||||
case "RenameView":
|
||||
err = h.handleRenameView(w, r, filerClient)
|
||||
|
||||
// Table Policy operations
|
||||
case "PutTablePolicy":
|
||||
err = h.handlePutTablePolicy(w, r, filerClient)
|
||||
case "GetTablePolicy":
|
||||
err = h.handleGetTablePolicy(w, r, filerClient)
|
||||
case "DeleteTablePolicy":
|
||||
err = h.handleDeleteTablePolicy(w, r, filerClient)
|
||||
|
||||
// Maintenance configuration operations
|
||||
case "PutTableBucketMaintenanceConfiguration":
|
||||
err = h.handlePutTableBucketMaintenanceConfiguration(w, r, filerClient)
|
||||
case "GetTableBucketMaintenanceConfiguration":
|
||||
err = h.handleGetTableBucketMaintenanceConfiguration(w, r, filerClient)
|
||||
case "PutTableMaintenanceConfiguration":
|
||||
err = h.handlePutTableMaintenanceConfiguration(w, r, filerClient)
|
||||
case "GetTableMaintenanceConfiguration":
|
||||
err = h.handleGetTableMaintenanceConfiguration(w, r, filerClient)
|
||||
case "GetTableMaintenanceJobStatus":
|
||||
err = h.handleGetTableMaintenanceJobStatus(w, r, filerClient)
|
||||
|
||||
// Tagging operations
|
||||
case "TagResource":
|
||||
err = h.handleTagResource(w, r, filerClient)
|
||||
case "ListTagsForResource":
|
||||
err = h.handleListTagsForResource(w, r, filerClient)
|
||||
case "UntagResource":
|
||||
err = h.handleUntagResource(w, r, filerClient)
|
||||
|
||||
default:
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, fmt.Sprintf("Unknown operation: %s", operation))
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("S3Tables: error handling %s: %v", operation, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Principal/authorization helpers
|
||||
|
||||
// getAccountID returns a stable caller identifier for ownership and permission checks.
|
||||
// Reflection depends on the identity shape produced by JWT/STS auth (Account *struct{Id string},
|
||||
// Claims map[string]interface{} containing string values for preferred_username/sub, and optional
|
||||
// identity name/header values). Changing those fields without updating the reflection here will
|
||||
// break the handler, so refactorers should replace this with a typed interface if needed.
|
||||
func (h *S3TablesHandler) getAccountID(r *http.Request) string {
|
||||
identityRaw := s3_constants.GetIdentityFromContext(r)
|
||||
if identityRaw != nil {
|
||||
// Use reflection to access identity fields and avoid import cycles.
|
||||
val := reflect.ValueOf(identityRaw)
|
||||
if val.Kind() == reflect.Ptr {
|
||||
val = val.Elem()
|
||||
}
|
||||
if val.Kind() == reflect.Struct {
|
||||
// Prefer stable claims from JWT/STS identities. Only "sub" is guaranteed durable per OIDC;
|
||||
// preferred_username is ergonomic but can rotate and may orphan ownership data, while email
|
||||
// is explicitly excluded to avoid storing PII in metadata.
|
||||
claimsField := val.FieldByName("Claims")
|
||||
if claimsField.IsValid() && claimsField.Kind() == reflect.Map && !claimsField.IsNil() && claimsField.Type().Key().Kind() == reflect.String {
|
||||
for _, claimKey := range []string{"sub", "preferred_username"} {
|
||||
claimVal := claimsField.MapIndex(reflect.ValueOf(claimKey))
|
||||
if !claimVal.IsValid() {
|
||||
continue
|
||||
}
|
||||
if claimVal.Kind() == reflect.Interface && !claimVal.IsNil() {
|
||||
claimVal = claimVal.Elem()
|
||||
}
|
||||
if claimVal.Kind() == reflect.String {
|
||||
if principal := normalizePrincipalID(claimVal.String()); principal != "" {
|
||||
return principal
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
accountField := val.FieldByName("Account")
|
||||
if accountField.IsValid() && !accountField.IsNil() {
|
||||
accountVal := accountField.Elem()
|
||||
if accountVal.Kind() == reflect.Struct {
|
||||
idField := accountVal.FieldByName("Id")
|
||||
if idField.IsValid() && idField.Kind() == reflect.String {
|
||||
if principal := normalizePrincipalID(idField.String()); principal != "" {
|
||||
// Account-less identities default to the admin account; only
|
||||
// keep it for real admins, else use the unique identity name.
|
||||
if principal != s3_constants.AccountAdminId || hasAdminAction(getIdentityActions(r)) {
|
||||
return principal
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if identityName := s3_constants.GetIdentityNameFromContext(r); identityName != "" {
|
||||
if principal := normalizePrincipalID(identityName); principal != "" {
|
||||
return principal
|
||||
}
|
||||
}
|
||||
|
||||
if accountID := r.Header.Get(s3_constants.AmzAccountId); accountID != "" {
|
||||
if principal := normalizePrincipalID(accountID); principal != "" {
|
||||
return principal
|
||||
}
|
||||
}
|
||||
return h.accountID
|
||||
}
|
||||
|
||||
// normalizePrincipalID collapses ARN and identity strings to a key that is stable within a single account.
|
||||
// WARNING: this assumes identity names are unique per account; distinct principals such as
|
||||
// arn:aws:iam::111:user/alice and arn:aws:iam::222:user/alice will both normalize to "alice".
|
||||
// If future work adds multi-account support, revisit this function to include the account ID or full ARN
|
||||
// so ownership checks remain correct.
|
||||
func normalizePrincipalID(id string) string {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
return ""
|
||||
}
|
||||
// If this is an ARN (common for assumed roles), use the trailing segment as a
|
||||
// stable-ish principal key instead of embedding the full ARN in ownership fields.
|
||||
if strings.HasPrefix(id, "arn:") {
|
||||
if idx := strings.LastIndex(id, "/"); idx >= 0 && idx+1 < len(id) {
|
||||
return strings.TrimSpace(id[idx+1:])
|
||||
}
|
||||
if idx := strings.LastIndex(id, ":"); idx >= 0 && idx+1 < len(id) {
|
||||
return strings.TrimSpace(id[idx+1:])
|
||||
}
|
||||
return strings.TrimSpace(id)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// getIdentityActions extracts the action list from the identity object in the request context.
|
||||
// Uses reflection to avoid import cycles with s3api package.
|
||||
func getIdentityActions(r *http.Request) []string {
|
||||
val, ok := getIdentityStructValue(r)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
actionsField := val.FieldByName("Actions")
|
||||
if !actionsField.IsValid() || actionsField.Kind() != reflect.Slice {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert actions to string slice
|
||||
actions := make([]string, actionsField.Len())
|
||||
for i := 0; i < actionsField.Len(); i++ {
|
||||
action := actionsField.Index(i)
|
||||
// Action is likely a custom type (e.g., type Action string)
|
||||
// Convert to string using String() or direct string conversion
|
||||
if action.Kind() == reflect.String {
|
||||
actions[i] = action.String()
|
||||
} else if action.CanInterface() {
|
||||
// Try to convert via fmt.Sprint
|
||||
actions[i] = fmt.Sprint(action.Interface())
|
||||
}
|
||||
}
|
||||
return actions
|
||||
}
|
||||
|
||||
// Request/Response helpers
|
||||
|
||||
func (h *S3TablesHandler) readRequestBody(r *http.Request, v interface{}) error {
|
||||
defer r.Body.Close()
|
||||
|
||||
// Limit request body size to prevent unbounded reads
|
||||
limitedReader := io.LimitReader(r.Body, maxRequestBodySize+1)
|
||||
body, err := io.ReadAll(limitedReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read request body: %w", err)
|
||||
}
|
||||
|
||||
// Check if body exceeds size limit
|
||||
if len(body) > maxRequestBodySize {
|
||||
return fmt.Errorf("request body too large: exceeds maximum size of %d bytes", maxRequestBodySize)
|
||||
}
|
||||
|
||||
if len(body) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(body, v); err != nil {
|
||||
return fmt.Errorf("failed to decode request: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Response writing helpers
|
||||
|
||||
func (h *S3TablesHandler) writeJSON(w http.ResponseWriter, status int, data interface{}) {
|
||||
w.Header().Set("Content-Type", "application/x-amz-json-1.1")
|
||||
w.WriteHeader(status)
|
||||
if data != nil {
|
||||
if err := json.NewEncoder(w).Encode(data); err != nil {
|
||||
glog.Errorf("S3Tables: failed to encode response: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *S3TablesHandler) writeError(w http.ResponseWriter, status int, code, message string) {
|
||||
w.Header().Set("Content-Type", "application/x-amz-json-1.1")
|
||||
w.WriteHeader(status)
|
||||
errorResponse := map[string]interface{}{
|
||||
"__type": code,
|
||||
"message": message,
|
||||
}
|
||||
if err := json.NewEncoder(w).Encode(errorResponse); err != nil {
|
||||
glog.Errorf("S3Tables: failed to encode error response: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ARN generation helpers
|
||||
|
||||
func (h *S3TablesHandler) generateTableBucketARN(ownerAccountID, bucketName string) string {
|
||||
return buildARN(h.region, ownerAccountID, fmt.Sprintf("bucket/%s", bucketName))
|
||||
}
|
||||
|
||||
func (h *S3TablesHandler) generateTableARN(ownerAccountID, bucketName, tableID string) string {
|
||||
return buildARN(h.region, ownerAccountID, fmt.Sprintf("bucket/%s/table/%s", bucketName, tableID))
|
||||
}
|
||||
|
||||
func (h *S3TablesHandler) generateViewARN(ownerAccountID, bucketName, viewID string) string {
|
||||
return buildARN(h.region, ownerAccountID, fmt.Sprintf("bucket/%s/view/%s", bucketName, viewID))
|
||||
}
|
||||
|
||||
// generateS3BucketARN builds the plain S3 ARN used for IAM resource matching.
|
||||
func (h *S3TablesHandler) generateS3BucketARN(bucketName string) string {
|
||||
return fmt.Sprintf("arn:%s:s3:::%s", arnPartitionForRegion(h.region), bucketName)
|
||||
}
|
||||
|
||||
func isAuthError(err error) bool {
|
||||
var authErr *AuthError
|
||||
return errors.As(err, &authErr) || errors.Is(err, ErrAccessDenied)
|
||||
}
|
||||
|
||||
func (h *S3TablesHandler) readTags(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) (map[string]string, error) {
|
||||
data, err := h.getExtendedAttribute(ctx, client, path, ExtendedKeyTags)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
tags := make(map[string]string)
|
||||
if err := json.Unmarshal(data, &tags); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal tags: %w", err)
|
||||
}
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
func mapKeys(tags map[string]string) []string {
|
||||
if len(tags) == 0 {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, len(tags))
|
||||
for key := range tags {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
190
weed/s3api/s3tables/handler_bucket_create.go
Normal file
190
weed/s3api/s3tables/handler_bucket_create.go
Normal file
@@ -0,0 +1,190 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
// handleCreateTableBucket creates a new table bucket
|
||||
func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req CreateTableBucketRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Validate bucket name
|
||||
if err := validateBucketName(req.Name); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// A bucket is a catalog, and a catalog serves one protocol. Saying which one
|
||||
// at creation is what lets everything downstream - the endpoint the UI
|
||||
// shows, the tables the bucket accepts - be answered without opening a table.
|
||||
bucketFormat := FormatIceberg
|
||||
if req.Format != "" {
|
||||
normalized, ok := NormalizeFormat(req.Format)
|
||||
if !ok {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, fmt.Sprintf("unsupported format %q", req.Format))
|
||||
return fmt.Errorf("invalid format")
|
||||
}
|
||||
bucketFormat = normalized
|
||||
}
|
||||
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
identityPolicyNames := getIdentityPolicyNames(r)
|
||||
useIAM := h.shouldUseIAM(r, identityActions, identityPolicyNames)
|
||||
useLegacy := !useIAM
|
||||
if useIAM {
|
||||
allowed, err := h.authorizeIAMAction(r, identityPolicyNames, "CreateTableBucket", h.generateTableBucketARN(principal, req.Name), h.generateS3BucketARN(req.Name))
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create table buckets")
|
||||
return NewAuthError("CreateTableBucket", principal, "not authorized to create table buckets")
|
||||
}
|
||||
if !allowed {
|
||||
if h.defaultAllow && len(identityActions) == 0 && len(identityPolicyNames) == 0 {
|
||||
useLegacy = true
|
||||
} else {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create table buckets")
|
||||
return NewAuthError("CreateTableBucket", principal, "not authorized to create table buckets")
|
||||
}
|
||||
}
|
||||
}
|
||||
if useLegacy {
|
||||
owner := h.accountID
|
||||
if owner == "" {
|
||||
owner = DefaultAccountID
|
||||
}
|
||||
if !CheckPermissionWithContext("CreateTableBucket", principal, owner, "", "", &PolicyContext{
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create table buckets")
|
||||
return NewAuthError("CreateTableBucket", principal, "not authorized to create table buckets")
|
||||
}
|
||||
}
|
||||
|
||||
bucketPath := GetTableBucketPath(req.Name)
|
||||
|
||||
// Check if bucket already exists and ensure no conflict with object store buckets
|
||||
tableBucketExists := false
|
||||
s3BucketExists := false
|
||||
bucketsPath := s3_constants.DefaultBucketsPath
|
||||
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.GetFilerConfiguration(r.Context(), &filer_pb.GetFilerConfigurationRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.DirBuckets != "" {
|
||||
bucketsPath = resp.DirBuckets
|
||||
}
|
||||
entryResp, err := filer_pb.LookupEntry(r.Context(), client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: bucketsPath,
|
||||
Name: req.Name,
|
||||
})
|
||||
if err != nil {
|
||||
if !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if entryResp != nil && entryResp.Entry != nil {
|
||||
if IsTableBucketEntry(entryResp.Entry) {
|
||||
tableBucketExists = true
|
||||
} else {
|
||||
s3BucketExists = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("S3Tables: failed to check bucket existence: %v", err)
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to check bucket existence")
|
||||
return err
|
||||
}
|
||||
|
||||
if tableBucketExists {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeBucketAlreadyExists, fmt.Sprintf("table bucket %s already exists", req.Name))
|
||||
return fmt.Errorf("bucket already exists")
|
||||
}
|
||||
if s3BucketExists {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeBucketAlreadyExists, fmt.Sprintf("bucket %s already exists", req.Name))
|
||||
return fmt.Errorf("bucket already exists")
|
||||
}
|
||||
|
||||
// Create the bucket directory and set metadata as extended attributes
|
||||
now := time.Now()
|
||||
metadata := &tableBucketMetadata{
|
||||
Name: req.Name,
|
||||
CreatedAt: now,
|
||||
OwnerAccountID: principal,
|
||||
Format: bucketFormat,
|
||||
}
|
||||
|
||||
metadataBytes, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
glog.Errorf("S3Tables: failed to marshal metadata: %v", err)
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal metadata")
|
||||
return err
|
||||
}
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Ensure root tables directory exists
|
||||
if !h.entryExists(r.Context(), client, TablesPath) {
|
||||
if err := h.createDirectory(r.Context(), client, TablesPath); err != nil {
|
||||
return fmt.Errorf("failed to create root tables directory: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create bucket directory
|
||||
if err := h.createDirectory(r.Context(), client, bucketPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Mark as a table bucket
|
||||
if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyTableBucket, []byte("true")); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set metadata as extended attribute
|
||||
if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata, metadataBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set tags if provided
|
||||
if len(req.Tags) > 0 {
|
||||
tagsBytes, err := json.Marshal(req.Tags)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal tags: %w", err)
|
||||
}
|
||||
if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyTags, tagsBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("S3Tables: failed to create table bucket %s: %v", req.Name, err)
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to create table bucket")
|
||||
return err
|
||||
}
|
||||
|
||||
resp := &CreateTableBucketResponse{
|
||||
ARN: h.generateTableBucketARN(metadata.OwnerAccountID, req.Name),
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
109
weed/s3api/s3tables/handler_bucket_format_test.go
Normal file
109
weed/s3api/s3tables/handler_bucket_format_test.go
Normal file
@@ -0,0 +1,109 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const formatTestBucket = "formats"
|
||||
|
||||
// bucketWithFormat lays down a table bucket holding one namespace, declared as
|
||||
// the given format. An empty format is a bucket from before the declaration
|
||||
// existed.
|
||||
func bucketWithFormat(t *testing.T, format string) (*s3tablestest.MemFiler, *Manager) {
|
||||
t.Helper()
|
||||
fs := s3tablestest.Start(t)
|
||||
m := NewManager()
|
||||
|
||||
bucketMeta, _ := json.Marshal(tableBucketMetadata{
|
||||
Name: formatTestBucket,
|
||||
OwnerAccountID: DefaultAccountID,
|
||||
Format: format,
|
||||
})
|
||||
fs.Put(TablesPath, formatTestBucket, map[string][]byte{
|
||||
ExtendedKeyTableBucket: []byte("{}"),
|
||||
ExtendedKeyMetadata: bucketMeta,
|
||||
})
|
||||
|
||||
nsMeta, _ := json.Marshal(namespaceMetadata{Namespace: []string{"ns"}, OwnerAccountID: DefaultAccountID})
|
||||
fs.Put(GetTableBucketPath(formatTestBucket), "ns", map[string][]byte{ExtendedKeyMetadata: nsMeta})
|
||||
|
||||
return fs, m
|
||||
}
|
||||
|
||||
func createTableOfFormat(t *testing.T, m *Manager, fs *s3tablestest.MemFiler, name, format string) error {
|
||||
t.Helper()
|
||||
return m.Execute(context.Background(), NewManagerClient(fs.Client), "CreateTable", &CreateTableRequest{
|
||||
TableBucketARN: "arn:aws:s3tables:::bucket/" + formatTestBucket,
|
||||
Namespace: []string{"ns"},
|
||||
Name: name,
|
||||
Format: format,
|
||||
}, nil, "")
|
||||
}
|
||||
|
||||
// The declaration is the point: a bucket that says it holds Lance cannot be
|
||||
// handed an Iceberg table, because the catalog serving it would never show one.
|
||||
func TestCreateTableRefusesAForeignFormat(t *testing.T) {
|
||||
fs, m := bucketWithFormat(t, FormatLance)
|
||||
|
||||
require.NoError(t, createTableOfFormat(t, m, fs, "vectors", FormatLance))
|
||||
|
||||
err := createTableOfFormat(t, m, fs, "events", FormatIceberg)
|
||||
require.Error(t, err, "an Iceberg table in a Lance bucket should be refused")
|
||||
require.Contains(t, err.Error(), "holds LANCE")
|
||||
}
|
||||
|
||||
func TestCreateTableRefusesLanceInAnIcebergBucket(t *testing.T) {
|
||||
fs, m := bucketWithFormat(t, FormatIceberg)
|
||||
|
||||
err := createTableOfFormat(t, m, fs, "vectors", FormatLance)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "holds ICEBERG")
|
||||
}
|
||||
|
||||
// A bucket made before formats were declared keeps taking anything. Nothing is
|
||||
// migrated, so nothing that worked stops working.
|
||||
func TestUndeclaredBucketAcceptsEitherFormat(t *testing.T) {
|
||||
fs, m := bucketWithFormat(t, "")
|
||||
|
||||
require.NoError(t, createTableOfFormat(t, m, fs, "events", FormatIceberg))
|
||||
require.NoError(t, createTableOfFormat(t, m, fs, "vectors", FormatLance))
|
||||
}
|
||||
|
||||
// A view is Iceberg metadata, so it has no meaning in a bucket of another format.
|
||||
func TestCreateViewRefusedInALanceBucket(t *testing.T) {
|
||||
fs, m := bucketWithFormat(t, FormatLance)
|
||||
|
||||
err := m.Execute(context.Background(), NewManagerClient(fs.Client), "CreateView", &CreateViewRequest{
|
||||
TableBucketARN: "arn:aws:s3tables:::bucket/" + formatTestBucket,
|
||||
Namespace: []string{"ns"},
|
||||
Name: "v",
|
||||
}, nil, "")
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "cannot hold views")
|
||||
}
|
||||
|
||||
func TestNormalizeFormat(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
ok bool
|
||||
}{
|
||||
{"ICEBERG", FormatIceberg, true},
|
||||
{"iceberg", FormatIceberg, true},
|
||||
{" Lance ", FormatLance, true},
|
||||
{"delta", "", false},
|
||||
{"", "", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, ok := NormalizeFormat(c.in)
|
||||
if got != c.want || ok != c.ok {
|
||||
t.Errorf("NormalizeFormat(%q) = %q,%v; want %q,%v", c.in, got, ok, c.want, c.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
366
weed/s3api/s3tables/handler_bucket_get_list_delete.go
Normal file
366
weed/s3api/s3tables/handler_bucket_get_list_delete.go
Normal file
@@ -0,0 +1,366 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// handleGetTableBucket gets details of a table bucket
|
||||
func (h *S3TablesHandler) handleGetTableBucket(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
// Check permission
|
||||
|
||||
var req GetTableBucketRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
|
||||
var metadata tableBucketMetadata
|
||||
var bucketPolicy string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get table bucket: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("GetTableBucket", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table bucket details")
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
resp := &GetTableBucketResponse{
|
||||
ARN: h.generateTableBucketARN(metadata.OwnerAccountID, bucketName),
|
||||
Name: metadata.Name,
|
||||
OwnerAccountID: metadata.OwnerAccountID,
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
Format: metadata.Format,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleListTableBuckets lists all table buckets
|
||||
func (h *S3TablesHandler) handleListTableBuckets(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req ListTableBucketsRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// No account-level gate: visibility is enforced per bucket below, so an
|
||||
// owner can always list its own buckets and others are filtered out.
|
||||
accountID := h.getAccountID(r)
|
||||
|
||||
maxBuckets := req.MaxBuckets
|
||||
if maxBuckets <= 0 {
|
||||
maxBuckets = 100
|
||||
}
|
||||
// Cap to prevent uint32 overflow when used in uint32(maxBuckets*2)
|
||||
const maxBucketsLimit = 1000
|
||||
if maxBuckets > maxBucketsLimit {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "MaxBuckets exceeds maximum allowed value")
|
||||
return fmt.Errorf("invalid maxBuckets value: %d", maxBuckets)
|
||||
}
|
||||
|
||||
var buckets []TableBucketSummary
|
||||
|
||||
lastFileName := req.ContinuationToken
|
||||
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
for len(buckets) < maxBuckets {
|
||||
resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{
|
||||
Directory: TablesPath,
|
||||
Limit: uint32(maxBuckets * 2), // Fetch more than needed to account for filtering
|
||||
StartFromFileName: lastFileName,
|
||||
InclusiveStartFrom: lastFileName == "" || lastFileName == req.ContinuationToken,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hasMore := false
|
||||
for {
|
||||
entry, respErr := resp.Recv()
|
||||
if respErr != nil {
|
||||
if respErr == io.EOF {
|
||||
break
|
||||
}
|
||||
return respErr
|
||||
}
|
||||
if entry.Entry == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip the start item if it was included in the previous page
|
||||
if len(buckets) == 0 && req.ContinuationToken != "" && entry.Entry.Name == req.ContinuationToken {
|
||||
continue
|
||||
}
|
||||
|
||||
hasMore = true
|
||||
lastFileName = entry.Entry.Name
|
||||
|
||||
if !entry.Entry.IsDirectory {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip entries starting with "."
|
||||
if strings.HasPrefix(entry.Entry.Name, ".") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Apply prefix filter
|
||||
if req.Prefix != "" && !strings.HasPrefix(entry.Entry.Name, req.Prefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
if !IsTableBucketEntry(entry.Entry) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Read metadata from extended attribute
|
||||
data, ok := entry.Entry.Extended[ExtendedKeyMetadata]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
var metadata tableBucketMetadata
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
bucketPath := GetTableBucketPath(entry.Entry.Name)
|
||||
bucketPolicy := ""
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, entry.Entry.Name)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("GetTableBucket", accountID, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: entry.Entry.Name,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
continue
|
||||
}
|
||||
|
||||
buckets = append(buckets, TableBucketSummary{
|
||||
ARN: bucketARN,
|
||||
Name: entry.Entry.Name,
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
Format: metadata.Format,
|
||||
})
|
||||
|
||||
if len(buckets) >= maxBuckets {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if !hasMore {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
// Check if it's a "not found" error - return empty list in that case
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
buckets = []TableBucketSummary{}
|
||||
} else {
|
||||
// For other errors, return error response
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list table buckets: %v", err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
paginationToken := ""
|
||||
if len(buckets) >= maxBuckets {
|
||||
paginationToken = lastFileName
|
||||
}
|
||||
|
||||
resp := &ListTableBucketsResponse{
|
||||
TableBuckets: buckets,
|
||||
ContinuationToken: paginationToken,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleDeleteTableBucket deletes a table bucket
|
||||
func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req DeleteTableBucketRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
|
||||
// Check if bucket exists and perform ownership + emptiness check in one block
|
||||
var metadata tableBucketMetadata
|
||||
var bucketPolicy string
|
||||
hasChildren := false
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// 1. Get metadata for ownership check
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
// No bucket policy set; proceed with empty bucketPolicy
|
||||
} else {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %w", err)
|
||||
}
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("DeleteTableBucket", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
return NewAuthError("DeleteTableBucket", principal, fmt.Sprintf("not authorized to delete bucket %s", bucketName))
|
||||
}
|
||||
|
||||
// 3. Check if bucket is empty
|
||||
resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{
|
||||
Directory: bucketPath,
|
||||
Limit: 10,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
entry, err := resp.Recv()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
if entry.Entry != nil && !strings.HasPrefix(entry.Entry.Name, ".") {
|
||||
hasChildren = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
} else if isAuthError(err) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, err.Error())
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to delete table bucket: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if hasChildren {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeBucketNotEmpty, "table bucket is not empty")
|
||||
return fmt.Errorf("bucket not empty")
|
||||
}
|
||||
|
||||
// Delete the bucket
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Drop the leaf entry first, so it does not outlive the directory.
|
||||
if tableObjErr := h.deleteEntryIfExists(r.Context(), client, GetTableObjectBucketPath(bucketName)); tableObjErr != nil {
|
||||
glog.V(1).Infof("failed to delete table object for %s: %v", bucketName, tableObjErr)
|
||||
}
|
||||
// The bucket is the directory, so a refused delete leaves it in place
|
||||
// and the caller must hear about it.
|
||||
return h.deleteDirectory(r.Context(), client, bucketPath)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table bucket")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
122
weed/s3api/s3tables/handler_delete_decouple_test.go
Normal file
122
weed/s3api/s3tables/handler_delete_decouple_test.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func runDeleteTable(t *testing.T, m *Manager, fs *s3tablestest.MemFiler, namespace, name string) error {
|
||||
t.Helper()
|
||||
return m.Execute(context.Background(), NewManagerClient(fs.Client), "DeleteTable", &DeleteTableRequest{
|
||||
TableBucketARN: mustBucketARN(t),
|
||||
Namespace: []string{namespace},
|
||||
Name: name,
|
||||
}, nil, "")
|
||||
}
|
||||
|
||||
// A table whose data was decoupled from its name (created over a leftover from a
|
||||
// rename): catalog marker at ns/newt, data at ns/newt-x, and ns/newt still holds
|
||||
// another table's leftover data. Dropping it must purge only its own data and
|
||||
// clear the marker, never the data under the reused name path.
|
||||
func TestDeleteTableDecoupledKeepsReusedNamePath(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
|
||||
newtMeta, _ := json.Marshal(tableMetadataInternal{
|
||||
Name: "newt",
|
||||
Namespace: "ns",
|
||||
Format: "ICEBERG",
|
||||
OwnerAccountID: DefaultAccountID,
|
||||
MetadataLocation: "s3://" + renameTestBucket + "/ns/newt-x/metadata/v1.metadata.json",
|
||||
})
|
||||
markerKeys := []string{ExtendedKeyMetadata, ExtendedKeyMetadataVersion, ExtendedKeyPolicy, ExtendedKeyTags, ExtendedKeyEntryType}
|
||||
fs.Put(GetNamespacePath(renameTestBucket, "ns"), "newt", map[string][]byte{
|
||||
ExtendedKeyMetadata: newtMeta,
|
||||
ExtendedKeyMetadataVersion: []byte("v1"),
|
||||
ExtendedKeyPolicy: []byte(`{"Version":"2012-10-17"}`),
|
||||
ExtendedKeyTags: []byte(`{"k":"v"}`),
|
||||
ExtendedKeyEntryType: []byte(EntryTypeTable),
|
||||
})
|
||||
fs.Put(GetTablePath(renameTestBucket, "ns", "newt"), "leftover", nil) // another table's data under the name path
|
||||
fs.Put(GetNamespacePath(renameTestBucket, "ns"), "newt-x", nil) // this table's own (decoupled) data
|
||||
fs.Put(GetTablePath(renameTestBucket, "ns", "newt-x"), "metadata", nil)
|
||||
|
||||
require.NoError(t, runDeleteTable(t, m, fs, "ns", "newt"))
|
||||
|
||||
assert.Nil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "newt-x"),
|
||||
"the table's own data location must be purged")
|
||||
assert.NotNil(t, fs.Get(GetTablePath(renameTestBucket, "ns", "newt"), "leftover"),
|
||||
"data under the reused name path must survive")
|
||||
marker := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "newt")
|
||||
require.NotNil(t, marker)
|
||||
for _, key := range markerKeys {
|
||||
_, present := marker.Extended[key]
|
||||
assert.Falsef(t, present, "catalog attribute %s must be cleared", key)
|
||||
}
|
||||
}
|
||||
|
||||
// A table whose MetadataLocation resolves to an ancestor of its own name path
|
||||
// (here the namespace root, e.g. from corrupt metadata) must not be deleted: a
|
||||
// recursive purge of that data path would wipe sibling tables. The delete is
|
||||
// refused and the namespace's other tables survive.
|
||||
func TestDeleteTableRefusesAncestorDataPath(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
|
||||
badMeta, _ := json.Marshal(tableMetadataInternal{
|
||||
Name: "badt",
|
||||
Namespace: "ns",
|
||||
Format: "ICEBERG",
|
||||
OwnerAccountID: DefaultAccountID,
|
||||
MetadataLocation: "s3://" + renameTestBucket + "/ns/metadata/v1.metadata.json",
|
||||
})
|
||||
fs.Put(GetNamespacePath(renameTestBucket, "ns"), "badt", map[string][]byte{ExtendedKeyMetadata: badMeta})
|
||||
|
||||
require.Error(t, runDeleteTable(t, m, fs, "ns", "badt"))
|
||||
|
||||
// The sibling table seeded by startRenameManager and its data must survive.
|
||||
assert.NotNil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t"),
|
||||
"sibling table marker must survive a refused delete")
|
||||
assert.NotNil(t, fs.Get(GetTablePath(renameTestBucket, "ns", "t"), "data"),
|
||||
"sibling table data must survive a refused delete")
|
||||
}
|
||||
|
||||
// A normal colocated table (data under its own name path) is removed wholesale.
|
||||
func TestDeleteTableColocatedRemovesData(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
|
||||
require.NoError(t, runDeleteTable(t, m, fs, "ns", "t"))
|
||||
|
||||
assert.Nil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t"), "colocated table entry must be deleted")
|
||||
assert.Nil(t, fs.Get(GetTablePath(renameTestBucket, "ns", "t"), "metadata"), "colocated table data must be deleted")
|
||||
}
|
||||
|
||||
// A table whose MetadataLocation points at a sibling that is still a live
|
||||
// catalog entry must not be deleted: only the named table was authorized, so a
|
||||
// recursive purge of that location would destroy another tenant's table.
|
||||
func TestDeleteTableRefusesLiveSiblingDataPath(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
|
||||
nsMeta, _ := json.Marshal(namespaceMetadata{Namespace: []string{"attackerns"}, OwnerAccountID: DefaultAccountID})
|
||||
fs.Put(GetTableBucketPath(renameTestBucket), "attackerns", map[string][]byte{ExtendedKeyMetadata: nsMeta})
|
||||
|
||||
decoyMeta, _ := json.Marshal(tableMetadataInternal{
|
||||
Name: "decoy",
|
||||
Namespace: "attackerns",
|
||||
Format: "ICEBERG",
|
||||
OwnerAccountID: DefaultAccountID,
|
||||
MetadataLocation: "s3://" + renameTestBucket + "/ns/t/metadata/v3.metadata.json",
|
||||
})
|
||||
fs.Put(GetNamespacePath(renameTestBucket, "attackerns"), "decoy", map[string][]byte{ExtendedKeyMetadata: decoyMeta})
|
||||
|
||||
require.Error(t, runDeleteTable(t, m, fs, "attackerns", "decoy"))
|
||||
|
||||
assert.NotNil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t"),
|
||||
"the targeted table's catalog entry must survive")
|
||||
assert.NotNil(t, fs.Get(GetTablePath(renameTestBucket, "ns", "t"), "data"),
|
||||
"the targeted table's data must survive")
|
||||
}
|
||||
173
weed/s3api/s3tables/handler_identity_test.go
Normal file
173
weed/s3api/s3tables/handler_identity_test.go
Normal file
@@ -0,0 +1,173 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// testIdentity/testIdentityAccount mirror the production identity shape used via reflection.
|
||||
// Keep these field names in sync with getAccountID to avoid silent breaks.
|
||||
type testIdentityAccount struct {
|
||||
Id string
|
||||
}
|
||||
|
||||
type testIdentity struct {
|
||||
Name string
|
||||
Account *testIdentityAccount
|
||||
Actions []string
|
||||
Claims map[string]interface{}
|
||||
}
|
||||
|
||||
func TestGetAccountIDPrefersClaimsOverAccountID(t *testing.T) {
|
||||
h := NewS3TablesHandler()
|
||||
id := &testIdentity{
|
||||
Account: &testIdentityAccount{Id: s3_constants.AccountAdminId},
|
||||
Claims: map[string]interface{}{
|
||||
"preferred_username": "alice",
|
||||
"sub": "alice-sub",
|
||||
},
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id))
|
||||
|
||||
got := h.getAccountID(req)
|
||||
assert.Equal(t, "alice-sub", got, "expected sub claim to be used before preferred_username")
|
||||
assert.NotEqual(t, DefaultAccountID, got, "claims should override default handler account")
|
||||
}
|
||||
|
||||
func TestGetAccountIDUsesSubWhenPreferredUsernameMissing(t *testing.T) {
|
||||
h := NewS3TablesHandler()
|
||||
id := &testIdentity{
|
||||
Account: &testIdentityAccount{Id: s3_constants.AccountAdminId},
|
||||
Claims: map[string]interface{}{
|
||||
"sub": "user-123",
|
||||
},
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id))
|
||||
|
||||
got := h.getAccountID(req)
|
||||
assert.Equal(t, "user-123", got, "expected sub claim to be used when preferred_username missing")
|
||||
assert.NotEqual(t, DefaultAccountID, got, "claims should override default handler account")
|
||||
}
|
||||
|
||||
func TestGetAccountIDFallsBackToHandlerDefaultAccount(t *testing.T) {
|
||||
h := NewS3TablesHandler()
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
|
||||
assert.Equal(t, DefaultAccountID, h.getAccountID(req), "expected handler default account to be returned when no identity is set")
|
||||
}
|
||||
|
||||
func TestGetAccountIDIgnoresEmptyClaimValues(t *testing.T) {
|
||||
h := NewS3TablesHandler()
|
||||
id := &testIdentity{
|
||||
Account: &testIdentityAccount{Id: s3_constants.AccountAdminId},
|
||||
Claims: map[string]interface{}{
|
||||
"preferred_username": " ",
|
||||
"sub": "user-123",
|
||||
},
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id))
|
||||
|
||||
assert.Equal(t, "user-123", h.getAccountID(req), "expected whitespace preferred_username to be ignored")
|
||||
}
|
||||
|
||||
func TestGetAccountIDFallsBackToIdentityName(t *testing.T) {
|
||||
h := NewS3TablesHandler()
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req = req.WithContext(s3_constants.SetIdentityNameInContext(req.Context(), "arn:aws:sts::123456789012:assumed-role/S3UserRole/alice-session"))
|
||||
|
||||
assert.Equal(t, "alice-session", h.getAccountID(req), "expected ARN session suffix to be extracted")
|
||||
}
|
||||
|
||||
func TestGetAccountIDFallsBackToARNColonSegment(t *testing.T) {
|
||||
h := NewS3TablesHandler()
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req = req.WithContext(s3_constants.SetIdentityNameInContext(req.Context(), "arn:aws:iam::123456789012:root"))
|
||||
|
||||
assert.Equal(t, "root", h.getAccountID(req), "expected ARN colon segment to be returned as principal")
|
||||
}
|
||||
|
||||
func TestGetAccountIDFallsBackToAmzAccountIdHeader(t *testing.T) {
|
||||
h := NewS3TablesHandler()
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req.Header.Set(s3_constants.AmzAccountId, "header-account")
|
||||
|
||||
assert.Equal(t, "header-account", h.getAccountID(req), "expected header value to be used when no identity is present")
|
||||
}
|
||||
|
||||
func TestGetAccountIDFallsBackToAccountID(t *testing.T) {
|
||||
h := NewS3TablesHandler()
|
||||
id := &testIdentity{
|
||||
Account: &testIdentityAccount{Id: "my-account-id"},
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id))
|
||||
|
||||
assert.Equal(t, "my-account-id", h.getAccountID(req), "expected Account.Id to be returned when claims are missing")
|
||||
}
|
||||
|
||||
func TestGetAccountIDNonAdminDoesNotInheritAdminAccount(t *testing.T) {
|
||||
h := NewS3TablesHandler()
|
||||
id := &testIdentity{
|
||||
Account: &testIdentityAccount{Id: s3_constants.AccountAdminId},
|
||||
Actions: []string{"Read", "List"},
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id))
|
||||
req = req.WithContext(s3_constants.SetIdentityNameInContext(req.Context(), "readonly"))
|
||||
|
||||
assert.Equal(t, "readonly", h.getAccountID(req), "a non-admin identity must not inherit the shared admin account")
|
||||
}
|
||||
|
||||
func TestGetAccountIDAdminActionKeepsAdminAccount(t *testing.T) {
|
||||
h := NewS3TablesHandler()
|
||||
id := &testIdentity{
|
||||
Account: &testIdentityAccount{Id: s3_constants.AccountAdminId},
|
||||
Actions: []string{s3_constants.ACTION_ADMIN},
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id))
|
||||
req = req.WithContext(s3_constants.SetIdentityNameInContext(req.Context(), "root"))
|
||||
|
||||
assert.Equal(t, s3_constants.AccountAdminId, h.getAccountID(req), "an admin identity keeps the admin account as principal")
|
||||
}
|
||||
|
||||
func TestDefaultAllowForOnlyAppliesToUnauthenticatedOrAnonymous(t *testing.T) {
|
||||
h := NewS3TablesHandler()
|
||||
h.SetDefaultAllow(true)
|
||||
|
||||
noIdentity := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
assert.True(t, h.defaultAllowFor(noIdentity), "zero-config requests with no identity keep the open default")
|
||||
|
||||
anon := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
anon = anon.WithContext(s3_constants.SetIdentityInContext(anon.Context(),
|
||||
&testIdentity{Name: s3_constants.AccountAnonymousId}))
|
||||
assert.True(t, h.defaultAllowFor(anon), "anonymous requests keep the open default")
|
||||
|
||||
authed := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
authed = authed.WithContext(s3_constants.SetIdentityInContext(authed.Context(),
|
||||
&testIdentity{Name: "readonly", Account: &testIdentityAccount{Id: s3_constants.AccountAdminId}, Actions: []string{"Read"}}))
|
||||
assert.False(t, h.defaultAllowFor(authed), "an authenticated identity must not benefit from the open default")
|
||||
|
||||
h.SetDefaultAllow(false)
|
||||
assert.False(t, h.defaultAllowFor(noIdentity), "default-allow disabled is never open")
|
||||
}
|
||||
|
||||
func TestGetAccountIDNormalizesAccountIDARN(t *testing.T) {
|
||||
h := NewS3TablesHandler()
|
||||
id := &testIdentity{
|
||||
Account: &testIdentityAccount{Id: "arn:aws:iam::123456789012:user/bob"},
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), id))
|
||||
|
||||
assert.Equal(t, "bob", h.getAccountID(req), "expected ARN account ID to be normalized to the suffix")
|
||||
}
|
||||
529
weed/s3api/s3tables/handler_maintenance.go
Normal file
529
weed/s3api/s3tables/handler_maintenance.go
Normal file
@@ -0,0 +1,529 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// Maintenance types are scoped: AWS configures unreferenced file removal on the
|
||||
// table bucket and compaction and snapshot management on the table.
|
||||
var (
|
||||
bucketMaintenanceTypes = map[string]bool{
|
||||
MaintenanceTypeIcebergUnreferencedFileRemoval: true,
|
||||
}
|
||||
tableMaintenanceTypes = map[string]bool{
|
||||
MaintenanceTypeIcebergCompaction: true,
|
||||
MaintenanceTypeIcebergSnapshotManagement: true,
|
||||
}
|
||||
)
|
||||
|
||||
// handlePutTableBucketMaintenanceConfiguration sets a maintenance configuration on a table bucket
|
||||
func (h *S3TablesHandler) handlePutTableBucketMaintenanceConfiguration(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req PutTableBucketMaintenanceConfigurationRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if err := validateMaintenanceValue(req.Type, bucketMaintenanceTypes, req.Value); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := h.authorizeMaintenanceBucket(r, filerClient, "PutTableBucketMaintenanceConfiguration", bucketName); err != nil {
|
||||
h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
return err
|
||||
}
|
||||
|
||||
if err := h.putMaintenanceConfiguration(r, filerClient, GetTableBucketPath(bucketName), req.Type, req.Value); err != nil {
|
||||
h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleGetTableBucketMaintenanceConfiguration gets the maintenance configuration of a table bucket
|
||||
func (h *S3TablesHandler) handleGetTableBucketMaintenanceConfiguration(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req GetTableBucketMaintenanceConfigurationRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN, err := h.authorizeMaintenanceBucket(r, filerClient, "GetTableBucketMaintenanceConfiguration", bucketName)
|
||||
if err != nil {
|
||||
h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
return err
|
||||
}
|
||||
|
||||
config, err := h.readMaintenanceConfiguration(r, filerClient, GetTableBucketPath(bucketName))
|
||||
if err != nil {
|
||||
h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, &GetTableBucketMaintenanceConfigurationResponse{
|
||||
TableBucketARN: bucketARN,
|
||||
Configuration: config,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// handlePutTableMaintenanceConfiguration sets a maintenance configuration on a table
|
||||
func (h *S3TablesHandler) handlePutTableMaintenanceConfiguration(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req PutTableMaintenanceConfigurationRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, namespaceName, tableName, err := h.parseTableTarget(req.TableBucketARN, req.Namespace, req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if err := validateMaintenanceValue(req.Type, tableMaintenanceTypes, req.Value); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := h.authorizeMaintenanceTable(r, filerClient, "PutTableMaintenanceConfiguration", bucketName, namespaceName, tableName); err != nil {
|
||||
h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
return err
|
||||
}
|
||||
|
||||
tablePath := GetTablePath(bucketName, namespaceName, tableName)
|
||||
if err := h.putMaintenanceConfiguration(r, filerClient, tablePath, req.Type, req.Value); err != nil {
|
||||
h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleGetTableMaintenanceConfiguration gets the maintenance configuration of a table
|
||||
func (h *S3TablesHandler) handleGetTableMaintenanceConfiguration(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req GetTableMaintenanceConfigurationRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, namespaceName, tableName, err := h.parseTableTarget(req.TableBucketARN, req.Namespace, req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
tableARN, err := h.authorizeMaintenanceTable(r, filerClient, "GetTableMaintenanceConfiguration", bucketName, namespaceName, tableName)
|
||||
if err != nil {
|
||||
h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
return err
|
||||
}
|
||||
|
||||
tablePath := GetTablePath(bucketName, namespaceName, tableName)
|
||||
config, err := h.readMaintenanceConfiguration(r, filerClient, tablePath)
|
||||
if err != nil {
|
||||
h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, &GetTableMaintenanceConfigurationResponse{
|
||||
TableARN: tableARN,
|
||||
Namespace: []string{namespaceName},
|
||||
Name: tableName,
|
||||
Configuration: config,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleGetTableMaintenanceJobStatus reports the outcome of the last maintenance run on a table
|
||||
func (h *S3TablesHandler) handleGetTableMaintenanceJobStatus(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req GetTableMaintenanceJobStatusRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, namespaceName, tableName, err := h.parseTableTarget(req.TableBucketARN, req.Namespace, req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
tableARN, err := h.authorizeMaintenanceTable(r, filerClient, "GetTableMaintenanceJobStatus", bucketName, namespaceName, tableName)
|
||||
if err != nil {
|
||||
h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
return err
|
||||
}
|
||||
|
||||
tablePath := GetTablePath(bucketName, namespaceName, tableName)
|
||||
|
||||
// Unreferenced file removal is configured on the bucket, so a bucket-level
|
||||
// disable has to show up here as Disabled rather than a stale table status.
|
||||
bucketConfig, err := h.readMaintenanceConfiguration(r, filerClient, GetTableBucketPath(bucketName))
|
||||
if err != nil {
|
||||
h.writeMaintenanceError(w, err, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
return err
|
||||
}
|
||||
tableConfig, err := h.readMaintenanceConfiguration(r, filerClient, tablePath)
|
||||
if err != nil {
|
||||
h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
return err
|
||||
}
|
||||
config := MergeMaintenanceConfiguration(bucketConfig, tableConfig)
|
||||
|
||||
recorded := MaintenanceJobStatus{}
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMaintenanceStatus)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(data, &recorded)
|
||||
})
|
||||
if err != nil {
|
||||
h.writeMaintenanceError(w, err, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, &GetTableMaintenanceJobStatusResponse{
|
||||
TableARN: tableARN,
|
||||
Status: buildJobStatusResponse(recorded, config),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildJobStatusResponse reports every job type: what the worker recorded, or
|
||||
// Disabled when the configuration switched the type off, or Not_Yet_Run.
|
||||
func buildJobStatusResponse(recorded MaintenanceJobStatus, config MaintenanceConfiguration) MaintenanceJobStatus {
|
||||
jobTypes := []string{
|
||||
MaintenanceTypeIcebergCompaction,
|
||||
MaintenanceTypeIcebergSnapshotManagement,
|
||||
MaintenanceTypeIcebergUnreferencedFileRemoval,
|
||||
}
|
||||
|
||||
status := make(MaintenanceJobStatus, len(jobTypes))
|
||||
for _, jobType := range jobTypes {
|
||||
if value, ok := config[jobType]; ok && value != nil && value.Status == MaintenanceStatusDisabled {
|
||||
status[jobType] = &MaintenanceJobStatusValue{Status: MaintenanceJobStatusDisabled}
|
||||
continue
|
||||
}
|
||||
if value, ok := recorded[jobType]; ok && value != nil {
|
||||
status[jobType] = value
|
||||
continue
|
||||
}
|
||||
status[jobType] = &MaintenanceJobStatusValue{Status: MaintenanceJobStatusNotYetRun}
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
// putMaintenanceConfiguration merges one maintenance type into the stored
|
||||
// configuration, leaving the other types alone. The write checks the catalog
|
||||
// identity in the same conditional mutation, so it cannot land on a name a
|
||||
// concurrent rename or delete has already soft-deleted.
|
||||
func (h *S3TablesHandler) putMaintenanceConfiguration(
|
||||
r *http.Request,
|
||||
filerClient FilerClient,
|
||||
resourcePath, maintenanceType string,
|
||||
value *MaintenanceConfigurationValue,
|
||||
) error {
|
||||
return filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.mutateEntryExtended(r.Context(), client, resourcePath, func(extended map[string][]byte) error {
|
||||
if len(extended[ExtendedKeyMetadata]) == 0 {
|
||||
return fmt.Errorf("%w: %s", filer_pb.ErrNotFound, resourcePath)
|
||||
}
|
||||
|
||||
config := MaintenanceConfiguration{}
|
||||
if current := extended[ExtendedKeyMaintenance]; len(current) > 0 {
|
||||
if err := json.Unmarshal(current, &config); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal maintenance configuration: %w", err)
|
||||
}
|
||||
}
|
||||
config[maintenanceType] = value
|
||||
|
||||
data, err := json.Marshal(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
extended[ExtendedKeyMaintenance] = data
|
||||
return nil
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// readMaintenanceConfiguration returns the stored configuration, or an empty
|
||||
// one when the resource has never been configured.
|
||||
func (h *S3TablesHandler) readMaintenanceConfiguration(r *http.Request, filerClient FilerClient, resourcePath string) (MaintenanceConfiguration, error) {
|
||||
config := MaintenanceConfiguration{}
|
||||
|
||||
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMaintenance)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(data, &config)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// parseTableTarget validates the bucket/namespace/table triple every
|
||||
// table-scoped maintenance request carries.
|
||||
func (h *S3TablesHandler) parseTableTarget(tableBucketARN string, namespace []string, name string) (bucketName, namespaceName, tableName string, err error) {
|
||||
if tableBucketARN == "" || len(namespace) == 0 || name == "" {
|
||||
return "", "", "", fmt.Errorf("tableBucketARN, namespace, and name are required")
|
||||
}
|
||||
if namespaceName, err = validateNamespace(namespace); err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
if bucketName, err = parseBucketNameFromARN(tableBucketARN); err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
if tableName, err = validateTableName(name); err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
return bucketName, namespaceName, tableName, nil
|
||||
}
|
||||
|
||||
// authorizeMaintenanceBucket checks the caller may perform operation on the
|
||||
// table bucket, returning the bucket ARN.
|
||||
func (h *S3TablesHandler) authorizeMaintenanceBucket(r *http.Request, filerClient FilerClient, operation, bucketName string) (string, error) {
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
|
||||
var bucketMetadata tableBucketMetadata
|
||||
var bucketPolicy string
|
||||
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
bucketPolicy, err = h.readBucketPolicy(r, client, bucketPath)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
if !CheckPermissionWithContext(operation, principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: getIdentityActions(r),
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
return "", NewAuthError(operation, principal, "not authorized to "+operation)
|
||||
}
|
||||
|
||||
return bucketARN, nil
|
||||
}
|
||||
|
||||
// authorizeMaintenanceTable checks the caller may perform operation on the
|
||||
// table, returning the table ARN.
|
||||
func (h *S3TablesHandler) authorizeMaintenanceTable(r *http.Request, filerClient FilerClient, operation, bucketName, namespaceName, tableName string) (string, error) {
|
||||
tablePath := GetTablePath(bucketName, namespaceName, tableName)
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
|
||||
var metadata tableMetadataInternal
|
||||
var bucketPolicy string
|
||||
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal table metadata: %w", err)
|
||||
}
|
||||
bucketPolicy, err = h.readBucketPolicy(r, client, bucketPath)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName)
|
||||
principal := h.getAccountID(r)
|
||||
if !CheckPermissionWithContext(operation, principal, metadata.OwnerAccountID, bucketPolicy, tableARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableName: tableName,
|
||||
IdentityActions: getIdentityActions(r),
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
return "", NewAuthError(operation, principal, "not authorized to "+operation)
|
||||
}
|
||||
|
||||
return tableARN, nil
|
||||
}
|
||||
|
||||
// readBucketPolicy returns the bucket policy, or an empty string when the
|
||||
// bucket has none.
|
||||
func (h *S3TablesHandler) readBucketPolicy(r *http.Request, client filer_pb.SeaweedFilerClient, bucketPath string) (string, error) {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
return "", nil
|
||||
}
|
||||
return "", fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
// writeMaintenanceError maps the shared failure modes of the maintenance
|
||||
// handlers onto responses.
|
||||
func (h *S3TablesHandler) writeMaintenanceError(w http.ResponseWriter, err error, notFoundCode, notFoundMessage string) {
|
||||
switch {
|
||||
case errors.Is(err, filer_pb.ErrNotFound):
|
||||
h.writeError(w, http.StatusNotFound, notFoundCode, notFoundMessage)
|
||||
case isAuthError(err):
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, err.Error())
|
||||
case errors.Is(err, ErrConcurrentUpdate):
|
||||
h.writeError(w, http.StatusConflict, ErrCodeConflict, "maintenance configuration changed concurrently, retry the request")
|
||||
default:
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// validateMaintenanceValue rejects a type that does not belong to the scope and
|
||||
// settings that do not name that same type.
|
||||
func validateMaintenanceValue(maintenanceType string, allowed map[string]bool, value *MaintenanceConfigurationValue) error {
|
||||
if maintenanceType == "" {
|
||||
return fmt.Errorf("type is required")
|
||||
}
|
||||
if !allowed[maintenanceType] {
|
||||
return fmt.Errorf("unsupported maintenance type %q", maintenanceType)
|
||||
}
|
||||
if value == nil {
|
||||
return fmt.Errorf("value is required")
|
||||
}
|
||||
|
||||
switch value.Status {
|
||||
case MaintenanceStatusEnabled, MaintenanceStatusDisabled:
|
||||
case "":
|
||||
return fmt.Errorf("value.status is required")
|
||||
default:
|
||||
return fmt.Errorf("invalid value.status %q, expected %q or %q", value.Status, MaintenanceStatusEnabled, MaintenanceStatusDisabled)
|
||||
}
|
||||
|
||||
if value.Settings != nil && !settingsMatchType(maintenanceType, value.Settings) {
|
||||
return fmt.Errorf("value.settings must only contain %s", maintenanceType)
|
||||
}
|
||||
return validateMaintenanceSettings(value.Settings)
|
||||
}
|
||||
|
||||
// AWS bounds every maintenance setting to a positive 32-bit value. Accepting
|
||||
// anything else would store a number the worker then ignores or saturates, so
|
||||
// the configuration read back would not be the one that runs.
|
||||
const (
|
||||
maintenanceSettingMin int64 = 1
|
||||
maintenanceSettingMax int64 = 2147483647
|
||||
)
|
||||
|
||||
func validateMaintenanceSettings(settings *MaintenanceSettings) error {
|
||||
if settings == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if c := settings.IcebergCompaction; c != nil {
|
||||
if err := validateCompactionStrategy(c.Strategy); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateMaintenanceSetting("targetFileSizeMB", c.TargetFileSizeMB); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if m := settings.IcebergSnapshotManagement; m != nil {
|
||||
if err := validateMaintenanceSetting("minSnapshotsToKeep", m.MinSnapshotsToKeep); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateMaintenanceSetting("maxSnapshotAgeHours", m.MaxSnapshotAgeHours); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if u := settings.IcebergUnreferencedFileRemoval; u != nil {
|
||||
if err := validateMaintenanceSetting("unreferencedDays", u.UnreferencedDays); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateMaintenanceSetting("nonCurrentDays", u.NonCurrentDays); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateMaintenanceSetting(name string, value *int64) error {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
if *value < maintenanceSettingMin || *value > maintenanceSettingMax {
|
||||
return fmt.Errorf("%s must be between %d and %d, got %d", name, maintenanceSettingMin, maintenanceSettingMax, *value)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateCompactionStrategy rejects a strategy the maintenance worker cannot
|
||||
// carry out, rather than accepting it and quietly compacting some other way.
|
||||
func validateCompactionStrategy(strategy string) error {
|
||||
switch strategy {
|
||||
case "", CompactionStrategyAuto, CompactionStrategyBinpack, CompactionStrategySort:
|
||||
return nil
|
||||
case CompactionStrategyZOrder:
|
||||
return fmt.Errorf("compaction strategy %q is not supported", strategy)
|
||||
default:
|
||||
return fmt.Errorf("invalid compaction strategy %q, expected one of %q, %q, %q",
|
||||
strategy, CompactionStrategyAuto, CompactionStrategyBinpack, CompactionStrategySort)
|
||||
}
|
||||
}
|
||||
|
||||
func settingsMatchType(maintenanceType string, settings *MaintenanceSettings) bool {
|
||||
switch maintenanceType {
|
||||
case MaintenanceTypeIcebergCompaction:
|
||||
return settings.IcebergSnapshotManagement == nil && settings.IcebergUnreferencedFileRemoval == nil
|
||||
case MaintenanceTypeIcebergSnapshotManagement:
|
||||
return settings.IcebergCompaction == nil && settings.IcebergUnreferencedFileRemoval == nil
|
||||
case MaintenanceTypeIcebergUnreferencedFileRemoval:
|
||||
return settings.IcebergCompaction == nil && settings.IcebergSnapshotManagement == nil
|
||||
}
|
||||
return false
|
||||
}
|
||||
476
weed/s3api/s3tables/handler_maintenance_test.go
Normal file
476
weed/s3api/s3tables/handler_maintenance_test.go
Normal file
@@ -0,0 +1,476 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func settingPtr(v int64) *int64 { return &v }
|
||||
|
||||
func TestValidateMaintenanceValueAcceptsScopedTypes(t *testing.T) {
|
||||
err := validateMaintenanceValue(MaintenanceTypeIcebergCompaction, tableMaintenanceTypes, &MaintenanceConfigurationValue{
|
||||
Status: MaintenanceStatusEnabled,
|
||||
Settings: &MaintenanceSettings{IcebergCompaction: &IcebergCompactionSettings{TargetFileSizeMB: settingPtr(512)}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("expected compaction accepted on a table, got %v", err)
|
||||
}
|
||||
|
||||
err = validateMaintenanceValue(MaintenanceTypeIcebergUnreferencedFileRemoval, bucketMaintenanceTypes, &MaintenanceConfigurationValue{
|
||||
Status: MaintenanceStatusDisabled,
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("expected unreferenced file removal accepted on a bucket, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMaintenanceValueRejectsWrongScope(t *testing.T) {
|
||||
if err := validateMaintenanceValue(MaintenanceTypeIcebergCompaction, bucketMaintenanceTypes, &MaintenanceConfigurationValue{
|
||||
Status: MaintenanceStatusEnabled,
|
||||
}); err == nil {
|
||||
t.Error("expected compaction rejected on a table bucket")
|
||||
}
|
||||
|
||||
if err := validateMaintenanceValue(MaintenanceTypeIcebergUnreferencedFileRemoval, tableMaintenanceTypes, &MaintenanceConfigurationValue{
|
||||
Status: MaintenanceStatusEnabled,
|
||||
}); err == nil {
|
||||
t.Error("expected unreferenced file removal rejected on a table")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMaintenanceValueRejectsBadInput(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
maintenanceType string
|
||||
value *MaintenanceConfigurationValue
|
||||
}{
|
||||
"empty type": {"", &MaintenanceConfigurationValue{Status: MaintenanceStatusEnabled}},
|
||||
"unknown type": {"icebergSomethingElse", &MaintenanceConfigurationValue{Status: MaintenanceStatusEnabled}},
|
||||
"nil value": {MaintenanceTypeIcebergCompaction, nil},
|
||||
"no status": {MaintenanceTypeIcebergCompaction, &MaintenanceConfigurationValue{}},
|
||||
"bad status": {MaintenanceTypeIcebergCompaction, &MaintenanceConfigurationValue{Status: "Enabled"}},
|
||||
"settings for another type": {MaintenanceTypeIcebergCompaction, &MaintenanceConfigurationValue{
|
||||
Status: MaintenanceStatusEnabled,
|
||||
Settings: &MaintenanceSettings{IcebergSnapshotManagement: &IcebergSnapshotManagementSettings{MinSnapshotsToKeep: settingPtr(3)}},
|
||||
}},
|
||||
}
|
||||
|
||||
for name, tc := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if err := validateMaintenanceValue(tc.maintenanceType, tableMaintenanceTypes, tc.value); err == nil {
|
||||
t.Error("expected rejection")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The stored attribute is the wire shape, so Get can hand it straight back.
|
||||
func TestMaintenanceConfigurationRoundTrip(t *testing.T) {
|
||||
original := MaintenanceConfiguration{
|
||||
MaintenanceTypeIcebergCompaction: {
|
||||
Status: MaintenanceStatusEnabled,
|
||||
Settings: &MaintenanceSettings{IcebergCompaction: &IcebergCompactionSettings{TargetFileSizeMB: settingPtr(512)}},
|
||||
},
|
||||
MaintenanceTypeIcebergSnapshotManagement: {
|
||||
Status: MaintenanceStatusDisabled,
|
||||
Settings: &MaintenanceSettings{IcebergSnapshotManagement: &IcebergSnapshotManagementSettings{
|
||||
MinSnapshotsToKeep: settingPtr(3),
|
||||
MaxSnapshotAgeHours: settingPtr(120),
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
data, err := json.Marshal(original)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
|
||||
var decoded MaintenanceConfiguration
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
|
||||
compaction := decoded[MaintenanceTypeIcebergCompaction]
|
||||
if compaction == nil || compaction.Settings == nil || compaction.Settings.IcebergCompaction == nil {
|
||||
t.Fatalf("expected compaction settings, got %s", data)
|
||||
}
|
||||
if got := compaction.Settings.IcebergCompaction.TargetFileSizeMB; got == nil || *got != 512 {
|
||||
t.Errorf("expected targetFileSizeMB=512, got %v", got)
|
||||
}
|
||||
|
||||
snapshots := decoded[MaintenanceTypeIcebergSnapshotManagement]
|
||||
if snapshots == nil || snapshots.Status != MaintenanceStatusDisabled {
|
||||
t.Fatalf("expected snapshot management disabled, got %s", data)
|
||||
}
|
||||
if got := snapshots.Settings.IcebergSnapshotManagement.MaxSnapshotAgeHours; got == nil || *got != 120 {
|
||||
t.Errorf("expected maxSnapshotAgeHours=120, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Putting one type must not drop the others already stored.
|
||||
func TestMaintenanceConfigurationMergeKeepsOtherTypes(t *testing.T) {
|
||||
stored, err := json.Marshal(MaintenanceConfiguration{
|
||||
MaintenanceTypeIcebergCompaction: {Status: MaintenanceStatusEnabled},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
|
||||
config := MaintenanceConfiguration{}
|
||||
if err := json.Unmarshal(stored, &config); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
config[MaintenanceTypeIcebergSnapshotManagement] = &MaintenanceConfigurationValue{Status: MaintenanceStatusDisabled}
|
||||
|
||||
if len(config) != 2 {
|
||||
t.Errorf("expected both types retained, got %+v", config)
|
||||
}
|
||||
if config[MaintenanceTypeIcebergCompaction].Status != MaintenanceStatusEnabled {
|
||||
t.Error("expected the existing compaction entry preserved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTableTarget(t *testing.T) {
|
||||
h := NewS3TablesHandler()
|
||||
arn := h.generateTableBucketARN(DefaultAccountID, "analytics")
|
||||
|
||||
bucketName, namespaceName, tableName, err := h.parseTableTarget(arn, []string{"sales"}, "orders")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if bucketName != "analytics" || namespaceName != "sales" || tableName != "orders" {
|
||||
t.Errorf("got %q/%q/%q", bucketName, namespaceName, tableName)
|
||||
}
|
||||
|
||||
for name, tc := range map[string]struct {
|
||||
arn string
|
||||
namespace []string
|
||||
table string
|
||||
}{
|
||||
"missing arn": {"", []string{"sales"}, "orders"},
|
||||
"missing namespace": {arn, nil, "orders"},
|
||||
"missing name": {arn, []string{"sales"}, ""},
|
||||
"bad arn": {"not-an-arn", []string{"sales"}, "orders"},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if _, _, _, err := h.parseTableTarget(tc.arn, tc.namespace, tc.table); err == nil {
|
||||
t.Error("expected rejection")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildJobStatusResponseReportsEveryType(t *testing.T) {
|
||||
status := buildJobStatusResponse(nil, nil)
|
||||
if len(status) != 3 {
|
||||
t.Fatalf("expected all three job types reported, got %v", status)
|
||||
}
|
||||
for jobType, value := range status {
|
||||
if value.Status != MaintenanceJobStatusNotYetRun {
|
||||
t.Errorf("expected %s not yet run, got %q", jobType, value.Status)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildJobStatusResponseDisabledBeatsRecorded(t *testing.T) {
|
||||
recorded := MaintenanceJobStatus{
|
||||
MaintenanceTypeIcebergCompaction: {Status: MaintenanceJobStatusSuccessful},
|
||||
MaintenanceTypeIcebergSnapshotManagement: {Status: MaintenanceJobStatusFailed, FailureMessage: "boom"},
|
||||
}
|
||||
config := MaintenanceConfiguration{
|
||||
MaintenanceTypeIcebergCompaction: {Status: MaintenanceStatusDisabled},
|
||||
}
|
||||
|
||||
status := buildJobStatusResponse(recorded, config)
|
||||
if status[MaintenanceTypeIcebergCompaction].Status != MaintenanceJobStatusDisabled {
|
||||
t.Errorf("expected a disabled type reported as disabled, got %q", status[MaintenanceTypeIcebergCompaction].Status)
|
||||
}
|
||||
if status[MaintenanceTypeIcebergSnapshotManagement].FailureMessage != "boom" {
|
||||
t.Error("expected the recorded failure preserved")
|
||||
}
|
||||
if status[MaintenanceTypeIcebergUnreferencedFileRemoval].Status != MaintenanceJobStatusNotYetRun {
|
||||
t.Error("expected an unrun type reported as not yet run")
|
||||
}
|
||||
}
|
||||
|
||||
// A strategy the worker cannot carry out must be rejected, not accepted and
|
||||
// silently compacted some other way.
|
||||
func TestValidateMaintenanceValueCompactionStrategy(t *testing.T) {
|
||||
for _, strategy := range []string{"", CompactionStrategyAuto, CompactionStrategyBinpack, CompactionStrategySort} {
|
||||
if err := validateMaintenanceValue(MaintenanceTypeIcebergCompaction, tableMaintenanceTypes, &MaintenanceConfigurationValue{
|
||||
Status: MaintenanceStatusEnabled,
|
||||
Settings: &MaintenanceSettings{IcebergCompaction: &IcebergCompactionSettings{Strategy: strategy}},
|
||||
}); err != nil {
|
||||
t.Errorf("expected strategy %q accepted, got %v", strategy, err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, strategy := range []string{CompactionStrategyZOrder, "nonsense"} {
|
||||
if err := validateMaintenanceValue(MaintenanceTypeIcebergCompaction, tableMaintenanceTypes, &MaintenanceConfigurationValue{
|
||||
Status: MaintenanceStatusEnabled,
|
||||
Settings: &MaintenanceSettings{IcebergCompaction: &IcebergCompactionSettings{Strategy: strategy}},
|
||||
}); err == nil {
|
||||
t.Errorf("expected strategy %q rejected", strategy)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Put then Get must return the strategy unchanged.
|
||||
func TestMaintenanceConfigurationPreservesStrategy(t *testing.T) {
|
||||
data, err := json.Marshal(MaintenanceConfiguration{
|
||||
MaintenanceTypeIcebergCompaction: {
|
||||
Status: MaintenanceStatusEnabled,
|
||||
Settings: &MaintenanceSettings{IcebergCompaction: &IcebergCompactionSettings{Strategy: CompactionStrategySort, TargetFileSizeMB: settingPtr(512)}},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
|
||||
var decoded MaintenanceConfiguration
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if got := decoded[MaintenanceTypeIcebergCompaction].Settings.IcebergCompaction.Strategy; got != CompactionStrategySort {
|
||||
t.Errorf("expected the strategy preserved, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Unreferenced file removal is configured on the bucket, so a bucket-level
|
||||
// disable has to surface here rather than a stale table status.
|
||||
func TestBuildJobStatusResponseHonoursBucketDisable(t *testing.T) {
|
||||
recorded := MaintenanceJobStatus{
|
||||
MaintenanceTypeIcebergUnreferencedFileRemoval: {Status: MaintenanceJobStatusSuccessful},
|
||||
}
|
||||
bucketConfig := MaintenanceConfiguration{
|
||||
MaintenanceTypeIcebergUnreferencedFileRemoval: {Status: MaintenanceStatusDisabled},
|
||||
}
|
||||
|
||||
status := buildJobStatusResponse(recorded, MergeMaintenanceConfiguration(bucketConfig, nil))
|
||||
if got := status[MaintenanceTypeIcebergUnreferencedFileRemoval].Status; got != MaintenanceJobStatusDisabled {
|
||||
t.Errorf("expected Disabled from the bucket configuration, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeMaintenanceConfiguration(t *testing.T) {
|
||||
bucket := MaintenanceConfiguration{
|
||||
MaintenanceTypeIcebergUnreferencedFileRemoval: {Status: MaintenanceStatusEnabled},
|
||||
MaintenanceTypeIcebergCompaction: {Status: MaintenanceStatusEnabled},
|
||||
}
|
||||
table := MaintenanceConfiguration{
|
||||
MaintenanceTypeIcebergCompaction: {Status: MaintenanceStatusDisabled},
|
||||
}
|
||||
|
||||
merged := MergeMaintenanceConfiguration(bucket, table)
|
||||
if merged[MaintenanceTypeIcebergCompaction].Status != MaintenanceStatusDisabled {
|
||||
t.Error("expected the table entry to win")
|
||||
}
|
||||
if merged[MaintenanceTypeIcebergUnreferencedFileRemoval].Status != MaintenanceStatusEnabled {
|
||||
t.Error("expected the bucket-only entry retained")
|
||||
}
|
||||
if got := MergeMaintenanceConfiguration(nil, table); len(got) != 1 {
|
||||
t.Errorf("expected the table configuration passed through, got %v", got)
|
||||
}
|
||||
if got := MergeMaintenanceConfiguration(bucket, nil); len(got) != 2 {
|
||||
t.Errorf("expected the bucket configuration passed through, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateEntry rewrites the whole entry, so the precondition has to cover every
|
||||
// attribute this package stores — including the ones absent right now, which a
|
||||
// concurrent writer can create and this write would then delete.
|
||||
func TestSnapshotExtendedCoversEveryAttribute(t *testing.T) {
|
||||
extended := map[string][]byte{
|
||||
ExtendedKeyMetadata: []byte(`{"a":1}`),
|
||||
ExtendedKeyMaintenance: []byte(`{"icebergCompaction":{"status":"disabled"}}`),
|
||||
}
|
||||
|
||||
expected := SnapshotExtended(extended, ExtendedKeyMaintenanceStatus)
|
||||
|
||||
for key, want := range extended {
|
||||
if string(expected[key]) != string(want) {
|
||||
t.Errorf("attribute %s not carried into the precondition", key)
|
||||
}
|
||||
}
|
||||
for _, key := range s3tablesExtendedKeys {
|
||||
if _, ok := expected[key]; !ok {
|
||||
t.Errorf("attribute %s missing from the precondition", key)
|
||||
}
|
||||
}
|
||||
|
||||
// The absent ones are asserted absent, so a concurrent create fails the write.
|
||||
for _, key := range []string{ExtendedKeyMaintenanceStatus, ExtendedKeyPolicy, ExtendedKeyTags} {
|
||||
if got, ok := expected[key]; !ok || len(got) != 0 {
|
||||
t.Errorf("expected %s asserted absent, got %v", key, got)
|
||||
}
|
||||
}
|
||||
|
||||
// Mutating the snapshot must not disturb the entry it came from.
|
||||
expected[ExtendedKeyMetadata] = []byte("changed")
|
||||
if string(extended[ExtendedKeyMetadata]) != `{"a":1}` {
|
||||
t.Error("expected the source map left alone")
|
||||
}
|
||||
}
|
||||
|
||||
// The regression this closes: a maintenance configuration that did not exist
|
||||
// when the writer read the entry must still be asserted, or a first-time
|
||||
// disable lands between the read and the write and gets erased.
|
||||
func TestSnapshotExtendedAssertsAbsentMaintenanceKey(t *testing.T) {
|
||||
expected := SnapshotExtended(map[string][]byte{ExtendedKeyMetadata: []byte(`{}`)})
|
||||
|
||||
value, ok := expected[ExtendedKeyMaintenance]
|
||||
if !ok {
|
||||
t.Fatal("expected the maintenance key asserted even when absent")
|
||||
}
|
||||
if len(value) != 0 {
|
||||
t.Errorf("expected an absent assertion, got %v", value)
|
||||
}
|
||||
}
|
||||
|
||||
// AWS bounds every numeric setting to 1..2147483647. Accepting anything else
|
||||
// would store a value the worker ignores or saturates, so the configuration
|
||||
// read back would not be the one that runs.
|
||||
func TestValidateMaintenanceSettingRange(t *testing.T) {
|
||||
compaction := func(v *int64) error {
|
||||
return validateMaintenanceValue(MaintenanceTypeIcebergCompaction, tableMaintenanceTypes, &MaintenanceConfigurationValue{
|
||||
Status: MaintenanceStatusEnabled,
|
||||
Settings: &MaintenanceSettings{IcebergCompaction: &IcebergCompactionSettings{TargetFileSizeMB: v}},
|
||||
})
|
||||
}
|
||||
|
||||
if err := compaction(nil); err != nil {
|
||||
t.Errorf("expected an unset setting accepted, got %v", err)
|
||||
}
|
||||
for _, v := range []int64{1, 512, maintenanceSettingMax} {
|
||||
if err := compaction(settingPtr(v)); err != nil {
|
||||
t.Errorf("expected %d accepted, got %v", v, err)
|
||||
}
|
||||
}
|
||||
for _, v := range []int64{0, -1, maintenanceSettingMax + 1, 1 << 62} {
|
||||
if err := compaction(settingPtr(v)); err == nil {
|
||||
t.Errorf("expected %d rejected", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMaintenanceSettingRangeCoversEveryField(t *testing.T) {
|
||||
cases := map[string]*MaintenanceSettings{
|
||||
"minSnapshotsToKeep": {IcebergSnapshotManagement: &IcebergSnapshotManagementSettings{MinSnapshotsToKeep: settingPtr(0)}},
|
||||
"maxSnapshotAgeHours": {IcebergSnapshotManagement: &IcebergSnapshotManagementSettings{MaxSnapshotAgeHours: settingPtr(-5)}},
|
||||
"unreferencedDays": {IcebergUnreferencedFileRemoval: &IcebergUnreferencedFileRemovalSettings{UnreferencedDays: settingPtr(maintenanceSettingMax + 1)}},
|
||||
"nonCurrentDays": {IcebergUnreferencedFileRemoval: &IcebergUnreferencedFileRemovalSettings{NonCurrentDays: settingPtr(0)}},
|
||||
}
|
||||
|
||||
for name, settings := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if err := validateMaintenanceSettings(settings); err == nil {
|
||||
t.Errorf("expected %s out of range to be rejected", name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// An explicit zero has to be distinguishable from an omitted field, or it is
|
||||
// silently accepted and then ignored.
|
||||
func TestMaintenanceSettingsDistinguishZeroFromUnset(t *testing.T) {
|
||||
var decoded MaintenanceSettings
|
||||
if err := json.Unmarshal([]byte(`{"icebergCompaction":{"targetFileSizeMB":0}}`), &decoded); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if decoded.IcebergCompaction.TargetFileSizeMB == nil {
|
||||
t.Fatal("expected an explicit zero to survive decoding")
|
||||
}
|
||||
|
||||
var omitted MaintenanceSettings
|
||||
if err := json.Unmarshal([]byte(`{"icebergCompaction":{}}`), &omitted); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
if omitted.IcebergCompaction.TargetFileSizeMB != nil {
|
||||
t.Error("expected an omitted field to stay unset")
|
||||
}
|
||||
}
|
||||
|
||||
// aws-cn and aws-us-gov ARNs are valid and must parse; the emitted ARN has to
|
||||
// use the partition its region belongs to so it round-trips.
|
||||
func TestARNPartitionsBeyondCommercial(t *testing.T) {
|
||||
for _, arn := range []string{
|
||||
"arn:aws:s3tables:us-east-1:123456789012:bucket/analytics",
|
||||
"arn:aws-cn:s3tables:cn-north-1:123456789012:bucket/analytics",
|
||||
"arn:aws-us-gov:s3tables:us-gov-west-1:123456789012:bucket/analytics",
|
||||
} {
|
||||
got, err := parseBucketNameFromARN(arn)
|
||||
if err != nil {
|
||||
t.Errorf("expected %s to parse, got %v", arn, err)
|
||||
continue
|
||||
}
|
||||
if got != "analytics" {
|
||||
t.Errorf("expected bucket analytics from %s, got %q", arn, got)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := parseBucketNameFromARN("arn:notaws:s3tables:us-east-1:123456789012:bucket/analytics"); err == nil {
|
||||
t.Error("expected a non-AWS partition to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestARNPartitionForRegion(t *testing.T) {
|
||||
for region, want := range map[string]string{
|
||||
"us-east-1": "aws",
|
||||
"eu-west-2": "aws",
|
||||
"cn-north-1": "aws-cn",
|
||||
"cn-northwest-1": "aws-cn",
|
||||
"us-gov-west-1": "aws-us-gov",
|
||||
"us-iso-east-1": "aws-iso",
|
||||
"us-isob-east-1": "aws-iso-b",
|
||||
"eu-isoe-west-1": "aws-iso-e",
|
||||
"us-isof-south-1": "aws-iso-f",
|
||||
"eusc-de-east-1": "aws-eusc",
|
||||
} {
|
||||
if got := arnPartitionForRegion(region); got != want {
|
||||
t.Errorf("region %s: expected partition %q, got %q", region, want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An ARN this handler emits has to carry the partition its region belongs to,
|
||||
// not just parse back — a commercial ARN returned for a gov region still parses
|
||||
// but is wrong for IAM matching and for the caller.
|
||||
func TestGeneratedARNsUseTheRegionPartition(t *testing.T) {
|
||||
for region, partition := range map[string]string{
|
||||
"us-east-1": "aws",
|
||||
"cn-north-1": "aws-cn",
|
||||
"us-gov-west-1": "aws-us-gov",
|
||||
} {
|
||||
h := NewS3TablesHandler()
|
||||
h.SetRegion(region)
|
||||
|
||||
bucketARN := h.generateTableBucketARN(DefaultAccountID, "analytics")
|
||||
tableARN := h.generateTableARN(DefaultAccountID, "analytics", "sales/orders")
|
||||
viewARN := h.generateViewARN(DefaultAccountID, "analytics", "sales/v1")
|
||||
s3ARN := h.generateS3BucketARN("analytics")
|
||||
|
||||
for _, arn := range []string{bucketARN, tableARN, viewARN, s3ARN} {
|
||||
if !strings.HasPrefix(arn, "arn:"+partition+":") {
|
||||
t.Errorf("region %s: expected partition %q, got %s", region, partition, arn)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := parseBucketNameFromARN(bucketARN)
|
||||
if err != nil {
|
||||
t.Errorf("generated ARN %s does not parse: %v", bucketARN, err)
|
||||
continue
|
||||
}
|
||||
if got != "analytics" {
|
||||
t.Errorf("expected analytics from %s, got %q", bucketARN, got)
|
||||
}
|
||||
|
||||
bucket, namespace, table, err := parseTableFromARN(tableARN)
|
||||
if err != nil {
|
||||
t.Errorf("generated table ARN %s does not parse: %v", tableARN, err)
|
||||
continue
|
||||
}
|
||||
if bucket != "analytics" || namespace != "sales" || table != "orders" {
|
||||
t.Errorf("unexpected parse of %s: %s/%s/%s", tableARN, bucket, namespace, table)
|
||||
}
|
||||
}
|
||||
}
|
||||
37
weed/s3api/s3tables/handler_name_charset_test.go
Normal file
37
weed/s3api/s3tables/handler_name_charset_test.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNameValidationAllowsHyphens(t *testing.T) {
|
||||
// Hyphen interior is accepted (Iceberg REST clients use names like these).
|
||||
require.NoError(t, validateNamespacePart("rest-integration-test"))
|
||||
if _, err := validateTableName("test-create-table"); err != nil {
|
||||
t.Fatalf("hyphenated table name rejected: %v", err)
|
||||
}
|
||||
|
||||
// A leading or trailing hyphen on a namespace is still rejected (must start
|
||||
// and end with a letter or digit), as are characters outside the charset.
|
||||
require.Error(t, validateNamespacePart("-leading"))
|
||||
require.Error(t, validateNamespacePart("trailing-"))
|
||||
require.Error(t, validateNamespacePart("Upper"))
|
||||
|
||||
// Table names must still start with a letter or digit and stay in charset.
|
||||
if _, err := validateTableName("-leading"); err == nil {
|
||||
t.Fatal("table name with leading hyphen should be rejected")
|
||||
}
|
||||
if _, err := validateTableName("Upper"); err == nil {
|
||||
t.Fatal("uppercase table name should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTableFromARNAllowsHyphens(t *testing.T) {
|
||||
bucket, ns, table, err := parseTableFromARN("arn:aws:s3tables:us-east-1:123456789012:bucket/my-bucket/table/rest-integration-test/test-create-table")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "my-bucket", bucket)
|
||||
require.Equal(t, "rest-integration-test", ns)
|
||||
require.Equal(t, "test-create-table", table)
|
||||
}
|
||||
704
weed/s3api/s3tables/handler_namespace.go
Normal file
704
weed/s3api/s3tables/handler_namespace.go
Normal file
@@ -0,0 +1,704 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// handleCreateNamespace creates a new namespace in a table bucket
|
||||
func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
glog.Errorf("S3Tables: handleCreateNamespace called")
|
||||
var req CreateNamespaceRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
glog.Errorf("S3Tables: handleCreateNamespace failed to read request body: %v", err)
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
if len(req.Namespace) == 0 {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "namespace is required")
|
||||
return fmt.Errorf("namespace is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if table bucket exists
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
var bucketMetadata tableBucketMetadata
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
ownerAccountID := h.getAccountID(r)
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
dir, name := splitPath(bucketPath)
|
||||
entryResp, lookupErr := filer_pb.LookupEntry(r.Context(), client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
if lookupErr != nil {
|
||||
return lookupErr
|
||||
}
|
||||
if entryResp.Entry == nil || !IsTableBucketEntry(entryResp.Entry) {
|
||||
return filer_pb.ErrNotFound
|
||||
}
|
||||
bucketMetadata = tableBucketMetadata{
|
||||
Name: bucketName,
|
||||
CreatedAt: time.Now(),
|
||||
OwnerAccountID: ownerAccountID,
|
||||
}
|
||||
metadataBytes, err := json.Marshal(&bucketMetadata)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal bucket metadata: %w", err)
|
||||
}
|
||||
if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata, metadataBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
} else if err := json.Unmarshal(data, &bucketMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table bucket: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
glog.Infof("S3Tables: CreateNamespace permission check - principal=%s, owner=%s, actions=%v", principal, bucketMetadata.OwnerAccountID, identityActions)
|
||||
if !CheckPermissionWithContext("CreateNamespace", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
glog.Infof("S3Tables: Permission denied for CreateNamespace - principal=%s, owner=%s", principal, bucketMetadata.OwnerAccountID)
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create namespace in this bucket")
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
namespacePath := GetNamespacePath(bucketName, namespaceName)
|
||||
|
||||
// Check if namespace already exists
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
_, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
|
||||
return err
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeNamespaceAlreadyExists, fmt.Sprintf("namespace %s already exists", namespaceName))
|
||||
return fmt.Errorf("namespace already exists")
|
||||
} else if !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check namespace: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
// Create the namespace with bucket owner to maintain consistency
|
||||
// (authorization above ensures the caller has permission to create in this bucket)
|
||||
now := time.Now()
|
||||
metadata := &namespaceMetadata{
|
||||
Namespace: req.Namespace,
|
||||
CreatedAt: now,
|
||||
OwnerAccountID: bucketMetadata.OwnerAccountID,
|
||||
Properties: req.Properties,
|
||||
}
|
||||
|
||||
metadataBytes, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal namespace metadata")
|
||||
return fmt.Errorf("failed to marshal metadata: %w", err)
|
||||
}
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Create namespace directory
|
||||
if err := h.createDirectory(r.Context(), client, namespacePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set metadata as extended attribute
|
||||
if err := h.setExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata, metadataBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to create namespace")
|
||||
return err
|
||||
}
|
||||
|
||||
resp := &CreateNamespaceResponse{
|
||||
Namespace: req.Namespace,
|
||||
TableBucketARN: req.TableBucketARN,
|
||||
Properties: req.Properties,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleGetNamespace gets details of a namespace
|
||||
func (h *S3TablesHandler) handleGetNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req GetNamespaceRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
namespacePath := GetNamespacePath(bucketName, namespaceName)
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
|
||||
// Get namespace and bucket policy
|
||||
var metadata namespaceMetadata
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, fmt.Sprintf("namespace %s not found", flattenNamespace(req.Namespace)))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get namespace: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("GetNamespace", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, "namespace not found")
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
resp := &GetNamespaceResponse{
|
||||
Namespace: metadata.Namespace,
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
OwnerAccountID: metadata.OwnerAccountID,
|
||||
Properties: metadata.Properties,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleUpdateNamespace replaces the stored properties of an existing namespace.
|
||||
// AWS S3 Tables namespaces have no properties; this backs the Iceberg REST
|
||||
// catalog's namespace-property updates, which carry the merged map.
|
||||
func (h *S3TablesHandler) handleUpdateNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req UpdateNamespaceRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
namespacePath := GetNamespacePath(bucketName, namespaceName)
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
|
||||
var metadata namespaceMetadata
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, fmt.Sprintf("namespace %s not found", flattenNamespace(req.Namespace)))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get namespace: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("UpdateNamespace", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to update namespace")
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
metadata.Properties = req.Properties
|
||||
metadataBytes, err := json.Marshal(&metadata)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal namespace metadata")
|
||||
return fmt.Errorf("failed to marshal metadata: %w", err)
|
||||
}
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.setExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata, metadataBytes)
|
||||
})
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to update namespace")
|
||||
return err
|
||||
}
|
||||
|
||||
resp := &UpdateNamespaceResponse{
|
||||
Namespace: metadata.Namespace,
|
||||
TableBucketARN: req.TableBucketARN,
|
||||
Properties: metadata.Properties,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleListNamespaces lists all namespaces in a table bucket
|
||||
func (h *S3TablesHandler) handleListNamespaces(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req ListNamespacesRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
maxNamespaces := req.MaxNamespaces
|
||||
if maxNamespaces <= 0 {
|
||||
maxNamespaces = 100
|
||||
}
|
||||
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
|
||||
// Check permission (check bucket ownership)
|
||||
var bucketMetadata tableBucketMetadata
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list namespaces: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("ListNamespaces", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
var namespaces []NamespaceSummary
|
||||
|
||||
lastFileName := req.ContinuationToken
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
for len(namespaces) < maxNamespaces {
|
||||
resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{
|
||||
Directory: bucketPath,
|
||||
Limit: uint32(maxNamespaces * 2),
|
||||
StartFromFileName: lastFileName,
|
||||
InclusiveStartFrom: lastFileName == "" || lastFileName == req.ContinuationToken,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hasMore := false
|
||||
for {
|
||||
entry, respErr := resp.Recv()
|
||||
if respErr != nil {
|
||||
if respErr == io.EOF {
|
||||
break
|
||||
}
|
||||
return respErr
|
||||
}
|
||||
if entry.Entry == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip the start item if it was included in the previous page
|
||||
if len(namespaces) == 0 && req.ContinuationToken != "" && entry.Entry.Name == req.ContinuationToken {
|
||||
continue
|
||||
}
|
||||
|
||||
hasMore = true
|
||||
lastFileName = entry.Entry.Name
|
||||
|
||||
if !entry.Entry.IsDirectory {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip hidden entries
|
||||
if strings.HasPrefix(entry.Entry.Name, ".") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Apply prefix filter
|
||||
if req.Prefix != "" && !strings.HasPrefix(entry.Entry.Name, req.Prefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Read metadata from extended attribute
|
||||
data, ok := entry.Entry.Extended[ExtendedKeyMetadata]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
var metadata namespaceMetadata
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if metadata.OwnerAccountID != bucketMetadata.OwnerAccountID {
|
||||
continue
|
||||
}
|
||||
|
||||
namespaces = append(namespaces, NamespaceSummary{
|
||||
Namespace: metadata.Namespace,
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
})
|
||||
|
||||
if len(namespaces) >= maxNamespaces {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if !hasMore {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
namespaces = []NamespaceSummary{}
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list namespaces: %v", err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
paginationToken := ""
|
||||
if len(namespaces) >= maxNamespaces {
|
||||
paginationToken = lastFileName
|
||||
}
|
||||
|
||||
resp := &ListNamespacesResponse{
|
||||
Namespaces: namespaces,
|
||||
ContinuationToken: paginationToken,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleDeleteNamespace deletes a namespace from a table bucket
|
||||
func (h *S3TablesHandler) handleDeleteNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req DeleteNamespaceRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
namespacePath := GetNamespacePath(bucketName, namespaceName)
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
|
||||
// Check if namespace exists and get metadata for permission check
|
||||
var metadata namespaceMetadata
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, fmt.Sprintf("namespace %s not found", flattenNamespace(req.Namespace)))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get namespace metadata: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("DeleteNamespace", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, "namespace not found")
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
// Check if namespace is empty
|
||||
hasChildren := false
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{
|
||||
Directory: namespacePath,
|
||||
Limit: 10,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
entry, err := resp.Recv()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
if entry.Entry != nil && !strings.HasPrefix(entry.Entry.Name, ".") {
|
||||
// Only consider it a child if it's a table (has metadata) or a file.
|
||||
// Empty directories left by dropped tables or S3 clients should not
|
||||
// prevent the namespace from being deleted.
|
||||
_, hasMetadata := entry.Entry.Extended[ExtendedKeyMetadata]
|
||||
if hasMetadata || !entry.Entry.IsDirectory {
|
||||
hasChildren = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, fmt.Sprintf("namespace %s not found", flattenNamespace(req.Namespace)))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list namespace entries: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if hasChildren {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeNamespaceNotEmpty, "namespace is not empty")
|
||||
return fmt.Errorf("namespace not empty")
|
||||
}
|
||||
|
||||
// Delete the namespace
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.deleteDirectory(r.Context(), client, namespacePath)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete namespace")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
45
weed/s3api/s3tables/handler_namespace_resolution_test.go
Normal file
45
weed/s3api/s3tables/handler_namespace_resolution_test.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
|
||||
)
|
||||
|
||||
// A directory under a table bucket that carries no namespace metadata is not a
|
||||
// namespace. Reporting that as an internal error, which is what the three
|
||||
// callers of this did while each testing only for a missing entry, turns a
|
||||
// client mistake into a 500 and hides it behind "attribute not found".
|
||||
func TestLoadNamespaceMetadataTreatsMissingAttributeAsAbsent(t *testing.T) {
|
||||
filer := s3tablestest.Start(t)
|
||||
handler := NewS3TablesHandler()
|
||||
client := NewManagerClient(filer.Client)
|
||||
ctx := context.Background()
|
||||
|
||||
filer.Put(TablesPath, "bkt", map[string][]byte{ExtendedKeyTableBucket: []byte("{}")})
|
||||
|
||||
// A stray directory where a namespace would live, with no metadata on it.
|
||||
filer.Put(GetTableBucketPath("bkt"), "stray", nil)
|
||||
|
||||
if _, err := handler.loadNamespaceMetadata(ctx, client, "bkt", "stray"); !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
t.Fatalf("loadNamespaceMetadata on a directory without metadata = %v, want ErrNotFound", err)
|
||||
}
|
||||
|
||||
if _, err := handler.loadNamespaceMetadata(ctx, client, "bkt", "absent"); !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
t.Fatalf("loadNamespaceMetadata on a missing directory = %v, want ErrNotFound", err)
|
||||
}
|
||||
|
||||
filer.Put(GetTableBucketPath("bkt"), "real", map[string][]byte{
|
||||
ExtendedKeyMetadata: []byte(`{"namespace":["real"],"ownerAccountId":"000000000000"}`),
|
||||
})
|
||||
metadata, err := handler.loadNamespaceMetadata(ctx, client, "bkt", "real")
|
||||
if err != nil {
|
||||
t.Fatalf("loadNamespaceMetadata on a real namespace: %v", err)
|
||||
}
|
||||
if metadata.OwnerAccountID != "000000000000" {
|
||||
t.Fatalf("owner = %q, want the stored one", metadata.OwnerAccountID)
|
||||
}
|
||||
}
|
||||
936
weed/s3api/s3tables/handler_policy.go
Normal file
936
weed/s3api/s3tables/handler_policy.go
Normal file
@@ -0,0 +1,936 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// extractResourceOwnerAndBucket extracts ownership info and bucket name from resource metadata.
|
||||
// This helper consolidates the repeated pattern used in handleTagResource, handleListTagsForResource,
|
||||
// and handleUntagResource.
|
||||
func (h *S3TablesHandler) extractResourceOwnerAndBucket(
|
||||
data []byte,
|
||||
resourcePath string,
|
||||
rType ResourceType,
|
||||
) (ownerAccountID, bucketName string, err error) {
|
||||
// Extract bucket name from resource path (format: /buckets/{bucket}/... for both tables and buckets)
|
||||
parts := strings.Split(strings.Trim(resourcePath, "/"), "/")
|
||||
if len(parts) >= 2 {
|
||||
bucketName = parts[1]
|
||||
}
|
||||
|
||||
if rType == ResourceTypeTable {
|
||||
var meta tableMetadataInternal
|
||||
if err := json.Unmarshal(data, &meta); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
ownerAccountID = meta.OwnerAccountID
|
||||
} else {
|
||||
var meta tableBucketMetadata
|
||||
if err := json.Unmarshal(data, &meta); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
ownerAccountID = meta.OwnerAccountID
|
||||
}
|
||||
return ownerAccountID, bucketName, nil
|
||||
}
|
||||
|
||||
// handlePutTableBucketPolicy puts a policy on a table bucket
|
||||
func (h *S3TablesHandler) handlePutTableBucketPolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req PutTableBucketPolicyRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
if req.ResourcePolicy == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourcePolicy is required")
|
||||
return fmt.Errorf("resourcePolicy is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if bucket exists and get metadata for ownership check
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
var bucketMetadata tableBucketMetadata
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table bucket: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("PutTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, "", bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to put table bucket policy")
|
||||
return NewAuthError("PutTableBucketPolicy", principal, "not authorized to put table bucket policy")
|
||||
}
|
||||
|
||||
// Write policy
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy, []byte(req.ResourcePolicy))
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to put table bucket policy")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleGetTableBucketPolicy gets the policy of a table bucket
|
||||
func (h *S3TablesHandler) handleGetTableBucketPolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req GetTableBucketPolicyRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
var policy []byte
|
||||
var bucketMetadata tableBucketMetadata
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Get metadata for ownership check
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
|
||||
// Get policy
|
||||
policy, err = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
return err
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
return err
|
||||
}
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchPolicy, "table bucket policy not found")
|
||||
return err
|
||||
}
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get table bucket policy: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("GetTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, string(policy), bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table bucket policy")
|
||||
return NewAuthError("GetTableBucketPolicy", principal, "not authorized to get table bucket policy")
|
||||
}
|
||||
|
||||
resp := &GetTableBucketPolicyResponse{
|
||||
ResourcePolicy: string(policy),
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleDeleteTableBucketPolicy deletes the policy of a table bucket
|
||||
func (h *S3TablesHandler) handleDeleteTableBucketPolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req DeleteTableBucketPolicyRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
|
||||
// Check if bucket exists and get metadata for ownership check
|
||||
var bucketMetadata tableBucketMetadata
|
||||
var bucketPolicy string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
// Policy not found is not an error; bucketPolicy remains empty
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table bucket: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("DeleteTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table bucket policy")
|
||||
return NewAuthError("DeleteTableBucketPolicy", principal, "not authorized to delete table bucket policy")
|
||||
}
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.deleteExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
})
|
||||
|
||||
if err != nil && !errors.Is(err, ErrAttributeNotFound) {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table bucket policy")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handlePutTablePolicy puts a policy on a table
|
||||
func (h *S3TablesHandler) handlePutTablePolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req PutTablePolicyRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" || len(req.Namespace) == 0 || req.Name == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN, namespace, and name are required")
|
||||
return fmt.Errorf("missing required parameters")
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.ResourcePolicy == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourcePolicy is required")
|
||||
return fmt.Errorf("resourcePolicy is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if table exists
|
||||
tableName, err := validateTableName(req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
tablePath := GetTablePath(bucketName, namespaceName, tableName)
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
|
||||
var metadata tableMetadataInternal
|
||||
var bucketPolicy string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal table metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
// Policy not found is not an error; bucketPolicy remains empty
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("PutTablePolicy", principal, metadata.OwnerAccountID, bucketPolicy, tableARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableName: tableName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to put table policy")
|
||||
return NewAuthError("PutTablePolicy", principal, "not authorized to put table policy")
|
||||
}
|
||||
|
||||
// Write policy
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.setExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyPolicy, []byte(req.ResourcePolicy))
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to put table policy")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleGetTablePolicy gets the policy of a table
|
||||
func (h *S3TablesHandler) handleGetTablePolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req GetTablePolicyRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" || len(req.Namespace) == 0 || req.Name == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN, namespace, and name are required")
|
||||
return fmt.Errorf("missing required parameters")
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
tableName, err := validateTableName(req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
tablePath := GetTablePath(bucketName, namespaceName, tableName)
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
var policy []byte
|
||||
var metadata tableMetadataInternal
|
||||
var bucketPolicy string
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Get metadata for ownership check
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal table metadata: %w", err)
|
||||
}
|
||||
|
||||
// Get policy
|
||||
policy, err = h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
// Policy not found is not an error; bucketPolicy remains empty
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
return err
|
||||
}
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchPolicy, "table policy not found")
|
||||
return err
|
||||
}
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get table policy: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("GetTablePolicy", principal, metadata.OwnerAccountID, bucketPolicy, tableARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableName: tableName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table policy")
|
||||
return NewAuthError("GetTablePolicy", principal, "not authorized to get table policy")
|
||||
}
|
||||
|
||||
resp := &GetTablePolicyResponse{
|
||||
ResourcePolicy: string(policy),
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleDeleteTablePolicy deletes the policy of a table
|
||||
func (h *S3TablesHandler) handleDeleteTablePolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req DeleteTablePolicyRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" || len(req.Namespace) == 0 || req.Name == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN, namespace, and name are required")
|
||||
return fmt.Errorf("missing required parameters")
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
tableName, err := validateTableName(req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
tablePath := GetTablePath(bucketName, namespaceName, tableName)
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
|
||||
// Check if table exists
|
||||
var metadata tableMetadataInternal
|
||||
var bucketPolicy string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal table metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
// Policy not found is not an error; bucketPolicy remains empty
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("DeleteTablePolicy", principal, metadata.OwnerAccountID, bucketPolicy, tableARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableName: tableName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table policy")
|
||||
return NewAuthError("DeleteTablePolicy", principal, "not authorized to delete table policy")
|
||||
}
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.deleteExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyPolicy)
|
||||
})
|
||||
|
||||
if err != nil && !errors.Is(err, ErrAttributeNotFound) {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table policy")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleTagResource adds tags to a resource
|
||||
func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req TagResourceRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.ResourceARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourceArn is required")
|
||||
return fmt.Errorf("resourceArn is required")
|
||||
}
|
||||
|
||||
if len(req.Tags) == 0 {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tags are required")
|
||||
return fmt.Errorf("tags are required")
|
||||
}
|
||||
|
||||
// Parse resource ARN to determine if it's a bucket or table
|
||||
resourcePath, extendedKey, rType, err := h.resolveResourcePath(req.ResourceARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Read existing tags and merge, AND check permissions based on metadata ownership
|
||||
existingTags := make(map[string]string)
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
requestTagKeys := mapKeys(req.Tags)
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Read metadata for ownership check
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fetch bucket policy if we have a bucket name
|
||||
if bucketName != "" {
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
// Policy not found is not an error; bucketPolicy remains empty
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Read existing tags
|
||||
data, err = h.getExtendedAttribute(r.Context(), client, resourcePath, extendedKey)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return err
|
||||
}
|
||||
} else if err := json.Unmarshal(data, &existingTags); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resourceARN := req.ResourceARN
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("TagResource", principal, ownerAccountID, bucketPolicy, resourceARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
TableBucketTags: bucketTags,
|
||||
RequestTags: req.Tags,
|
||||
TagKeys: requestTagKeys,
|
||||
ResourceTags: existingTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
return NewAuthError("TagResource", principal, "not authorized to tag resource")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
errorCode := ErrCodeNoSuchBucket
|
||||
if rType == ResourceTypeTable {
|
||||
errorCode = ErrCodeNoSuchTable
|
||||
}
|
||||
h.writeError(w, http.StatusNotFound, errorCode, "resource not found")
|
||||
} else if isAuthError(err) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, err.Error())
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to read existing tags: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Merge new tags
|
||||
for k, v := range req.Tags {
|
||||
existingTags[k] = v
|
||||
}
|
||||
|
||||
// Write merged tags
|
||||
tagsBytes, err := json.Marshal(existingTags)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal tags")
|
||||
return fmt.Errorf("failed to marshal tags: %w", err)
|
||||
}
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.setExtendedAttribute(r.Context(), client, resourcePath, extendedKey, tagsBytes)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to tag resource")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleListTagsForResource lists tags for a resource
|
||||
func (h *S3TablesHandler) handleListTagsForResource(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req ListTagsForResourceRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.ResourceARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourceArn is required")
|
||||
return fmt.Errorf("resourceArn is required")
|
||||
}
|
||||
|
||||
resourcePath, extendedKey, rType, err := h.resolveResourcePath(req.ResourceARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
tags := make(map[string]string)
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Read metadata for ownership check
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fetch bucket policy if we have a bucket name
|
||||
if bucketName != "" {
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
// Policy not found is not an error; bucketPolicy remains empty
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
data, err = h.getExtendedAttribute(r.Context(), client, resourcePath, extendedKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
return nil // No tags is not an error.
|
||||
}
|
||||
return err // Propagate other errors.
|
||||
}
|
||||
if err := json.Unmarshal(data, &tags); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resourceARN := req.ResourceARN
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("ListTagsForResource", principal, ownerAccountID, bucketPolicy, resourceARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
TableBucketTags: bucketTags,
|
||||
ResourceTags: tags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
return NewAuthError("ListTagsForResource", principal, "not authorized to list tags for resource")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
errorCode := ErrCodeNoSuchBucket
|
||||
if rType == ResourceTypeTable {
|
||||
errorCode = ErrCodeNoSuchTable
|
||||
}
|
||||
h.writeError(w, http.StatusNotFound, errorCode, "resource not found")
|
||||
} else if isAuthError(err) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, err.Error())
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list tags: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
resp := &ListTagsForResourceResponse{
|
||||
Tags: tags,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleUntagResource removes tags from a resource
|
||||
func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req UntagResourceRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.ResourceARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourceArn is required")
|
||||
return fmt.Errorf("resourceArn is required")
|
||||
}
|
||||
|
||||
if len(req.TagKeys) == 0 {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tagKeys are required")
|
||||
return fmt.Errorf("tagKeys are required")
|
||||
}
|
||||
|
||||
resourcePath, extendedKey, rType, err := h.resolveResourcePath(req.ResourceARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Read existing tags, check permission
|
||||
tags := make(map[string]string)
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Read metadata for ownership check
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fetch bucket policy if we have a bucket name
|
||||
if bucketName != "" {
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
// Policy not found is not an error; bucketPolicy remains empty
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
data, err = h.getExtendedAttribute(r.Context(), client, resourcePath, extendedKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &tags); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resourceARN := req.ResourceARN
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("UntagResource", principal, ownerAccountID, bucketPolicy, resourceARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
TableBucketTags: bucketTags,
|
||||
TagKeys: req.TagKeys,
|
||||
ResourceTags: tags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}) {
|
||||
return NewAuthError("UntagResource", principal, "not authorized to untag resource")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
errorCode := ErrCodeNoSuchBucket
|
||||
if rType == ResourceTypeTable {
|
||||
errorCode = ErrCodeNoSuchTable
|
||||
}
|
||||
h.writeError(w, http.StatusNotFound, errorCode, "resource not found")
|
||||
} else if isAuthError(err) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, err.Error())
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to read existing tags")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove specified tags
|
||||
for _, key := range req.TagKeys {
|
||||
delete(tags, key)
|
||||
}
|
||||
|
||||
// Write updated tags
|
||||
tagsBytes, err := json.Marshal(tags)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal tags")
|
||||
return fmt.Errorf("failed to marshal tags: %w", err)
|
||||
}
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.setExtendedAttribute(r.Context(), client, resourcePath, extendedKey, tagsBytes)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to untag resource")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveResourcePath determines the resource path and extended attribute key from a resource ARN
|
||||
func (h *S3TablesHandler) resolveResourcePath(resourceARN string) (path string, key string, rType ResourceType, err error) {
|
||||
// Try parsing as table ARN first
|
||||
bucketName, namespace, tableName, err := parseTableFromARN(resourceARN)
|
||||
if err == nil {
|
||||
return GetTablePath(bucketName, namespace, tableName), ExtendedKeyTags, ResourceTypeTable, nil
|
||||
}
|
||||
|
||||
// Try parsing as bucket ARN
|
||||
bucketName, err = parseBucketNameFromARN(resourceARN)
|
||||
if err == nil {
|
||||
return GetTableBucketPath(bucketName), ExtendedKeyTags, ResourceTypeBucket, nil
|
||||
}
|
||||
|
||||
return "", "", "", fmt.Errorf("invalid resource ARN: %s", resourceARN)
|
||||
}
|
||||
289
weed/s3api/s3tables/handler_rename_test.go
Normal file
289
weed/s3api/s3tables/handler_rename_test.go
Normal file
@@ -0,0 +1,289 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const renameTestBucket = "renamebkt"
|
||||
|
||||
func mustBucketARN(t *testing.T) string {
|
||||
t.Helper()
|
||||
arn, err := BuildBucketARN(DefaultRegion, DefaultAccountID, renameTestBucket)
|
||||
require.NoError(t, err)
|
||||
return arn
|
||||
}
|
||||
|
||||
// startRenameManager seeds a bucket/namespace/table and returns a trusted Manager.
|
||||
func startRenameManager(t *testing.T) (*s3tablestest.MemFiler, *Manager) {
|
||||
t.Helper()
|
||||
fs := s3tablestest.Start(t)
|
||||
|
||||
bucketMeta, _ := json.Marshal(tableBucketMetadata{Name: renameTestBucket, OwnerAccountID: DefaultAccountID})
|
||||
fs.Put(TablesPath, renameTestBucket, map[string][]byte{
|
||||
ExtendedKeyTableBucket: []byte("{}"),
|
||||
ExtendedKeyMetadata: bucketMeta,
|
||||
})
|
||||
|
||||
nsMeta, _ := json.Marshal(namespaceMetadata{Namespace: []string{"ns"}, OwnerAccountID: DefaultAccountID})
|
||||
fs.Put(GetTableBucketPath(renameTestBucket), "ns", map[string][]byte{ExtendedKeyMetadata: nsMeta})
|
||||
|
||||
tableMeta, _ := json.Marshal(tableMetadataInternal{
|
||||
Name: "t",
|
||||
Namespace: "ns",
|
||||
Format: "ICEBERG",
|
||||
OwnerAccountID: DefaultAccountID,
|
||||
MetadataVersion: 3,
|
||||
MetadataLocation: "s3://" + renameTestBucket + "/ns/t/metadata/v3.metadata.json",
|
||||
})
|
||||
fs.Put(GetNamespacePath(renameTestBucket, "ns"), "t", map[string][]byte{
|
||||
ExtendedKeyMetadata: tableMeta,
|
||||
ExtendedKeyMetadataVersion: []byte("3"),
|
||||
})
|
||||
|
||||
// Physical metadata.json and data files live under the table directory.
|
||||
tablePath := GetTablePath(renameTestBucket, "ns", "t")
|
||||
fs.Put(tablePath, "metadata", nil)
|
||||
fs.Put(tablePath, "data", nil)
|
||||
fs.Put(path.Join(tablePath, "metadata"), "v3.metadata.json", nil)
|
||||
|
||||
m := NewManager()
|
||||
m.SetTrusted(true)
|
||||
return fs, m
|
||||
}
|
||||
|
||||
func runRename(t *testing.T, m *Manager, fs *s3tablestest.MemFiler, req *RenameTableRequest) error {
|
||||
t.Helper()
|
||||
return m.Execute(context.Background(), NewManagerClient(fs.Client), "RenameTable", req, nil, "")
|
||||
}
|
||||
|
||||
func runGetTable(t *testing.T, m *Manager, fs *s3tablestest.MemFiler, namespace, name string) (*GetTableResponse, error) {
|
||||
t.Helper()
|
||||
resp := &GetTableResponse{}
|
||||
err := m.Execute(context.Background(), NewManagerClient(fs.Client), "GetTable", &GetTableRequest{
|
||||
TableBucketARN: mustBucketARN(t),
|
||||
Namespace: []string{namespace},
|
||||
Name: name,
|
||||
}, resp, "")
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func TestRenameTablePreservesData(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
|
||||
req := &RenameTableRequest{
|
||||
TableBucketARN: mustBucketARN(t),
|
||||
SourceNamespace: []string{"ns"},
|
||||
SourceName: "t",
|
||||
DestNamespace: []string{"ns"},
|
||||
DestName: "t2",
|
||||
}
|
||||
require.NoError(t, runRename(t, m, fs, req))
|
||||
|
||||
// The source directory and its metadata.json/data children must survive: rename
|
||||
// is catalog-only and the destination still points at the original location.
|
||||
srcPath := GetTablePath(renameTestBucket, "ns", "t")
|
||||
assert.NotNil(t, fs.Get(srcPath, "metadata"), "source metadata dir must survive")
|
||||
assert.NotNil(t, fs.Get(srcPath, "data"), "source data dir must survive")
|
||||
assert.NotNil(t, fs.Get(path.Join(srcPath, "metadata"), "v3.metadata.json"), "metadata.json must survive")
|
||||
|
||||
// Source catalog xattrs are dropped so the name stops resolving.
|
||||
src := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
|
||||
require.NotNil(t, src, "source directory must remain to hold the data children")
|
||||
_, hasMeta := src.Extended[ExtendedKeyMetadata]
|
||||
assert.False(t, hasMeta, "source table-metadata xattr must be removed")
|
||||
|
||||
_, err := runGetTable(t, m, fs, "ns", "t")
|
||||
require.Error(t, err)
|
||||
var s3Err *S3TablesError
|
||||
require.ErrorAs(t, err, &s3Err)
|
||||
assert.Equal(t, ErrCodeNoSuchTable, s3Err.Type)
|
||||
|
||||
// The destination resolves to the preserved (original) MetadataLocation.
|
||||
got, err := runGetTable(t, m, fs, "ns", "t2")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "t2", got.Name)
|
||||
assert.Equal(t, "s3://"+renameTestBucket+"/ns/t/metadata/v3.metadata.json", got.MetadataLocation)
|
||||
|
||||
dest := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t2")
|
||||
require.NotNil(t, dest)
|
||||
assert.Equal(t, []byte("3"), dest.Extended[ExtendedKeyMetadataVersion])
|
||||
}
|
||||
|
||||
func TestRenameTableSourceMissing(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
err := runRename(t, m, fs, &RenameTableRequest{
|
||||
TableBucketARN: mustBucketARN(t),
|
||||
SourceNamespace: []string{"ns"},
|
||||
SourceName: "ghost",
|
||||
DestNamespace: []string{"ns"},
|
||||
DestName: "t2",
|
||||
})
|
||||
require.Error(t, err)
|
||||
var s3Err *S3TablesError
|
||||
require.ErrorAs(t, err, &s3Err)
|
||||
assert.Equal(t, ErrCodeNoSuchTable, s3Err.Type)
|
||||
}
|
||||
|
||||
func TestRenameTableDestExists(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
existing, _ := json.Marshal(tableMetadataInternal{Name: "t2", Namespace: "ns", OwnerAccountID: DefaultAccountID})
|
||||
fs.Put(GetNamespacePath(renameTestBucket, "ns"), "t2", map[string][]byte{ExtendedKeyMetadata: existing})
|
||||
|
||||
err := runRename(t, m, fs, &RenameTableRequest{
|
||||
TableBucketARN: mustBucketARN(t),
|
||||
SourceNamespace: []string{"ns"},
|
||||
SourceName: "t",
|
||||
DestNamespace: []string{"ns"},
|
||||
DestName: "t2",
|
||||
})
|
||||
require.Error(t, err)
|
||||
var s3Err *S3TablesError
|
||||
require.ErrorAs(t, err, &s3Err)
|
||||
assert.Equal(t, ErrCodeTableAlreadyExists, s3Err.Type)
|
||||
assert.NotNil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t"), "source must be untouched on conflict")
|
||||
}
|
||||
|
||||
func TestRenameTableDestNamespaceMissing(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
err := runRename(t, m, fs, &RenameTableRequest{
|
||||
TableBucketARN: mustBucketARN(t),
|
||||
SourceNamespace: []string{"ns"},
|
||||
SourceName: "t",
|
||||
DestNamespace: []string{"other"},
|
||||
DestName: "t2",
|
||||
})
|
||||
require.Error(t, err)
|
||||
var s3Err *S3TablesError
|
||||
require.ErrorAs(t, err, &s3Err)
|
||||
assert.Equal(t, ErrCodeNoSuchNamespace, s3Err.Type)
|
||||
assert.NotNil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t"), "source must be untouched")
|
||||
}
|
||||
|
||||
// A principal allowed to rename the source must still be denied when it cannot
|
||||
// create a table in the destination namespace.
|
||||
func TestRenameTableDestNamespaceUnauthorized(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
m.SetTrusted(false)
|
||||
m.SetDefaultAllow(false)
|
||||
|
||||
// "mover" may rename the source table but holds no rights on "dest".
|
||||
srcPolicy, _ := json.Marshal(map[string]interface{}{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": []map[string]interface{}{{
|
||||
"Effect": "Allow",
|
||||
"Principal": "mover",
|
||||
"Action": "s3tables:RenameTable",
|
||||
"Resource": "*",
|
||||
}},
|
||||
})
|
||||
srcEntry := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
|
||||
require.NotNil(t, srcEntry)
|
||||
srcEntry.Extended[ExtendedKeyPolicy] = srcPolicy
|
||||
|
||||
destNsMeta, _ := json.Marshal(namespaceMetadata{Namespace: []string{"dest"}, OwnerAccountID: DefaultAccountID})
|
||||
fs.Put(GetTableBucketPath(renameTestBucket), "dest", map[string][]byte{ExtendedKeyMetadata: destNsMeta})
|
||||
|
||||
mover := &testIdentity{Name: "mover", Account: &testIdentityAccount{Id: "mover"}}
|
||||
ctx := s3_constants.SetIdentityInContext(context.Background(), mover)
|
||||
err := m.Execute(ctx, NewManagerClient(fs.Client), "RenameTable", &RenameTableRequest{
|
||||
TableBucketARN: mustBucketARN(t),
|
||||
SourceNamespace: []string{"ns"},
|
||||
SourceName: "t",
|
||||
DestNamespace: []string{"dest"},
|
||||
DestName: "t2",
|
||||
}, nil, "mover")
|
||||
require.Error(t, err)
|
||||
var s3Err *S3TablesError
|
||||
require.ErrorAs(t, err, &s3Err)
|
||||
assert.Equal(t, ErrCodeAccessDenied, s3Err.Type)
|
||||
|
||||
assert.NotNil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t"), "source must be untouched")
|
||||
assert.Nil(t, fs.Get(GetNamespacePath(renameTestBucket, "dest"), "t2"), "destination must not be written")
|
||||
}
|
||||
|
||||
func TestRenameTableInvalidName(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
err := runRename(t, m, fs, &RenameTableRequest{
|
||||
TableBucketARN: mustBucketARN(t),
|
||||
SourceNamespace: []string{"ns"},
|
||||
SourceName: "t",
|
||||
DestNamespace: []string{"ns"},
|
||||
DestName: "Bad/Name",
|
||||
})
|
||||
require.Error(t, err)
|
||||
var s3Err *S3TablesError
|
||||
require.ErrorAs(t, err, &s3Err)
|
||||
assert.Equal(t, ErrCodeInvalidRequest, s3Err.Type)
|
||||
}
|
||||
|
||||
// A disabled maintenance configuration has to move with the table. Leaving it
|
||||
// behind re-enables maintenance under the new name, and leaks the old settings
|
||||
// onto whatever is created at the old one.
|
||||
func TestRenameTableCarriesMaintenanceConfiguration(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
|
||||
src := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
|
||||
require.NotNil(t, src)
|
||||
config := []byte(`{"icebergSnapshotManagement":{"status":"disabled"}}`)
|
||||
status := []byte(`{"icebergCompaction":{"status":"Successful"}}`)
|
||||
src.Extended[ExtendedKeyMaintenance] = config
|
||||
src.Extended[ExtendedKeyMaintenanceStatus] = status
|
||||
|
||||
require.NoError(t, runRename(t, m, fs, &RenameTableRequest{
|
||||
TableBucketARN: mustBucketARN(t),
|
||||
SourceNamespace: []string{"ns"},
|
||||
SourceName: "t",
|
||||
DestNamespace: []string{"ns"},
|
||||
DestName: "t2",
|
||||
}))
|
||||
|
||||
dest := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t2")
|
||||
require.NotNil(t, dest)
|
||||
assert.Equal(t, config, dest.Extended[ExtendedKeyMaintenance], "the disable must move with the table")
|
||||
assert.Equal(t, status, dest.Extended[ExtendedKeyMaintenanceStatus])
|
||||
|
||||
// And must not linger on the old name, where a reused name would inherit it.
|
||||
moved := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
|
||||
require.NotNil(t, moved)
|
||||
_, hasConfig := moved.Extended[ExtendedKeyMaintenance]
|
||||
assert.False(t, hasConfig, "source maintenance configuration must be cleared")
|
||||
_, hasStatus := moved.Extended[ExtendedKeyMaintenanceStatus]
|
||||
assert.False(t, hasStatus, "source maintenance status must be cleared")
|
||||
}
|
||||
|
||||
// A maintenance configuration written after the rename copied the source must
|
||||
// not be deleted by the source cleanup: it would vanish without ever reaching
|
||||
// the destination.
|
||||
func TestRenameTableRejectsConcurrentMaintenanceWrite(t *testing.T) {
|
||||
fs, _ := startRenameManager(t)
|
||||
|
||||
src := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
|
||||
require.NotNil(t, src)
|
||||
copied := []byte(`{"icebergCompaction":{"status":"enabled"}}`)
|
||||
src.Extended[ExtendedKeyMaintenance] = copied
|
||||
|
||||
// Stand in for the Put that lands between the copy and the cleanup.
|
||||
landedLate := []byte(`{"icebergSnapshotManagement":{"status":"disabled"}}`)
|
||||
expected := map[string][]byte{ExtendedKeyMaintenance: copied}
|
||||
|
||||
src.Extended[ExtendedKeyMaintenance] = landedLate
|
||||
|
||||
h := NewS3TablesHandler()
|
||||
err := NewManagerClient(fs.Client).WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.removeExtendedAttributesIf(context.Background(),
|
||||
client, GetTablePath(renameTestBucket, "ns", "t"), expected, renamedTableAttributes...)
|
||||
})
|
||||
|
||||
require.Error(t, err, "cleanup must refuse to delete a configuration written after the copy")
|
||||
assert.ErrorIs(t, err, ErrConcurrentUpdate)
|
||||
assert.Equal(t, landedLate, src.Extended[ExtendedKeyMaintenance], "the late write must survive")
|
||||
}
|
||||
1754
weed/s3api/s3tables/handler_table.go
Normal file
1754
weed/s3api/s3tables/handler_table.go
Normal file
File diff suppressed because it is too large
Load Diff
25
weed/s3api/s3tables/handler_table_test.go
Normal file
25
weed/s3api/s3tables/handler_table_test.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package s3tables
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMetadataVersionFromLocation(t *testing.T) {
|
||||
cases := []struct {
|
||||
location string
|
||||
want int
|
||||
}{
|
||||
{"s3://bucket/ns/tbl/metadata/v1.metadata.json", 1},
|
||||
{"s3://bucket/ns/tbl/metadata/v7.metadata.json", 7},
|
||||
{"v42.metadata.json", 42},
|
||||
{"s3://bucket/ns/tbl/metadata/00003-9f1c2b3a-4d5e-6f70-8192-a3b4c5d6e7f8.metadata.json", 3},
|
||||
{"00012-abcdef.metadata.json", 12},
|
||||
{"s3://bucket/ns/tbl/metadata/00000-abc.metadata.json", 1},
|
||||
{"s3://bucket/ns/tbl/metadata/v0.metadata.json", 1},
|
||||
{"s3://bucket/ns/tbl/metadata/garbage.metadata.json", 1},
|
||||
{"", 1},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := metadataVersionFromLocation(c.location); got != c.want {
|
||||
t.Errorf("metadataVersionFromLocation(%q) = %d, want %d", c.location, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
98
weed/s3api/s3tables/handler_update_cas_test.go
Normal file
98
weed/s3api/s3tables/handler_update_cas_test.go
Normal file
@@ -0,0 +1,98 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func updateTableRequest(t *testing.T, metadataLocation string, version int) *UpdateTableRequest {
|
||||
t.Helper()
|
||||
return &UpdateTableRequest{
|
||||
TableBucketARN: mustBucketARN(t),
|
||||
Namespace: []string{"ns"},
|
||||
Name: "t",
|
||||
MetadataVersion: version,
|
||||
MetadataLocation: metadataLocation,
|
||||
}
|
||||
}
|
||||
|
||||
// A second commit that reads the same base must not overwrite the first one's
|
||||
// metadata pointer: the snapshot the first commit published would be lost.
|
||||
func TestUpdateTableRejectsLostUpdate(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
|
||||
winnerLocation := "s3://" + renameTestBucket + "/ns/t/metadata/v4-winner.metadata.json"
|
||||
fs.BeforeUpdate = func() {
|
||||
winner, err := json.Marshal(tableMetadataInternal{
|
||||
Name: "t",
|
||||
Namespace: "ns",
|
||||
Format: "ICEBERG",
|
||||
OwnerAccountID: DefaultAccountID,
|
||||
MetadataVersion: 4,
|
||||
MetadataLocation: winnerLocation,
|
||||
VersionToken: generateVersionToken(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
entry := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
|
||||
require.NotNil(t, entry)
|
||||
entry.Extended[ExtendedKeyMetadata] = winner
|
||||
}
|
||||
|
||||
loserLocation := "s3://" + renameTestBucket + "/ns/t/metadata/v4-loser.metadata.json"
|
||||
err := m.Execute(context.Background(), NewManagerClient(fs.Client), "UpdateTable",
|
||||
updateTableRequest(t, loserLocation, 4), nil, "")
|
||||
|
||||
require.Error(t, err)
|
||||
var s3Err *S3TablesError
|
||||
require.ErrorAs(t, err, &s3Err)
|
||||
assert.Equal(t, ErrCodeConflict, s3Err.Type)
|
||||
|
||||
got, err := runGetTable(t, m, fs, "ns", "t")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, winnerLocation, got.MetadataLocation, "the first commit must survive")
|
||||
}
|
||||
|
||||
// A policy written between the authorization check and the write decides
|
||||
// whether this caller may still commit, so the commit is rejected rather than
|
||||
// applied on a decision that has been overtaken - and the policy itself must
|
||||
// survive, not be reverted by the commit's stale copy of the entry.
|
||||
func TestUpdateTableRejectsWhenTheAuthorizingPolicyChanges(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
|
||||
policy := []byte(`{"Version":"2012-10-17","Statement":[]}`)
|
||||
fs.BeforeUpdate = func() {
|
||||
entry := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
|
||||
require.NotNil(t, entry)
|
||||
entry.Extended[ExtendedKeyPolicy] = policy
|
||||
}
|
||||
|
||||
location := "s3://" + renameTestBucket + "/ns/t/metadata/v4.metadata.json"
|
||||
err := m.Execute(context.Background(), NewManagerClient(fs.Client), "UpdateTable",
|
||||
updateTableRequest(t, location, 4), nil, "")
|
||||
|
||||
require.Error(t, err)
|
||||
var s3Err *S3TablesError
|
||||
require.ErrorAs(t, err, &s3Err)
|
||||
assert.Equal(t, ErrCodeConflict, s3Err.Type)
|
||||
|
||||
entry := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "t")
|
||||
require.NotNil(t, entry)
|
||||
assert.Equal(t, policy, entry.Extended[ExtendedKeyPolicy], "the concurrent policy write must survive")
|
||||
}
|
||||
|
||||
func TestUpdateTableAppliesWithoutContention(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
|
||||
location := "s3://" + renameTestBucket + "/ns/t/metadata/v4.metadata.json"
|
||||
require.NoError(t, m.Execute(context.Background(), NewManagerClient(fs.Client), "UpdateTable",
|
||||
updateTableRequest(t, location, 4), nil, ""))
|
||||
|
||||
got, err := runGetTable(t, m, fs, "ns", "t")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, location, got.MetadataLocation)
|
||||
assert.Equal(t, 4, got.MetadataVersion)
|
||||
}
|
||||
659
weed/s3api/s3tables/handler_view.go
Normal file
659
weed/s3api/s3tables/handler_view.go
Normal file
@@ -0,0 +1,659 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// handleCreateView creates a new view in a namespace. Views share the table
|
||||
// storage layout but are tagged ExtendedKeyEntryType="view". Names are unique
|
||||
// across tables and views in a namespace.
|
||||
func (h *S3TablesHandler) handleCreateView(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req CreateViewRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.Name == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "name is required")
|
||||
return fmt.Errorf("name is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
viewName, err := validateTableName(req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if namespace exists
|
||||
namespacePath := GetNamespacePath(bucketName, namespaceName)
|
||||
namespaceMetadata, err := h.loadNamespaceMetadata(r.Context(), filerClient, bucketName, namespaceName)
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, fmt.Sprintf("namespace %s not found", namespaceName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check namespace: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
accountID := h.getAccountID(r)
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
namespacePolicy, bucketPolicy, bucketTags, bucketMetadata, err := h.loadAuthContext(r, filerClient, namespacePath, bucketPath)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to fetch policies: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
// A view is Iceberg metadata, so it belongs only in a bucket that holds
|
||||
// Iceberg tables.
|
||||
if bucketMetadata.Format != "" && bucketMetadata.Format != FormatIceberg {
|
||||
message := fmt.Sprintf("table bucket %s holds %s tables and cannot hold views", bucketName, bucketMetadata.Format)
|
||||
h.writeError(w, http.StatusConflict, ErrCodeConflict, message)
|
||||
return fmt.Errorf("%s", message)
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
if !h.authorizeViewOp(r, "CreateView", accountID, namespaceMetadata.OwnerAccountID, bucketMetadata.OwnerAccountID, namespacePolicy, bucketPolicy, bucketARN, bucketName, namespaceName, viewName, bucketTags) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create view in this namespace")
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
viewPath := GetTablePath(bucketName, namespaceName, viewName)
|
||||
|
||||
// Reject if a table or view already exists at this name.
|
||||
var existingMetadata tableMetadataInternal
|
||||
var existingIsTable bool
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
entry, err := h.lookupEntry(r.Context(), client, viewPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if EntryType(entry.Extended) != EntryTypeView {
|
||||
existingIsTable = true
|
||||
return nil
|
||||
}
|
||||
data, ok := entry.Extended[ExtendedKeyMetadata]
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: %s", ErrAttributeNotFound, ExtendedKeyMetadata)
|
||||
}
|
||||
return json.Unmarshal(data, &existingMetadata)
|
||||
})
|
||||
if err == nil {
|
||||
if existingIsTable {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeViewAlreadyExists, fmt.Sprintf("a table named %s already exists", viewName))
|
||||
return fmt.Errorf("table name conflict: %s", viewName)
|
||||
}
|
||||
viewARN := h.generateViewARN(existingMetadata.OwnerAccountID, bucketName, namespaceName+"/"+viewName)
|
||||
h.writeJSON(w, http.StatusOK, &CreateViewResponse{
|
||||
ViewARN: viewARN,
|
||||
VersionToken: existingMetadata.VersionToken,
|
||||
MetadataLocation: existingMetadata.MetadataLocation,
|
||||
})
|
||||
return nil
|
||||
} else if !errors.Is(err, filer_pb.ErrNotFound) && !errors.Is(err, ErrAttributeNotFound) {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check view: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ValidateMetadataLocation(req.MetadataLocation, bucketName); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
versionToken := generateVersionToken()
|
||||
metadata := &tableMetadataInternal{
|
||||
Name: viewName,
|
||||
Namespace: namespaceName,
|
||||
Format: FormatIceberg,
|
||||
CreatedAt: now,
|
||||
ModifiedAt: now,
|
||||
OwnerAccountID: namespaceMetadata.OwnerAccountID,
|
||||
VersionToken: versionToken,
|
||||
MetadataVersion: max(req.MetadataVersion, 1),
|
||||
MetadataLocation: req.MetadataLocation,
|
||||
Metadata: req.Metadata,
|
||||
}
|
||||
|
||||
metadataBytes, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal view metadata")
|
||||
return err
|
||||
}
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
if err := h.ensureDirectory(r.Context(), client, viewPath); err != nil {
|
||||
return err
|
||||
}
|
||||
return h.setExtendedAttributes(r.Context(), client, viewPath, map[string][]byte{
|
||||
ExtendedKeyMetadata: metadataBytes,
|
||||
ExtendedKeyEntryType: []byte(EntryTypeView),
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to create view")
|
||||
return err
|
||||
}
|
||||
|
||||
viewARN := h.generateViewARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+viewName)
|
||||
h.writeJSON(w, http.StatusOK, &CreateViewResponse{
|
||||
ViewARN: viewARN,
|
||||
VersionToken: versionToken,
|
||||
MetadataLocation: metadata.MetadataLocation,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleGetView returns the metadata pointer for a view.
|
||||
func (h *S3TablesHandler) handleGetView(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req GetViewRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, namespace, viewName, err := h.parseViewRef(req.TableBucketARN, req.Namespace, req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
viewPath := GetTablePath(bucketName, namespace, viewName)
|
||||
var metadata tableMetadataInternal
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
entry, err := h.lookupEntry(r.Context(), client, viewPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if EntryType(entry.Extended) != EntryTypeView {
|
||||
return filer_pb.ErrNotFound
|
||||
}
|
||||
data, ok := entry.Extended[ExtendedKeyMetadata]
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: %s", ErrAttributeNotFound, ExtendedKeyMetadata)
|
||||
}
|
||||
return json.Unmarshal(data, &metadata)
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchView, fmt.Sprintf("view %s not found", viewName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get view: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
accountID := h.getAccountID(r)
|
||||
viewPolicy, bucketPolicy, bucketTags, bucketMetadata, err := h.loadAuthContext(r, filerClient, viewPath, GetTableBucketPath(bucketName))
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to fetch policies: %v", err))
|
||||
return err
|
||||
}
|
||||
viewARN := h.generateViewARN(metadata.OwnerAccountID, bucketName, namespace+"/"+viewName)
|
||||
if !h.authorizeViewOp(r, "GetView", accountID, metadata.OwnerAccountID, bucketMetadata.OwnerAccountID, viewPolicy, bucketPolicy, viewARN, bucketName, namespace, viewName, bucketTags) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchView, fmt.Sprintf("view %s not found", viewName))
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, &GetViewResponse{
|
||||
Name: metadata.Name,
|
||||
ViewARN: viewARN,
|
||||
Namespace: expandNamespace(metadata.Namespace),
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
ModifiedAt: metadata.ModifiedAt,
|
||||
OwnerAccountID: metadata.OwnerAccountID,
|
||||
MetadataLocation: metadata.MetadataLocation,
|
||||
MetadataVersion: metadata.MetadataVersion,
|
||||
VersionToken: metadata.VersionToken,
|
||||
Metadata: metadata.Metadata,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleListViews lists views in a namespace, excluding table entries.
|
||||
func (h *S3TablesHandler) handleListViews(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req ListViewsRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
maxViews := req.MaxViews
|
||||
if maxViews <= 0 {
|
||||
maxViews = 100
|
||||
}
|
||||
const maxViewsLimit = 1000
|
||||
if maxViews > maxViewsLimit {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "MaxViews exceeds maximum allowed value")
|
||||
return fmt.Errorf("invalid maxViews value: %d", maxViews)
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
var views []ViewSummary
|
||||
var paginationToken string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
namespacePath := GetNamespacePath(bucketName, namespaceName)
|
||||
bucketPath := GetTableBucketPath(bucketName)
|
||||
var nsMeta namespaceMetadata
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &nsMeta); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
namespacePolicy, bucketPolicy, bucketTags, bucketMeta, err := h.loadAuthContext(r, &singleClient{client}, namespacePath, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bucketARN := h.generateTableBucketARN(bucketMeta.OwnerAccountID, bucketName)
|
||||
if !h.authorizeViewOp(r, "ListViews", h.getAccountID(r), nsMeta.OwnerAccountID, bucketMeta.OwnerAccountID, namespacePolicy, bucketPolicy, bucketARN, bucketName, namespaceName, "", bucketTags) {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
views, paginationToken, err = h.listViewsInNamespace(r, client, bucketName, namespaceName, req.Prefix, req.ContinuationToken, maxViews)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
views = []ViewSummary{}
|
||||
paginationToken = ""
|
||||
} else if isAuthError(err) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "Access Denied")
|
||||
return err
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list views: %v", err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, &ListViewsResponse{
|
||||
Views: views,
|
||||
ContinuationToken: paginationToken,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *S3TablesHandler) listViewsInNamespace(r *http.Request, client filer_pb.SeaweedFilerClient, bucketName, namespaceName, prefix, continuationToken string, maxViews int) ([]ViewSummary, string, error) {
|
||||
namespacePath := GetNamespacePath(bucketName, namespaceName)
|
||||
var views []ViewSummary
|
||||
lastFileName := continuationToken
|
||||
ctx := r.Context()
|
||||
|
||||
for len(views) < maxViews {
|
||||
resp, err := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
|
||||
Directory: namespacePath,
|
||||
Limit: uint32(maxViews * 2),
|
||||
StartFromFileName: lastFileName,
|
||||
InclusiveStartFrom: lastFileName == "" || lastFileName == continuationToken,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
hasMore := false
|
||||
for {
|
||||
entry, respErr := resp.Recv()
|
||||
if respErr != nil {
|
||||
if respErr == io.EOF {
|
||||
break
|
||||
}
|
||||
return nil, "", respErr
|
||||
}
|
||||
if entry.Entry == nil {
|
||||
continue
|
||||
}
|
||||
if len(views) == 0 && continuationToken != "" && entry.Entry.Name == continuationToken {
|
||||
continue
|
||||
}
|
||||
|
||||
hasMore = true
|
||||
lastFileName = entry.Entry.Name
|
||||
|
||||
if !entry.Entry.IsDirectory || strings.HasPrefix(entry.Entry.Name, ".") {
|
||||
continue
|
||||
}
|
||||
if prefix != "" && !strings.HasPrefix(entry.Entry.Name, prefix) {
|
||||
continue
|
||||
}
|
||||
// Only include view entries; skip tables and untagged entries.
|
||||
if EntryType(entry.Entry.Extended) != EntryTypeView {
|
||||
continue
|
||||
}
|
||||
data, ok := entry.Entry.Extended[ExtendedKeyMetadata]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var metadata tableMetadataInternal
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
views = append(views, ViewSummary{
|
||||
Name: entry.Entry.Name,
|
||||
ViewARN: h.generateViewARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+entry.Entry.Name),
|
||||
Namespace: expandNamespace(namespaceName),
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
ModifiedAt: metadata.ModifiedAt,
|
||||
})
|
||||
|
||||
if len(views) >= maxViews {
|
||||
return views, lastFileName, nil
|
||||
}
|
||||
}
|
||||
|
||||
if !hasMore {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(views) < maxViews {
|
||||
lastFileName = ""
|
||||
}
|
||||
return views, lastFileName, nil
|
||||
}
|
||||
|
||||
// handleUpdateView replaces the metadata pointer for a view, mirroring UpdateTable.
|
||||
func (h *S3TablesHandler) handleUpdateView(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req UpdateViewRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, namespaceName, viewName, err := h.parseViewRef(req.TableBucketARN, req.Namespace, req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if err := ValidateMetadataLocation(req.MetadataLocation, bucketName); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
viewPath := GetTablePath(bucketName, namespaceName, viewName)
|
||||
var metadata tableMetadataInternal
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
entry, err := h.lookupEntry(r.Context(), client, viewPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if EntryType(entry.Extended) != EntryTypeView {
|
||||
return filer_pb.ErrNotFound
|
||||
}
|
||||
data, ok := entry.Extended[ExtendedKeyMetadata]
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: %s", ErrAttributeNotFound, ExtendedKeyMetadata)
|
||||
}
|
||||
return json.Unmarshal(data, &metadata)
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchView, "view not found")
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, err.Error())
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
accountID := h.getAccountID(r)
|
||||
viewPolicy, bucketPolicy, bucketTags, bucketMetadata, err := h.loadAuthContext(r, filerClient, viewPath, GetTableBucketPath(bucketName))
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, err.Error())
|
||||
return err
|
||||
}
|
||||
viewARN := h.generateViewARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+viewName)
|
||||
if !h.authorizeViewOp(r, "UpdateView", accountID, metadata.OwnerAccountID, bucketMetadata.OwnerAccountID, viewPolicy, bucketPolicy, viewARN, bucketName, namespaceName, viewName, bucketTags) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to update view")
|
||||
return NewAuthError("UpdateView", accountID, "not authorized to update view")
|
||||
}
|
||||
|
||||
if req.VersionToken != "" && req.VersionToken != metadata.VersionToken {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeConflict, "Version token mismatch")
|
||||
return ErrVersionTokenMismatch
|
||||
}
|
||||
|
||||
if req.Metadata != nil {
|
||||
if metadata.Metadata == nil {
|
||||
metadata.Metadata = &TableMetadata{}
|
||||
}
|
||||
if req.Metadata.Iceberg != nil {
|
||||
if metadata.Metadata.Iceberg == nil {
|
||||
metadata.Metadata.Iceberg = &IcebergMetadata{}
|
||||
}
|
||||
if req.Metadata.Iceberg.TableUUID != "" {
|
||||
metadata.Metadata.Iceberg.TableUUID = req.Metadata.Iceberg.TableUUID
|
||||
}
|
||||
}
|
||||
if len(req.Metadata.FullMetadata) > 0 {
|
||||
metadata.Metadata.FullMetadata = req.Metadata.FullMetadata
|
||||
}
|
||||
}
|
||||
if req.MetadataLocation != "" {
|
||||
metadata.MetadataLocation = req.MetadataLocation
|
||||
}
|
||||
if req.MetadataVersion > 0 {
|
||||
metadata.MetadataVersion = req.MetadataVersion
|
||||
} else if metadata.MetadataVersion == 0 {
|
||||
metadata.MetadataVersion = 1
|
||||
}
|
||||
metadata.ModifiedAt = time.Now()
|
||||
metadata.VersionToken = generateVersionToken()
|
||||
|
||||
metadataBytes, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal metadata")
|
||||
return err
|
||||
}
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.setExtendedAttribute(r.Context(), client, viewPath, ExtendedKeyMetadata, metadataBytes)
|
||||
})
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to update metadata")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, &UpdateViewResponse{
|
||||
ViewARN: viewARN,
|
||||
MetadataLocation: metadata.MetadataLocation,
|
||||
VersionToken: metadata.VersionToken,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleDeleteView deletes a view.
|
||||
func (h *S3TablesHandler) handleDeleteView(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req DeleteViewRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, namespaceName, viewName, err := h.parseViewRef(req.TableBucketARN, req.Namespace, req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
viewPath := GetTablePath(bucketName, namespaceName, viewName)
|
||||
var metadata tableMetadataInternal
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
entry, err := h.lookupEntry(r.Context(), client, viewPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if EntryType(entry.Extended) != EntryTypeView {
|
||||
return filer_pb.ErrNotFound
|
||||
}
|
||||
data, ok := entry.Extended[ExtendedKeyMetadata]
|
||||
if !ok {
|
||||
return fmt.Errorf("%w: %s", ErrAttributeNotFound, ExtendedKeyMetadata)
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return err
|
||||
}
|
||||
if req.VersionToken != "" && metadata.VersionToken != req.VersionToken {
|
||||
return ErrVersionTokenMismatch
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchView, fmt.Sprintf("view %s not found", viewName))
|
||||
} else if errors.Is(err, ErrVersionTokenMismatch) {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeConflict, "version token mismatch")
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check view: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
accountID := h.getAccountID(r)
|
||||
viewPolicy, bucketPolicy, bucketTags, bucketMetadata, err := h.loadAuthContext(r, filerClient, viewPath, GetTableBucketPath(bucketName))
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, err.Error())
|
||||
return err
|
||||
}
|
||||
viewARN := h.generateViewARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+viewName)
|
||||
if !h.authorizeViewOp(r, "DeleteView", accountID, metadata.OwnerAccountID, bucketMetadata.OwnerAccountID, viewPolicy, bucketPolicy, viewARN, bucketName, namespaceName, viewName, bucketTags) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete view")
|
||||
return NewAuthError("DeleteView", accountID, "not authorized to delete view")
|
||||
}
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.deleteDirectory(r.Context(), client, viewPath)
|
||||
})
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete view")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseViewRef validates and resolves a (bucketARN, namespace, name) triple.
|
||||
func (h *S3TablesHandler) parseViewRef(bucketARN string, namespace []string, name string) (bucketName, namespaceName, viewName string, err error) {
|
||||
if bucketARN == "" || len(namespace) == 0 || name == "" {
|
||||
return "", "", "", fmt.Errorf("tableBucketARN, namespace, and name are required")
|
||||
}
|
||||
if bucketName, err = parseBucketNameFromARN(bucketARN); err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
if namespaceName, err = validateNamespace(namespace); err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
if viewName, err = validateTableName(name); err != nil {
|
||||
return "", "", "", err
|
||||
}
|
||||
return bucketName, namespaceName, viewName, nil
|
||||
}
|
||||
|
||||
// loadAuthContext fetches the resource policy, bucket policy, bucket tags and
|
||||
// bucket metadata used to authorize a view operation.
|
||||
func (h *S3TablesHandler) loadAuthContext(r *http.Request, filerClient FilerClient, resourcePath, bucketPath string) (resourcePolicy, bucketPolicy string, bucketTags map[string]string, bucketMetadata tableBucketMetadata, err error) {
|
||||
bucketTags = map[string]string{}
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, e := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if e == nil {
|
||||
if e := json.Unmarshal(data, &bucketMetadata); e != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", e)
|
||||
}
|
||||
} else if !errors.Is(e, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket metadata: %w", e)
|
||||
}
|
||||
|
||||
policyData, e := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyPolicy)
|
||||
if e == nil {
|
||||
resourcePolicy = string(policyData)
|
||||
} else if !errors.Is(e, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch resource policy: %w", e)
|
||||
}
|
||||
|
||||
policyData, e = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if e == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(e, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %w", e)
|
||||
}
|
||||
if tags, e := h.readTags(r.Context(), client, bucketPath); e != nil {
|
||||
return e
|
||||
} else if tags != nil {
|
||||
bucketTags = tags
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return resourcePolicy, bucketPolicy, bucketTags, bucketMetadata, err
|
||||
}
|
||||
|
||||
// authorizeViewOp evaluates the resource and bucket policies for a view operation.
|
||||
func (h *S3TablesHandler) authorizeViewOp(r *http.Request, operation, principal, resourceOwner, bucketOwner, resourcePolicy, bucketPolicy, resourceARN, bucketName, namespace, viewName string, bucketTags map[string]string) bool {
|
||||
identityActions := getIdentityActions(r)
|
||||
ctx := &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespace,
|
||||
TableName: viewName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllowFor(r),
|
||||
}
|
||||
resourceAllowed := CheckPermissionWithContext(operation, principal, resourceOwner, resourcePolicy, resourceARN, ctx)
|
||||
bucketARN := h.generateTableBucketARN(bucketOwner, bucketName)
|
||||
bucketAllowed := CheckPermissionWithContext(operation, principal, bucketOwner, bucketPolicy, bucketARN, ctx)
|
||||
return resourceAllowed || bucketAllowed
|
||||
}
|
||||
|
||||
// singleClient adapts a bound filer client to the FilerClient interface so the
|
||||
// shared helpers can be reused inside an existing WithFilerClient closure.
|
||||
type singleClient struct {
|
||||
client filer_pb.SeaweedFilerClient
|
||||
}
|
||||
|
||||
func (s *singleClient) WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
|
||||
return fn(s.client)
|
||||
}
|
||||
136
weed/s3api/s3tables/handler_view_rename_test.go
Normal file
136
weed/s3api/s3tables/handler_view_rename_test.go
Normal file
@@ -0,0 +1,136 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// seedView adds a view alongside the table the rename harness already creates.
|
||||
func seedView(t *testing.T, fs *s3tablestest.MemFiler, name string) {
|
||||
t.Helper()
|
||||
|
||||
viewMeta, err := json.Marshal(tableMetadataInternal{
|
||||
Name: name,
|
||||
Namespace: "ns",
|
||||
Format: "ICEBERG",
|
||||
OwnerAccountID: DefaultAccountID,
|
||||
MetadataVersion: 1,
|
||||
MetadataLocation: "s3://" + renameTestBucket + "/ns/" + name + "/metadata/v1.metadata.json",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
fs.Put(GetNamespacePath(renameTestBucket, "ns"), name, map[string][]byte{
|
||||
ExtendedKeyMetadata: viewMeta,
|
||||
ExtendedKeyMetadataVersion: []byte("1"),
|
||||
ExtendedKeyEntryType: []byte(EntryTypeView),
|
||||
})
|
||||
viewPath := GetTablePath(renameTestBucket, "ns", name)
|
||||
fs.Put(viewPath, "metadata", nil)
|
||||
fs.Put(path.Join(viewPath, "metadata"), "v1.metadata.json", nil)
|
||||
}
|
||||
|
||||
func runRenameView(t *testing.T, m *Manager, fs *s3tablestest.MemFiler, sourceName, destName string) error {
|
||||
t.Helper()
|
||||
return m.Execute(context.Background(), NewManagerClient(fs.Client), "RenameView", &RenameTableRequest{
|
||||
TableBucketARN: mustBucketARN(t),
|
||||
SourceNamespace: []string{"ns"},
|
||||
SourceName: sourceName,
|
||||
DestNamespace: []string{"ns"},
|
||||
DestName: destName,
|
||||
}, nil, "")
|
||||
}
|
||||
|
||||
func TestRenameViewMovesCatalogPointer(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
seedView(t, fs, "v")
|
||||
|
||||
require.NoError(t, runRenameView(t, m, fs, "v", "v2"))
|
||||
|
||||
dest := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "v2")
|
||||
require.NotNil(t, dest)
|
||||
assert.Equal(t, EntryTypeView, EntryType(dest.Extended), "destination must stay a view")
|
||||
|
||||
var moved tableMetadataInternal
|
||||
require.NoError(t, json.Unmarshal(dest.Extended[ExtendedKeyMetadata], &moved))
|
||||
assert.Equal(t, "v2", moved.Name)
|
||||
assert.Equal(t, "s3://"+renameTestBucket+"/ns/v/metadata/v1.metadata.json", moved.MetadataLocation,
|
||||
"rename is catalog-only, the metadata stays where it was written")
|
||||
|
||||
src := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "v")
|
||||
require.NotNil(t, src)
|
||||
_, stillListed := src.Extended[ExtendedKeyMetadata]
|
||||
assert.False(t, stillListed, "source name must stop resolving")
|
||||
assert.NotNil(t, fs.Get(path.Join(GetTablePath(renameTestBucket, "ns", "v"), "metadata"), "v1.metadata.json"),
|
||||
"the view's metadata file must survive")
|
||||
}
|
||||
|
||||
// Tables and views share one namespace directory, so each rename must refuse
|
||||
// the other kind rather than moving it.
|
||||
func TestRenameViewRejectsTable(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
|
||||
err := runRenameView(t, m, fs, "t", "t2")
|
||||
require.Error(t, err)
|
||||
var s3Err *S3TablesError
|
||||
require.ErrorAs(t, err, &s3Err)
|
||||
assert.Equal(t, ErrCodeNoSuchView, s3Err.Type)
|
||||
}
|
||||
|
||||
func TestRenameTableRejectsView(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
seedView(t, fs, "v")
|
||||
|
||||
err := runRename(t, m, fs, &RenameTableRequest{
|
||||
TableBucketARN: mustBucketARN(t),
|
||||
SourceNamespace: []string{"ns"},
|
||||
SourceName: "v",
|
||||
DestNamespace: []string{"ns"},
|
||||
DestName: "v2",
|
||||
})
|
||||
require.Error(t, err)
|
||||
var s3Err *S3TablesError
|
||||
require.ErrorAs(t, err, &s3Err)
|
||||
assert.Equal(t, ErrCodeNoSuchTable, s3Err.Type)
|
||||
}
|
||||
|
||||
// A policy scoped to the view ARN must authorize renaming that view. Checking
|
||||
// the source against a table ARN would silently ignore it.
|
||||
func TestRenameViewAuthorizesAgainstTheViewARN(t *testing.T) {
|
||||
fs, m := startRenameManager(t)
|
||||
m.SetTrusted(false)
|
||||
m.SetDefaultAllow(false)
|
||||
seedView(t, fs, "v")
|
||||
|
||||
const principal = "analyst"
|
||||
viewARN := "arn:aws:s3tables:" + DefaultRegion + ":" + DefaultAccountID + ":bucket/" + renameTestBucket + "/view/ns/v"
|
||||
viewPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"` + principal +
|
||||
`","Action":"s3tables:RenameView","Resource":"` + viewARN + `"}]}`
|
||||
view := fs.Get(GetNamespacePath(renameTestBucket, "ns"), "v")
|
||||
require.NotNil(t, view)
|
||||
view.Extended[ExtendedKeyPolicy] = []byte(viewPolicy)
|
||||
|
||||
// Landing in the namespace needs create permission there as well.
|
||||
namespacePolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"` + principal +
|
||||
`","Action":"s3tables:CreateView","Resource":"` + mustBucketARN(t) + `"}]}`
|
||||
namespace := fs.Get(GetTableBucketPath(renameTestBucket), "ns")
|
||||
require.NotNil(t, namespace)
|
||||
namespace.Extended[ExtendedKeyPolicy] = []byte(namespacePolicy)
|
||||
|
||||
err := m.Execute(context.Background(), NewManagerClient(fs.Client), "RenameView", &RenameTableRequest{
|
||||
TableBucketARN: mustBucketARN(t),
|
||||
SourceNamespace: []string{"ns"},
|
||||
SourceName: "v",
|
||||
DestNamespace: []string{"ns"},
|
||||
DestName: "v2",
|
||||
}, nil, principal)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotNil(t, fs.Get(GetNamespacePath(renameTestBucket, "ns"), "v2"))
|
||||
}
|
||||
33
weed/s3api/s3tables/handler_view_test.go
Normal file
33
weed/s3api/s3tables/handler_view_test.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package s3tables
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestEntryType(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
extended map[string][]byte
|
||||
want string
|
||||
}{
|
||||
{"nil extended is table", nil, EntryTypeTable},
|
||||
{"absent marker is table", map[string][]byte{ExtendedKeyMetadata: []byte("{}")}, EntryTypeTable},
|
||||
{"empty marker is table", map[string][]byte{ExtendedKeyEntryType: {}}, EntryTypeTable},
|
||||
{"table marker", map[string][]byte{ExtendedKeyEntryType: []byte(EntryTypeTable)}, EntryTypeTable},
|
||||
{"view marker", map[string][]byte{ExtendedKeyEntryType: []byte(EntryTypeView)}, EntryTypeView},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := EntryType(c.extended); got != c.want {
|
||||
t.Fatalf("EntryType() = %q, want %q", got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateViewARN(t *testing.T) {
|
||||
h := NewS3TablesHandler()
|
||||
got := h.generateViewARN(DefaultAccountID, "warehouse", "ns/v")
|
||||
want := "arn:aws:s3tables:us-east-1:000000000000:bucket/warehouse/view/ns/v"
|
||||
if got != want {
|
||||
t.Fatalf("generateViewARN() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
284
weed/s3api/s3tables/iam.go
Normal file
284
weed/s3api/s3tables/iam.go
Normal file
@@ -0,0 +1,284 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/iam/integration"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
// IAMAuthorizer allows s3tables handlers to evaluate IAM policies without importing s3api.
|
||||
type IAMAuthorizer interface {
|
||||
IsActionAllowed(ctx context.Context, request *integration.ActionRequest) (bool, error)
|
||||
}
|
||||
|
||||
// SetIAMAuthorizer injects the IAM authorizer for policy-based access checks.
|
||||
func (h *S3TablesHandler) SetIAMAuthorizer(authorizer IAMAuthorizer) {
|
||||
h.iamAuthorizer = authorizer
|
||||
}
|
||||
|
||||
func (h *S3TablesHandler) shouldUseIAM(r *http.Request, identityActions, identityPolicyNames []string) bool {
|
||||
if h.iamAuthorizer == nil || r == nil {
|
||||
return false
|
||||
}
|
||||
if s3_constants.GetIdentityFromContext(r) == nil {
|
||||
return false
|
||||
}
|
||||
// When default-allow is enabled, keep anonymous requests on the legacy path
|
||||
// to preserve zero-config behavior (IAM policies are not available for anonymous).
|
||||
if h.defaultAllow && isAnonymousIdentity(r) {
|
||||
return false
|
||||
}
|
||||
// An empty inline `identityActions` slice doesn't mean the identity has no
|
||||
// permissions—it just means authorization lives in IAM policies or session
|
||||
// tokens instead of static action lists. We therefore prefer the IAM path
|
||||
// whenever inline actions are absent and fall back to default policy names
|
||||
// or session tokens.
|
||||
if hasSessionToken(r) {
|
||||
return true
|
||||
}
|
||||
if len(identityActions) == 0 {
|
||||
return true
|
||||
}
|
||||
return len(identityPolicyNames) > 0
|
||||
}
|
||||
|
||||
// defaultAllowFor reports whether the open-by-default fallback applies: only for
|
||||
// trusted tooling or unauthenticated/anonymous access. An authenticated principal
|
||||
// must pass an explicit check.
|
||||
func (h *S3TablesHandler) defaultAllowFor(r *http.Request) bool {
|
||||
if h.trusted {
|
||||
return true
|
||||
}
|
||||
if !h.defaultAllow {
|
||||
return false
|
||||
}
|
||||
// The Manager path forwards only the identity name, so a name alone (no
|
||||
// identity object) still counts as an authenticated principal.
|
||||
if s3_constants.GetIdentityFromContext(r) == nil && s3_constants.GetIdentityNameFromContext(r) == "" {
|
||||
return true
|
||||
}
|
||||
return isAnonymousIdentity(r)
|
||||
}
|
||||
|
||||
func isAnonymousIdentity(r *http.Request) bool {
|
||||
val, ok := getIdentityStructValue(r)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if nameField := val.FieldByName("Name"); nameField.IsValid() && nameField.Kind() == reflect.String {
|
||||
if nameField.String() == s3_constants.AccountAnonymousId {
|
||||
return true
|
||||
}
|
||||
}
|
||||
accountField := val.FieldByName("Account")
|
||||
if accountField.IsValid() && !accountField.IsNil() {
|
||||
if accountField.Kind() == reflect.Ptr {
|
||||
accountField = accountField.Elem()
|
||||
}
|
||||
if accountField.Kind() == reflect.Struct {
|
||||
if idField := accountField.FieldByName("Id"); idField.IsValid() && idField.Kind() == reflect.String {
|
||||
if idField.String() == s3_constants.AccountAnonymousId {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasSessionToken(r *http.Request) bool {
|
||||
return extractSessionToken(r) != ""
|
||||
}
|
||||
|
||||
func extractSessionToken(r *http.Request) string {
|
||||
if token := r.Header.Get(s3_constants.SeaweedFSSessionTokenHeader); token != "" {
|
||||
return token
|
||||
}
|
||||
if token := r.Header.Get("X-Amz-Security-Token"); token != "" {
|
||||
return token
|
||||
}
|
||||
return r.URL.Query().Get("X-Amz-Security-Token")
|
||||
}
|
||||
|
||||
func (h *S3TablesHandler) authorizeIAMAction(r *http.Request, identityPolicyNames []string, action string, resources ...string) (bool, error) {
|
||||
if h.iamAuthorizer == nil {
|
||||
err := fmt.Errorf("nil iamAuthorizer in authorizeIAMAction")
|
||||
glog.V(2).Infof("S3Tables: %v", err)
|
||||
return false, err
|
||||
}
|
||||
principal := r.Header.Get(s3_constants.SeaweedFSPrincipalHeader)
|
||||
if principal == "" {
|
||||
principal = getIdentityPrincipalArn(r)
|
||||
}
|
||||
if principal == "" {
|
||||
return false, fmt.Errorf("missing principal for IAM authorization")
|
||||
}
|
||||
|
||||
if !strings.Contains(action, ":") {
|
||||
action = "s3tables:" + action
|
||||
}
|
||||
|
||||
sessionToken := extractSessionToken(r)
|
||||
|
||||
requestContext := buildIAMRequestContext(r, getIdentityClaims(r))
|
||||
policyNames := identityPolicyNames
|
||||
if len(policyNames) == 0 {
|
||||
policyNames = getIdentityPolicyNames(r)
|
||||
}
|
||||
|
||||
if len(resources) == 0 {
|
||||
return false, fmt.Errorf("no resources provided to authorizeIAMAction")
|
||||
}
|
||||
checkedResource := false
|
||||
for _, resource := range resources {
|
||||
if resource == "" {
|
||||
continue
|
||||
}
|
||||
checkedResource = true
|
||||
allowed, err := h.iamAuthorizer.IsActionAllowed(r.Context(), &integration.ActionRequest{
|
||||
Principal: principal,
|
||||
Action: action,
|
||||
Resource: resource,
|
||||
SessionToken: sessionToken,
|
||||
RequestContext: requestContext,
|
||||
PolicyNames: policyNames,
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(2).Infof("S3Tables: IAM authorization error action=%s resource=%s principal=%s: %v", action, resource, principal, err)
|
||||
return false, err
|
||||
}
|
||||
if !allowed {
|
||||
err := fmt.Errorf("access denied by IAM for resource %s", resource)
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
if !checkedResource {
|
||||
return false, fmt.Errorf("no non-empty resources provided to authorizeIAMAction")
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func getIdentityPrincipalArn(r *http.Request) string {
|
||||
val, ok := getIdentityStructValue(r)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
field := val.FieldByName("PrincipalArn")
|
||||
if field.IsValid() && field.Kind() == reflect.String {
|
||||
return field.String()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func getIdentityPolicyNames(r *http.Request) []string {
|
||||
val, ok := getIdentityStructValue(r)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
field := val.FieldByName("PolicyNames")
|
||||
if !field.IsValid() || field.Kind() != reflect.Slice {
|
||||
return nil
|
||||
}
|
||||
policies := make([]string, 0, field.Len())
|
||||
for i := 0; i < field.Len(); i++ {
|
||||
item := field.Index(i)
|
||||
if item.Kind() == reflect.String {
|
||||
policies = append(policies, item.String())
|
||||
} else if item.CanInterface() {
|
||||
policies = append(policies, fmt.Sprint(item.Interface()))
|
||||
}
|
||||
}
|
||||
if len(policies) == 0 {
|
||||
return nil
|
||||
}
|
||||
return policies
|
||||
}
|
||||
|
||||
func getIdentityClaims(r *http.Request) map[string]interface{} {
|
||||
val, ok := getIdentityStructValue(r)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
field := val.FieldByName("Claims")
|
||||
if !field.IsValid() || field.Kind() != reflect.Map || field.IsNil() {
|
||||
return nil
|
||||
}
|
||||
if field.Type().Key().Kind() != reflect.String {
|
||||
return nil
|
||||
}
|
||||
claims := make(map[string]interface{}, field.Len())
|
||||
for _, key := range field.MapKeys() {
|
||||
if key.Kind() != reflect.String {
|
||||
continue
|
||||
}
|
||||
val := field.MapIndex(key)
|
||||
if !val.IsValid() {
|
||||
continue
|
||||
}
|
||||
claims[key.String()] = val.Interface()
|
||||
}
|
||||
if len(claims) == 0 {
|
||||
return nil
|
||||
}
|
||||
return claims
|
||||
}
|
||||
|
||||
func buildIAMRequestContext(r *http.Request, claims map[string]interface{}) map[string]interface{} {
|
||||
ctx := make(map[string]interface{})
|
||||
if ua := r.Header.Get("User-Agent"); ua != "" {
|
||||
ctx["userAgent"] = ua
|
||||
}
|
||||
if referer := r.Header.Get("Referer"); referer != "" {
|
||||
ctx["referer"] = referer
|
||||
}
|
||||
for k, v := range claims {
|
||||
if strings.HasPrefix(k, "jwt:") {
|
||||
if _, exists := ctx[k]; !exists {
|
||||
ctx[k] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
for k, v := range claims {
|
||||
if strings.HasPrefix(k, "jwt:") {
|
||||
continue
|
||||
}
|
||||
if _, exists := ctx[k]; !exists {
|
||||
ctx[k] = v
|
||||
}
|
||||
jwtKey := "jwt:" + k
|
||||
if _, exists := ctx[jwtKey]; !exists {
|
||||
ctx[jwtKey] = v
|
||||
}
|
||||
}
|
||||
if len(ctx) == 0 {
|
||||
return nil
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
|
||||
// getIdentityStructValue fetches the identity struct held in the request context.
|
||||
// The identity is expected to be a pointer to a struct with the fields used by
|
||||
// the reflection helpers (PrincipalArn string, PolicyNames []string,
|
||||
// Claims map[string]interface{}).
|
||||
// This helper centralizes the nil-check and ptr-deref logic so callers focus on
|
||||
// reading the specific fields they need.
|
||||
func getIdentityStructValue(r *http.Request) (reflect.Value, bool) {
|
||||
identityRaw := s3_constants.GetIdentityFromContext(r)
|
||||
if identityRaw == nil {
|
||||
return reflect.Value{}, false
|
||||
}
|
||||
val := reflect.ValueOf(identityRaw)
|
||||
if val.Kind() == reflect.Ptr {
|
||||
val = val.Elem()
|
||||
}
|
||||
if val.Kind() != reflect.Struct {
|
||||
return reflect.Value{}, false
|
||||
}
|
||||
return val, true
|
||||
}
|
||||
365
weed/s3api/s3tables/iceberg_layout.go
Normal file
365
weed/s3api/s3tables/iceberg_layout.go
Normal file
@@ -0,0 +1,365 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
pathpkg "path"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Iceberg file layout validation
|
||||
// Apache Iceberg tables follow a specific file layout structure:
|
||||
// - metadata/ directory containing metadata files (*.json, *.avro)
|
||||
// - data/ directory containing data files (*.parquet, *.orc, *.avro)
|
||||
//
|
||||
// Valid file patterns include:
|
||||
// - metadata/v*.metadata.json (table metadata)
|
||||
// - metadata/snap-*.avro (snapshot manifest lists)
|
||||
// - metadata/*.avro (manifest files)
|
||||
// - data/*.parquet, data/*.orc, data/*.avro (data files)
|
||||
|
||||
const uuidPattern = `[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}`
|
||||
|
||||
var (
|
||||
// Allowed directories in a table. The set is the union of what every
|
||||
// supported format writes, because this runs on the S3 door where the
|
||||
// table's format is not in hand: metadata/ and data/ for Iceberg,
|
||||
// _versions/ and _indices/ for Lance, which also puts its fragments in
|
||||
// data/.
|
||||
tableAllowedDirs = map[string]bool{
|
||||
"metadata": true,
|
||||
"data": true,
|
||||
}
|
||||
|
||||
// A format's own bookkeeping lives in underscore-prefixed directories that
|
||||
// the catalog does not interpret: Lance writes _versions, _transactions,
|
||||
// _indices and _deletions, and enumerating them here would mean guessing at
|
||||
// the next one. Iceberg writes none, so admitting them costs it nothing.
|
||||
formatOwnedDirPattern = regexp.MustCompile(`^_[a-z][a-z0-9_]*$`)
|
||||
|
||||
// Marker files a Lance table keeps at its root to record that it is
|
||||
// reserved but not yet written, or deregistered but still on storage.
|
||||
lanceMarkerFiles = map[string]bool{
|
||||
".lance-reserved": true,
|
||||
".lance-deregistered": true,
|
||||
}
|
||||
|
||||
// Patterns for valid metadata files.
|
||||
//
|
||||
// Note: Iceberg engines (Flink/Spark/Trino, plus different Iceberg versions)
|
||||
// use a variety of manifest/snapshot naming schemes that the strict patterns
|
||||
// below don't cover - e.g. Flink emits manifests like
|
||||
// "{flink-job-id}-{checkpoint}-{operator-id}-{n}.avro". The Iceberg spec
|
||||
// itself doesn't mandate a specific filename; engines just need a stable
|
||||
// unique name. So in addition to the strict patterns we keep below for
|
||||
// documentation, the catch-all entries at the bottom accept any
|
||||
// *.avro and *.metadata.json filename composed of safe characters
|
||||
// ([A-Za-z0-9._-]). The catch-all deliberately does NOT cover arbitrary
|
||||
// *.json — only the *.metadata.json suffix — to avoid letting random
|
||||
// JSON drop into the metadata/ directory.
|
||||
metadataFilePatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`^v\d+\.metadata\.json$`), // Table metadata: v1.metadata.json, v2.metadata.json
|
||||
regexp.MustCompile(`^snap-\d+-\d+-` + uuidPattern + `\.avro$`), // Snapshot manifests: snap-123-1-uuid.avro
|
||||
regexp.MustCompile(`^` + uuidPattern + `-m\d+\.avro$`), // Manifest files: uuid-m0.avro
|
||||
regexp.MustCompile(`^` + uuidPattern + `\.avro$`), // General manifest files
|
||||
regexp.MustCompile(`^version-hint\.text$`), // Version hint file
|
||||
regexp.MustCompile(`^` + uuidPattern + `\.metadata\.json$`), // UUID-named metadata
|
||||
regexp.MustCompile(`^[^/]+\.stats$`), // Trino/Iceberg stats files
|
||||
// Catch-all for Iceberg writer-generated manifest / metadata files
|
||||
// whose naming we can't anticipate across engines and versions.
|
||||
regexp.MustCompile(`^[A-Za-z0-9._-]+\.avro$`),
|
||||
regexp.MustCompile(`^[A-Za-z0-9._-]+\.metadata\.json$`),
|
||||
}
|
||||
|
||||
// Patterns for valid data files
|
||||
dataFilePatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`^[^/]+\.parquet$`), // Parquet files
|
||||
regexp.MustCompile(`^[^/]+\.orc$`), // ORC files
|
||||
regexp.MustCompile(`^[^/]+\.avro$`), // Avro files
|
||||
regexp.MustCompile(`^[^/]+\.lance$`), // Lance fragments
|
||||
}
|
||||
|
||||
// Data file partition path pattern (e.g., year=2024/month=01/)
|
||||
partitionPathPattern = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*=[^/]+$`)
|
||||
|
||||
// Pattern for valid subdirectory names (alphanumeric, underscore, hyphen, and UUID-style directories)
|
||||
validSubdirectoryPattern = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`)
|
||||
)
|
||||
|
||||
// IcebergLayoutValidator validates that files conform to Iceberg table layout
|
||||
type IcebergLayoutValidator struct{}
|
||||
|
||||
// NewIcebergLayoutValidator creates a new Iceberg layout validator
|
||||
func NewIcebergLayoutValidator() *IcebergLayoutValidator {
|
||||
return &IcebergLayoutValidator{}
|
||||
}
|
||||
|
||||
// ValidateFilePath validates that a file path conforms to Iceberg layout
|
||||
// The path should be relative to the table root (e.g., "metadata/v1.metadata.json" or "data/file.parquet")
|
||||
func (v *IcebergLayoutValidator) ValidateFilePath(relativePath string) error {
|
||||
// Normalize path separators
|
||||
relativePath = strings.TrimPrefix(relativePath, "/")
|
||||
if relativePath == "" {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "empty file path",
|
||||
}
|
||||
}
|
||||
|
||||
parts := strings.SplitN(relativePath, "/", 2)
|
||||
|
||||
topDir := parts[0]
|
||||
|
||||
// A Lance table's marker files sit at its root rather than in a directory.
|
||||
if len(parts) == 1 && lanceMarkerFiles[topDir] {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if top-level directory is allowed
|
||||
if !tableAllowedDirs[topDir] && !formatOwnedDirPattern.MatchString(topDir) {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "files must be placed in 'metadata/', 'data/' or a format-owned '_' directory",
|
||||
}
|
||||
}
|
||||
|
||||
// If it's just a bare top-level key (no trailing slash and no subpath), reject it
|
||||
if len(parts) == 1 {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "must be a directory (use trailing slash) or contain a subpath",
|
||||
}
|
||||
}
|
||||
|
||||
remainingPath := parts[1]
|
||||
if remainingPath == "" {
|
||||
return nil // allow paths like "data/" or "metadata/"
|
||||
}
|
||||
|
||||
switch topDir {
|
||||
case "metadata":
|
||||
return v.validateMetadataFile(remainingPath)
|
||||
case "data":
|
||||
return v.validateDataFile(remainingPath)
|
||||
}
|
||||
|
||||
// Everything else is a format-owned directory: the only guard is that the
|
||||
// path stays inside the table.
|
||||
return validateFileSegments(remainingPath)
|
||||
}
|
||||
|
||||
// IsTableMarkerFile reports whether a name is one of the marker files a table
|
||||
// keeps at its root.
|
||||
func IsTableMarkerFile(name string) bool {
|
||||
return lanceMarkerFiles[name]
|
||||
}
|
||||
|
||||
// validateFileSegments rejects traversal and empty segments in a path whose
|
||||
// contents the catalog does not otherwise interpret.
|
||||
func validateFileSegments(path string) error {
|
||||
for _, segment := range strings.Split(path, "/") {
|
||||
if segment == "" || segment == "." || segment == ".." {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "invalid path segment in: " + path,
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateDirectoryPath validates intermediate subdirectories in a path
|
||||
// isMetadata indicates if we're in the metadata directory (true) or data directory (false)
|
||||
func validateDirectoryPath(normalizedPath string, isMetadata bool) error {
|
||||
if isMetadata {
|
||||
// For metadata, reject any subdirectories (enforce flat structure under metadata/)
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "metadata directory does not support subdirectories",
|
||||
}
|
||||
}
|
||||
|
||||
// For data, validate each partition or subdirectory segment
|
||||
subdirs := strings.Split(normalizedPath, "/")
|
||||
for _, subdir := range subdirs {
|
||||
if subdir == "" {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "invalid partition or subdirectory format in data path: empty segment",
|
||||
}
|
||||
}
|
||||
// For data, allow both partitions and valid subdirectories
|
||||
if !partitionPathPattern.MatchString(subdir) && !isValidSubdirectory(subdir) {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "invalid partition or subdirectory format in data path",
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateFilePatterns validates a filename against allowed patterns
|
||||
// isMetadata indicates if we're validating metadata files (true) or data files (false)
|
||||
func validateFilePatterns(filename string, isMetadata bool) error {
|
||||
var patterns []*regexp.Regexp
|
||||
var errorMsg string
|
||||
|
||||
if isMetadata {
|
||||
patterns = metadataFilePatterns
|
||||
errorMsg = "invalid metadata file format: must be a valid Iceberg metadata, manifest, or snapshot file"
|
||||
} else {
|
||||
patterns = dataFilePatterns
|
||||
errorMsg = "invalid data file format: must be .parquet, .orc, or .avro"
|
||||
}
|
||||
|
||||
for _, pattern := range patterns {
|
||||
if pattern.MatchString(filename) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: errorMsg,
|
||||
}
|
||||
}
|
||||
|
||||
// validateFile validates files with a unified logic for metadata and data directories
|
||||
// isMetadata indicates whether we're validating metadata files (true) or data files (false)
|
||||
// The logic is:
|
||||
// 1. If path ends with "/", it's a directory - validate all parts and return nil
|
||||
// 2. Otherwise, validate intermediate parts, then check the filename against patterns
|
||||
func (v *IcebergLayoutValidator) validateFile(path string, isMetadata bool) error {
|
||||
// Detect if it's a directory (path ends with "/")
|
||||
if strings.HasSuffix(path, "/") {
|
||||
// Normalize by removing trailing slash
|
||||
normalizedPath := strings.TrimSuffix(path, "/")
|
||||
return validateDirectoryPath(normalizedPath, isMetadata)
|
||||
}
|
||||
|
||||
filename := pathpkg.Base(path)
|
||||
|
||||
// Validate intermediate subdirectories if present
|
||||
// Find if there are intermediate directories by looking for the last slash
|
||||
lastSlash := strings.LastIndex(path, "/")
|
||||
if lastSlash != -1 {
|
||||
dir := path[:lastSlash]
|
||||
if err := validateDirectoryPath(dir, isMetadata); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Check against allowed file patterns
|
||||
err := validateFilePatterns(filename, isMetadata)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Path could be for a directory without a trailing slash, e.g., "data/year=2024"
|
||||
if !isMetadata {
|
||||
if partitionPathPattern.MatchString(filename) || isValidSubdirectory(filename) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// validateMetadataFile validates files in the metadata/ directory
|
||||
// This is a thin wrapper that calls validateFile with isMetadata=true
|
||||
func (v *IcebergLayoutValidator) validateMetadataFile(path string) error {
|
||||
return v.validateFile(path, true)
|
||||
}
|
||||
|
||||
// validateDataFile validates files in the data/ directory
|
||||
// This is a thin wrapper that calls validateFile with isMetadata=false
|
||||
func (v *IcebergLayoutValidator) validateDataFile(path string) error {
|
||||
return v.validateFile(path, false)
|
||||
}
|
||||
|
||||
// isValidSubdirectory checks if a path component is a valid subdirectory name
|
||||
func isValidSubdirectory(name string) bool {
|
||||
// Allow alphanumeric, underscore, hyphen, and UUID-style directories
|
||||
return validSubdirectoryPattern.MatchString(name)
|
||||
}
|
||||
|
||||
// IcebergLayoutError represents an Iceberg layout validation error
|
||||
type IcebergLayoutError struct {
|
||||
Code string
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *IcebergLayoutError) Error() string {
|
||||
return e.Message
|
||||
}
|
||||
|
||||
// Error code for Iceberg layout violations
|
||||
const (
|
||||
ErrCodeInvalidIcebergLayout = "InvalidIcebergLayout"
|
||||
)
|
||||
|
||||
// TableBucketFileValidator validates file uploads to table buckets
|
||||
type TableBucketFileValidator struct {
|
||||
layoutValidator *IcebergLayoutValidator
|
||||
}
|
||||
|
||||
// NewTableBucketFileValidator creates a new table bucket file validator
|
||||
func NewTableBucketFileValidator() *TableBucketFileValidator {
|
||||
return &TableBucketFileValidator{
|
||||
layoutValidator: NewIcebergLayoutValidator(),
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateTableBucketUpload checks if a file upload to a table bucket conforms to Iceberg layout
|
||||
// fullPath is the complete filer path (e.g., /buckets/mybucket/mynamespace/mytable/data/file.parquet)
|
||||
// Returns nil if the path is not a table bucket path or if validation passes
|
||||
// Returns an error if the file doesn't conform to Iceberg layout
|
||||
func (v *TableBucketFileValidator) ValidateTableBucketUpload(fullPath string) error {
|
||||
// Check if this is a table bucket path
|
||||
if !strings.HasPrefix(fullPath, TablesPath+"/") {
|
||||
return nil // Not a table bucket, no validation needed
|
||||
}
|
||||
|
||||
// Extract the path relative to table bucket root
|
||||
// Format: /buckets/{bucket}/{namespace}/{table}/{relative-path}
|
||||
relativePath := strings.TrimPrefix(fullPath, TablesPath+"/")
|
||||
parts := strings.SplitN(relativePath, "/", 4)
|
||||
|
||||
// Need at least bucket/namespace/table/file
|
||||
if len(parts) < 4 {
|
||||
// Creating bucket, namespace, or table directories - allow only if preceding parts are non-empty
|
||||
for i := 0; i < len(parts); i++ {
|
||||
if parts[i] == "" {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "bucket, namespace, and table segments cannot be empty",
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// For full paths, also verify bucket, namespace, and table segments are non-empty
|
||||
if parts[0] == "" || parts[1] == "" || parts[2] == "" {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "bucket, namespace, and table segments cannot be empty",
|
||||
}
|
||||
}
|
||||
|
||||
// The last part is the path within the table (data/file.parquet or metadata/v1.json)
|
||||
tableRelativePath := parts[3]
|
||||
if tableRelativePath == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reject paths with empty segments (double slashes) within the table path
|
||||
if strings.HasPrefix(tableRelativePath, "/") || strings.Contains(tableRelativePath, "//") {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "bucket, namespace, and table segments cannot be empty",
|
||||
}
|
||||
}
|
||||
|
||||
return v.layoutValidator.ValidateFilePath(tableRelativePath)
|
||||
}
|
||||
136
weed/s3api/s3tables/iceberg_layout_test.go
Normal file
136
weed/s3api/s3tables/iceberg_layout_test.go
Normal file
@@ -0,0 +1,136 @@
|
||||
package s3tables
|
||||
|
||||
import "testing"
|
||||
|
||||
// TestIcebergLayoutValidator_AcceptsRealWorldManifestNames pins down the
|
||||
// filename patterns we must accept across Iceberg engines. The original strict
|
||||
// regex only covered Iceberg-internal naming (`{uuid}-m{n}.avro`,
|
||||
// `snap-{n}-{n}-{uuid}.avro`) and rejected real manifests written by Flink and
|
||||
// other writers, causing 403s during INSERT commits. The catch-all entries
|
||||
// added alongside this test must keep these names valid.
|
||||
func TestIcebergLayoutValidator_AcceptsRealWorldManifestNames(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
path string
|
||||
}{
|
||||
{
|
||||
"flink-style manifest (job-id + checkpoint + operator-id + counter)",
|
||||
"metadata/02678a59b3d6b460ba392851d77155fc-1-cbc357ccb763df2852fee8c4fc7d55f2-00001.avro",
|
||||
},
|
||||
{
|
||||
"spark-style manifest with two uuids and -m suffix",
|
||||
"metadata/00000000-0000-0000-0000-000000000000-m0.avro",
|
||||
},
|
||||
{
|
||||
"snapshot manifest list with iceberg-internal naming",
|
||||
"metadata/snap-7234891234567890123-1-82e3eec4-3aee-414f-a444-94c03c641d20.avro",
|
||||
},
|
||||
{
|
||||
"versioned table metadata",
|
||||
"metadata/v1.metadata.json",
|
||||
},
|
||||
{
|
||||
"uuid-named metadata json (newer iceberg)",
|
||||
"metadata/82e3eec4-3aee-414f-a444-94c03c641d20.metadata.json",
|
||||
},
|
||||
{
|
||||
"flink-style metadata json with dashes and digits",
|
||||
"metadata/02678a59-1-cbc357cc.metadata.json",
|
||||
},
|
||||
{
|
||||
"version hint",
|
||||
"metadata/version-hint.text",
|
||||
},
|
||||
{
|
||||
"trino/iceberg stats file",
|
||||
"metadata/table.stats",
|
||||
},
|
||||
}
|
||||
|
||||
v := NewIcebergLayoutValidator()
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if err := v.ValidateFilePath(tc.path); err != nil {
|
||||
t.Errorf("expected %q to be accepted, got error: %v", tc.path, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestIcebergLayoutValidator_RejectsClearlyBadMetadataNames guards the
|
||||
// catch-all from being too permissive — paths that look like attempts to
|
||||
// escape the metadata layout or have forbidden file types must still fail.
|
||||
func TestIcebergLayoutValidator_RejectsClearlyBadMetadataNames(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
path string
|
||||
}{
|
||||
{"random extension", "metadata/random-file.txt"},
|
||||
{"executable masquerading as avro path", "metadata/evil.sh"},
|
||||
{"subdirectory under metadata is not allowed", "metadata/sub/file.avro"},
|
||||
{"top-level dir other than metadata or data", "garbage/file.avro"},
|
||||
{"catch-all anchors must reject trailing extension", "metadata/file.avro.txt"},
|
||||
}
|
||||
|
||||
v := NewIcebergLayoutValidator()
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if err := v.ValidateFilePath(tc.path); err == nil {
|
||||
t.Errorf("expected %q to be rejected, but it passed validation", tc.path)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestIcebergLayoutValidator_AcceptsRealWorldDataFiles is a sanity check that
|
||||
// the patterns most engines actually emit for data files still pass.
|
||||
func TestIcebergLayoutValidator_AcceptsRealWorldDataFiles(t *testing.T) {
|
||||
cases := []string{
|
||||
"data/00000-0-ede83b82-08e1-40cd-af8a-6d83680a5194-00001.parquet",
|
||||
"data/part-00000.parquet",
|
||||
"data/some-file.orc",
|
||||
"data/year=2026/month=05/00000-0-uuid.parquet",
|
||||
}
|
||||
v := NewIcebergLayoutValidator()
|
||||
for _, p := range cases {
|
||||
t.Run(p, func(t *testing.T) {
|
||||
if err := v.ValidateFilePath(p); err != nil {
|
||||
t.Errorf("expected %q to be accepted, got error: %v", p, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A Lance dataset writes fragments into data/ and keeps its own bookkeeping in
|
||||
// underscore-prefixed directories. The validator runs on the S3 door, where the
|
||||
// table's format is not in hand, so it has to admit both formats' layouts.
|
||||
func TestValidateFilePathAcceptsLanceLayout(t *testing.T) {
|
||||
v := NewIcebergLayoutValidator()
|
||||
allowed := []string{
|
||||
"data/01111110101110001101011164cadc43919eace9c608107bd9.lance",
|
||||
"_versions/1.manifest",
|
||||
"_versions/9223372036854775806.manifest",
|
||||
"_transactions/0-ddb27ab7-2e5c-42c7-b4bf-265d8a3ff636.txn",
|
||||
"_indices/85814508-ed9a-41f2-b939-2050bb7a0ed5-fts/index.idx",
|
||||
"_deletions/_deletions-1.arrow",
|
||||
".lance-reserved",
|
||||
".lance-deregistered",
|
||||
}
|
||||
for _, path := range allowed {
|
||||
if err := v.ValidateFilePath(path); err != nil {
|
||||
t.Errorf("ValidateFilePath(%q) = %v, want nil", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
rejected := []string{
|
||||
"_versions/../../escape",
|
||||
"_versions//empty",
|
||||
"notadir/file.lance",
|
||||
"random.txt",
|
||||
}
|
||||
for _, path := range rejected {
|
||||
if err := v.ValidateFilePath(path); err == nil {
|
||||
t.Errorf("ValidateFilePath(%q) = nil, want an error", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
193
weed/s3api/s3tables/iceberg_manifest.go
Normal file
193
weed/s3api/s3tables/iceberg_manifest.go
Normal file
@@ -0,0 +1,193 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"time"
|
||||
|
||||
"github.com/apache/iceberg-go"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const secondsPerDay = int64(24 * 60 * 60)
|
||||
|
||||
// ReadManifest parses an Iceberg manifest and converts its partition values
|
||||
// while the manifest's own schema still describes them.
|
||||
//
|
||||
// iceberg-go converts what the Avro decoder returned for a logical type - a
|
||||
// time.Time for a date, a time.Duration for a time - to an Iceberg value
|
||||
// lazily, on the first Partition() call, using the logical types it read from
|
||||
// the manifest being parsed. ManifestWriter.addEntry rebinds those to the
|
||||
// logical types of the manifest it is about to write before it makes that
|
||||
// call, so an entry that is read and written again without anyone looking at
|
||||
// its partition converts against the wrong schema. A day partition is where
|
||||
// the two disagree: iceberg-go's day transform reports an int32 result type,
|
||||
// so the manifest it writes has no date logical type for that field, nothing
|
||||
// converts, and the time.Time reaches the encoder as "cannot use time.Time
|
||||
// with Avro type int".
|
||||
//
|
||||
// What survives that is a value iceberg-go never recognized on read either: it
|
||||
// takes a partition field's logical type from the last branch of its Avro
|
||||
// union, so a writer that spells an optional partition [<type>, null] rather
|
||||
// than [null, <type>] hides it, and a time partition is then worse than a
|
||||
// failed write - time.Duration converts to int64 nanoseconds and silently
|
||||
// records the wrong value. normalizePartitionValue puts those back.
|
||||
//
|
||||
// specs maps partition spec ID to spec the way the table metadata records them.
|
||||
func ReadManifest(m iceberg.ManifestFile, manifest []byte, discardDeleted bool, specs map[int]iceberg.PartitionSpec, schema *iceberg.Schema) ([]iceberg.ManifestEntry, error) {
|
||||
entries, err := iceberg.ReadManifest(m, bytes.NewReader(manifest), discardDeleted)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var partitionFields []iceberg.NestedField
|
||||
var spec iceberg.PartitionSpec
|
||||
var specFound bool
|
||||
if schema != nil {
|
||||
spec, specFound = specs[int(m.PartitionSpecID())]
|
||||
if specFound {
|
||||
if partitionType := spec.PartitionType(schema); partitionType != nil {
|
||||
partitionFields = partitionType.FieldList
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for i, entry := range entries {
|
||||
partition := entry.DataFile().Partition()
|
||||
changed := false
|
||||
for _, field := range partitionFields {
|
||||
value, ok := partition[field.ID]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if normalized, ok := normalizePartitionValue(value, field.Type); ok {
|
||||
partition[field.ID] = normalized
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if changed {
|
||||
// iceberg-go returns a defensive copy from Partition(), so the
|
||||
// in-place edits above never reach the underlying DataFile.
|
||||
// Rebuild the entry with a DataFile that carries the normalized
|
||||
// partition so a subsequent write encodes the Iceberg values.
|
||||
entries[i] = rebuildManifestEntry(entry, spec, partition)
|
||||
}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// rebuildManifestEntry returns a copy of entry whose DataFile carries the
|
||||
// given partition map. It is used to persist partition values normalized
|
||||
// after a read, since iceberg-go's Partition() getter hands back a clone
|
||||
// rather than the live map. Only the partition changes; every other DataFile
|
||||
// field is copied through the builder so manifest round-trips are preserved.
|
||||
func rebuildManifestEntry(entry iceberg.ManifestEntry, spec iceberg.PartitionSpec, partition map[int]any) iceberg.ManifestEntry {
|
||||
df := entry.DataFile()
|
||||
builder, err := iceberg.NewDataFileBuilder(
|
||||
spec,
|
||||
df.ContentType(),
|
||||
df.FilePath(),
|
||||
df.FileFormat(),
|
||||
partition,
|
||||
nil, // logical types are derived from the spec at write time
|
||||
nil,
|
||||
df.Count(),
|
||||
df.FileSizeBytes(),
|
||||
)
|
||||
if err != nil {
|
||||
// The original DataFile already validated these fields, so the only
|
||||
// ways to get here are a nil spec (which rebuildManifestEntry's caller
|
||||
// guards against) or a zero-record / zero-byte file, which the builder
|
||||
// rejects with recordCount <= 0 or fileSize <= 0. Fall back to the
|
||||
// original entry rather than panic; the unnormalized partition survives
|
||||
// but an empty file is unlikely to carry a logical partition value.
|
||||
return entry
|
||||
}
|
||||
|
||||
builder.BlockSizeInBytes(64 * 1024 * 1024) // v1 schema default; deprecated and omitted in v2+
|
||||
if sizes := df.ColumnSizes(); sizes != nil {
|
||||
builder.ColumnSizes(sizes)
|
||||
}
|
||||
if counts := df.ValueCounts(); counts != nil {
|
||||
builder.ValueCounts(counts)
|
||||
}
|
||||
if counts := df.NullValueCounts(); counts != nil {
|
||||
builder.NullValueCounts(counts)
|
||||
}
|
||||
if counts := df.NaNValueCounts(); counts != nil {
|
||||
builder.NaNValueCounts(counts)
|
||||
}
|
||||
if counts := df.DistinctValueCounts(); counts != nil {
|
||||
builder.DistinctValueCounts(counts)
|
||||
}
|
||||
if bounds := df.LowerBoundValues(); bounds != nil {
|
||||
builder.LowerBoundValues(bounds)
|
||||
}
|
||||
if bounds := df.UpperBoundValues(); bounds != nil {
|
||||
builder.UpperBoundValues(bounds)
|
||||
}
|
||||
if key := df.KeyMetadata(); key != nil {
|
||||
builder.KeyMetadata(key)
|
||||
}
|
||||
if offsets := df.SplitOffsets(); offsets != nil {
|
||||
builder.SplitOffsets(offsets)
|
||||
}
|
||||
if ids := df.EqualityFieldIDs(); ids != nil {
|
||||
builder.EqualityFieldIDs(ids)
|
||||
}
|
||||
if id := df.SortOrderID(); id != nil {
|
||||
builder.SortOrderID(*id)
|
||||
}
|
||||
if id := df.FirstRowID(); id != nil {
|
||||
builder.FirstRowID(*id)
|
||||
}
|
||||
if ref := df.ReferencedDataFile(); ref != nil {
|
||||
builder.ReferencedDataFile(*ref)
|
||||
}
|
||||
if off := df.ContentOffset(); off != nil {
|
||||
builder.ContentOffset(*off)
|
||||
}
|
||||
if size := df.ContentSizeInBytes(); size != nil {
|
||||
builder.ContentSizeInBytes(*size)
|
||||
}
|
||||
|
||||
newEntry := iceberg.NewManifestEntryBuilder(entry.Status(), ptrInt64(entry.SnapshotID()), builder.Build())
|
||||
if seq := entry.SequenceNum(); seq != -1 {
|
||||
newEntry.SequenceNum(seq)
|
||||
}
|
||||
if fileSeq := entry.FileSequenceNum(); fileSeq != nil {
|
||||
newEntry.FileSequenceNum(*fileSeq)
|
||||
}
|
||||
return newEntry.Build()
|
||||
}
|
||||
|
||||
func ptrInt64(v int64) *int64 { return &v }
|
||||
|
||||
// normalizePartitionValue converts one value the Avro decoder returned for a
|
||||
// logical type to the Iceberg representation the manifest writer expects,
|
||||
// mirroring what iceberg-go does itself for the unions it does recognize.
|
||||
func normalizePartitionValue(value any, fieldType iceberg.Type) (any, bool) {
|
||||
switch v := value.(type) {
|
||||
case time.Time:
|
||||
utc := v.UTC()
|
||||
switch fieldType.(type) {
|
||||
case iceberg.DateType, iceberg.Int32Type:
|
||||
// A day transform reports an int32 result type and an identity
|
||||
// transform on a date column reports date; both hold days.
|
||||
midnight := time.Date(utc.Year(), utc.Month(), utc.Day(), 0, 0, 0, 0, time.UTC)
|
||||
return iceberg.Date(midnight.Unix() / secondsPerDay), true
|
||||
case iceberg.TimestampType, iceberg.TimestampTzType:
|
||||
return iceberg.Timestamp(utc.UnixMicro()), true
|
||||
case iceberg.TimestampNsType, iceberg.TimestampTzNsType:
|
||||
return iceberg.TimestampNano(utc.UnixNano()), true
|
||||
}
|
||||
case time.Duration:
|
||||
if _, ok := fieldType.(iceberg.TimeType); ok {
|
||||
return iceberg.Time(v.Microseconds()), true
|
||||
}
|
||||
case [16]byte:
|
||||
if _, ok := fieldType.(iceberg.UUIDType); ok {
|
||||
return uuid.UUID(v), true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
168
weed/s3api/s3tables/iceberg_manifest_list.go
Normal file
168
weed/s3api/s3tables/iceberg_manifest_list.go
Normal file
@@ -0,0 +1,168 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/apache/iceberg-go"
|
||||
)
|
||||
|
||||
// avroFileMagic prefixes every Avro object container file.
|
||||
var avroFileMagic = []byte{'O', 'b', 'j', 1}
|
||||
|
||||
// formatVersionKey is the Avro header entry naming the Iceberg format version.
|
||||
const formatVersionKey = "format-version"
|
||||
|
||||
// ReadManifestList parses an Iceberg manifest list, tolerating writers that
|
||||
// leave the format version out of the Avro header.
|
||||
//
|
||||
// The Iceberg spec pins the header metadata of manifest *files* but says
|
||||
// nothing about manifest *lists*, so writers disagree: Java and PyIceberg
|
||||
// record "format-version", DuckDB writes no header metadata at all. iceberg-go
|
||||
// reads a missing entry as v1, which makes every v2 manifest reachable from
|
||||
// such a list fail with "manifest file's 'format-version' metadata indicates
|
||||
// version 2, but entry from manifest list indicates version 1", and silently
|
||||
// decodes delete manifests as data manifests because v1 has no "content"
|
||||
// field. Both leave DuckDB-written tables unmaintainable.
|
||||
//
|
||||
// When the entry is missing we derive the version from the record schema the
|
||||
// writer embedded — the fields it carries say which version wrote it — and add
|
||||
// it to the header before handing the bytes to iceberg-go.
|
||||
func ReadManifestList(manifestList []byte) ([]iceberg.ManifestFile, error) {
|
||||
if patched, ok := withFormatVersion(manifestList); ok {
|
||||
manifestList = patched
|
||||
}
|
||||
return iceberg.ReadManifestList(bytes.NewReader(manifestList))
|
||||
}
|
||||
|
||||
// withFormatVersion returns manifestList with a "format-version" header entry
|
||||
// spliced in. It reports false when the header already carries one, or when it
|
||||
// cannot be parsed — the untouched bytes then go to iceberg-go, which reports
|
||||
// the underlying problem.
|
||||
func withFormatVersion(manifestList []byte) ([]byte, bool) {
|
||||
metadata, terminator, err := readAvroFileMetadata(manifestList)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
if _, ok := metadata[formatVersionKey]; ok {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// The metadata map is a sequence of blocks closed by a zero count, so a
|
||||
// new single-entry block can be spliced in ahead of that closing count
|
||||
// without re-encoding what the writer already wrote.
|
||||
block := appendAvroLong(nil, 1)
|
||||
block = appendAvroBytes(block, []byte(formatVersionKey))
|
||||
block = appendAvroBytes(block, []byte(strconv.Itoa(manifestListFormatVersion(metadata["avro.schema"]))))
|
||||
|
||||
patched := make([]byte, 0, len(manifestList)+len(block))
|
||||
patched = append(patched, manifestList[:terminator]...)
|
||||
patched = append(patched, block...)
|
||||
return append(patched, manifestList[terminator:]...), true
|
||||
}
|
||||
|
||||
// manifestListFormatVersion infers the format version a manifest list was
|
||||
// written at from the fields of its manifest_file record: v2 added "content",
|
||||
// "sequence_number" and "min_sequence_number", v3 added "first_row_id".
|
||||
func manifestListFormatVersion(schema []byte) int {
|
||||
var record struct {
|
||||
Fields []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"fields"`
|
||||
}
|
||||
if err := json.Unmarshal(schema, &record); err != nil {
|
||||
return 1
|
||||
}
|
||||
version := 1
|
||||
for _, field := range record.Fields {
|
||||
switch field.Name {
|
||||
case "content", "sequence_number", "min_sequence_number":
|
||||
if version < 2 {
|
||||
version = 2
|
||||
}
|
||||
case "first_row_id":
|
||||
version = 3
|
||||
}
|
||||
}
|
||||
return version
|
||||
}
|
||||
|
||||
// readAvroFileMetadata decodes the header metadata map of an Avro object
|
||||
// container file, returning the map alongside the offset of the zero block
|
||||
// count that closes it.
|
||||
func readAvroFileMetadata(data []byte) (map[string][]byte, int, error) {
|
||||
if !bytes.HasPrefix(data, avroFileMagic) {
|
||||
return nil, 0, errors.New("not an avro object container file")
|
||||
}
|
||||
metadata := make(map[string][]byte)
|
||||
pos := len(avroFileMagic)
|
||||
for {
|
||||
blockStart := pos
|
||||
count, next, err := readAvroLong(data, pos)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
pos = next
|
||||
if count == 0 {
|
||||
return metadata, blockStart, nil
|
||||
}
|
||||
if count < 0 {
|
||||
// A negative count is followed by the block size in bytes.
|
||||
count = -count
|
||||
if _, pos, err = readAvroLong(data, pos); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
}
|
||||
for i := int64(0); i < count; i++ {
|
||||
key, next, err := readAvroBytes(data, pos)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
value, next, err := readAvroBytes(data, next)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
metadata[string(key)] = value
|
||||
pos = next
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// readAvroLong decodes the zig-zag varint at pos, returning it with the offset
|
||||
// just past it.
|
||||
func readAvroLong(data []byte, pos int) (int64, int, error) {
|
||||
if pos < 0 || pos >= len(data) {
|
||||
return 0, 0, fmt.Errorf("avro long at offset %d is out of range", pos)
|
||||
}
|
||||
value, n := binary.Varint(data[pos:])
|
||||
if n <= 0 {
|
||||
return 0, 0, fmt.Errorf("avro long at offset %d is truncated", pos)
|
||||
}
|
||||
return value, pos + n, nil
|
||||
}
|
||||
|
||||
// readAvroBytes decodes the length-prefixed byte sequence at pos, returning it
|
||||
// with the offset just past it. The result aliases data.
|
||||
func readAvroBytes(data []byte, pos int) ([]byte, int, error) {
|
||||
length, pos, err := readAvroLong(data, pos)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if length < 0 || int64(len(data)-pos) < length {
|
||||
return nil, 0, fmt.Errorf("avro bytes at offset %d are truncated", pos)
|
||||
}
|
||||
return data[pos : pos+int(length)], pos + int(length), nil
|
||||
}
|
||||
|
||||
func appendAvroLong(dst []byte, value int64) []byte {
|
||||
var scratch [binary.MaxVarintLen64]byte
|
||||
return append(dst, scratch[:binary.PutVarint(scratch[:], value)]...)
|
||||
}
|
||||
|
||||
func appendAvroBytes(dst []byte, value []byte) []byte {
|
||||
return append(appendAvroLong(dst, int64(len(value))), value...)
|
||||
}
|
||||
176
weed/s3api/s3tables/iceberg_manifest_list_test.go
Normal file
176
weed/s3api/s3tables/iceberg_manifest_list_test.go
Normal file
@@ -0,0 +1,176 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/apache/iceberg-go"
|
||||
)
|
||||
|
||||
// writeManifestList builds a manifest list holding one data and one delete
|
||||
// manifest, the shape DuckDB produces for a v2 table that has been updated.
|
||||
func writeManifestList(t *testing.T, version int) []byte {
|
||||
t.Helper()
|
||||
|
||||
dataManifest := iceberg.NewManifestFile(version, "s3://bucket/ns/tbl/metadata/data-m0.avro", 1024, 0, 7).
|
||||
SequenceNum(3, 3).
|
||||
Content(iceberg.ManifestContentData).
|
||||
AddedFiles(1).
|
||||
AddedRows(10).
|
||||
Build()
|
||||
files := []iceberg.ManifestFile{dataManifest}
|
||||
if version > 1 {
|
||||
// Delete manifests only exist from v2 onwards.
|
||||
files = append(files, iceberg.NewManifestFile(version, "s3://bucket/ns/tbl/metadata/deletes-m0.avro", 512, 0, 7).
|
||||
SequenceNum(3, 3).
|
||||
Content(iceberg.ManifestContentDeletes).
|
||||
AddedFiles(1).
|
||||
AddedRows(1).
|
||||
Build())
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
seqNum := int64(3)
|
||||
if err := iceberg.WriteManifestList(version, &buf, 7, nil, &seqNum, 0, files); err != nil {
|
||||
t.Fatalf("write v%d manifest list: %v", version, err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// stripFormatVersion rewrites an Avro header without its "format-version"
|
||||
// entry, reproducing what DuckDB writes: a manifest list carrying no Iceberg
|
||||
// header metadata at all.
|
||||
func stripFormatVersion(t *testing.T, data []byte) []byte {
|
||||
t.Helper()
|
||||
|
||||
metadata, terminator, err := readAvroFileMetadata(data)
|
||||
if err != nil {
|
||||
t.Fatalf("read avro header: %v", err)
|
||||
}
|
||||
if _, ok := metadata[formatVersionKey]; !ok {
|
||||
t.Fatal("fixture has no format-version to strip")
|
||||
}
|
||||
|
||||
var entries []byte
|
||||
count := 0
|
||||
for key, value := range metadata {
|
||||
if key == formatVersionKey {
|
||||
continue
|
||||
}
|
||||
entries = appendAvroBytes(appendAvroBytes(entries, []byte(key)), value)
|
||||
count++
|
||||
}
|
||||
|
||||
stripped := append([]byte{}, avroFileMagic...)
|
||||
stripped = appendAvroLong(stripped, int64(count))
|
||||
stripped = append(stripped, entries...)
|
||||
// Everything from the closing zero count on — sync marker and data
|
||||
// blocks included — is unaffected by the header rewrite.
|
||||
return append(stripped, data[terminator:]...)
|
||||
}
|
||||
|
||||
// DuckDB writes manifest lists with no Avro header metadata. iceberg-go now
|
||||
// infers the format version from the embedded writer schema on its own, so a
|
||||
// stripped v2 list is read as v2 even without the header entry. ReadManifestList
|
||||
// still patches the header for older iceberg-go releases and for writers whose
|
||||
// schema carries none of the version-distinguishing fields.
|
||||
func TestReadManifestListWithoutFormatVersion(t *testing.T) {
|
||||
stripped := stripFormatVersion(t, writeManifestList(t, 2))
|
||||
|
||||
// Baseline: iceberg-go reads the list as v2 by inferring the version from
|
||||
// the record schema's fields (content / sequence_number were added in v2).
|
||||
unpatched, err := iceberg.ReadManifestList(bytes.NewReader(stripped))
|
||||
if err != nil {
|
||||
t.Fatalf("iceberg.ReadManifestList: %v", err)
|
||||
}
|
||||
if got := unpatched[0].Version(); got != 2 {
|
||||
t.Fatalf("expected iceberg-go to infer v2, got v%d", got)
|
||||
}
|
||||
|
||||
manifests, err := ReadManifestList(stripped)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadManifestList: %v", err)
|
||||
}
|
||||
if len(manifests) != 2 {
|
||||
t.Fatalf("expected 2 manifests, got %d", len(manifests))
|
||||
}
|
||||
for _, mf := range manifests {
|
||||
if mf.Version() != 2 {
|
||||
t.Errorf("manifest %s: version = %d, want 2", mf.FilePath(), mf.Version())
|
||||
}
|
||||
if mf.SequenceNum() != 3 || mf.MinSequenceNum() != 3 {
|
||||
t.Errorf("manifest %s: sequence numbers = %d/%d, want 3/3", mf.FilePath(), mf.SequenceNum(), mf.MinSequenceNum())
|
||||
}
|
||||
}
|
||||
if got := manifests[0].ManifestContent(); got != iceberg.ManifestContentData {
|
||||
t.Errorf("first manifest content = %s, want data", got)
|
||||
}
|
||||
if got := manifests[1].ManifestContent(); got != iceberg.ManifestContentDeletes {
|
||||
t.Errorf("second manifest content = %s, want deletes", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadManifestListKeepsWrittenFormatVersion(t *testing.T) {
|
||||
for _, version := range []int{1, 2} {
|
||||
manifests, err := ReadManifestList(writeManifestList(t, version))
|
||||
if err != nil {
|
||||
t.Fatalf("v%d: ReadManifestList: %v", version, err)
|
||||
}
|
||||
if got := manifests[0].Version(); got != version {
|
||||
t.Errorf("v%d: version = %d", version, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A v1 manifest list without the header entry must stay v1: its record schema
|
||||
// has none of the fields v2 added.
|
||||
func TestReadManifestListWithoutFormatVersionStaysV1(t *testing.T) {
|
||||
manifests, err := ReadManifestList(stripFormatVersion(t, writeManifestList(t, 1)))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadManifestList: %v", err)
|
||||
}
|
||||
if got := manifests[0].Version(); got != 1 {
|
||||
t.Errorf("version = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Unparseable input is handed to iceberg-go untouched so it keeps reporting
|
||||
// the underlying problem rather than a header-rewriting one.
|
||||
func TestReadManifestListRejectsNonAvro(t *testing.T) {
|
||||
for name, data := range map[string][]byte{
|
||||
"empty": nil,
|
||||
"not avro": []byte("this is not an avro file"),
|
||||
"truncated magic": avroFileMagic[:3],
|
||||
"truncated header": func() []byte {
|
||||
list := writeManifestList(t, 2)
|
||||
return list[:len(avroFileMagic)+2]
|
||||
}(),
|
||||
} {
|
||||
if _, err := ReadManifestList(data); err == nil {
|
||||
t.Errorf("%s: expected an error", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestListFormatVersion(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
schema string
|
||||
want int
|
||||
}{
|
||||
{"v1 fields", `{"fields":[{"name":"manifest_path"},{"name":"added_snapshot_id"}]}`, 1},
|
||||
{"v2 adds content", `{"fields":[{"name":"manifest_path"},{"name":"content"}]}`, 2},
|
||||
{"v2 adds sequence numbers", `{"fields":[{"name":"sequence_number"},{"name":"min_sequence_number"}]}`, 2},
|
||||
{"v3 adds first_row_id", `{"fields":[{"name":"content"},{"name":"first_row_id"}]}`, 3},
|
||||
{"v3 field ordering", `{"fields":[{"name":"first_row_id"},{"name":"content"}]}`, 3},
|
||||
{"unparseable schema", `not json`, 1},
|
||||
{"missing schema", ``, 1},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := manifestListFormatVersion([]byte(tc.schema)); got != tc.want {
|
||||
t.Errorf("manifestListFormatVersion() = %d, want %d", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
275
weed/s3api/s3tables/iceberg_manifest_test.go
Normal file
275
weed/s3api/s3tables/iceberg_manifest_test.go
Normal file
@@ -0,0 +1,275 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/apache/iceberg-go"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
|
||||
"github.com/twmb/avro/ocf"
|
||||
)
|
||||
|
||||
// A manifest whose partition union puts the value branch first hides the
|
||||
// logical type from iceberg-go, which then hands back whatever the Avro decoder
|
||||
// produced even once the partition is read. Rewriting such an entry fails when
|
||||
// the writer builds its partition summaries, or encodes the wrong number for a
|
||||
// time partition, so every logical type a partition field can carry has to come
|
||||
// back as an Iceberg value.
|
||||
func TestReadManifestNormalizesForeignPartitions(t *testing.T) {
|
||||
day := time.Date(2026, time.October, 11, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
sourceType iceberg.PrimitiveType
|
||||
transform iceberg.Transform
|
||||
logicalType string
|
||||
// decoded is the value the foreign manifest carries, as the Avro
|
||||
// decoder hands it back.
|
||||
decoded any
|
||||
want any
|
||||
// nullFirst spells the union the way Java and iceberg-rust do, which
|
||||
// iceberg-go converts on its own once the partition is read.
|
||||
nullFirst bool
|
||||
// rawIsNormalized marks transforms iceberg-go now converts itself on
|
||||
// read (day transforms), so the raw value already matches want.
|
||||
rawIsNormalized bool
|
||||
}{
|
||||
{
|
||||
name: "day transform written null-first converts on read",
|
||||
sourceType: iceberg.PrimitiveTypes.Timestamp,
|
||||
transform: iceberg.DayTransform{},
|
||||
logicalType: "date",
|
||||
decoded: day,
|
||||
want: iceberg.Date(20737),
|
||||
nullFirst: true,
|
||||
},
|
||||
{
|
||||
name: "day transform on a timestamp column",
|
||||
sourceType: iceberg.PrimitiveTypes.Timestamp,
|
||||
transform: iceberg.DayTransform{},
|
||||
logicalType: "date",
|
||||
decoded: day,
|
||||
want: iceberg.Date(20737),
|
||||
// iceberg-go now converts day-transform partitions to iceberg.Date
|
||||
// itself (applyDayTransformDates), so the raw read already yields
|
||||
// the Iceberg value regardless of union branch ordering.
|
||||
rawIsNormalized: true,
|
||||
},
|
||||
{
|
||||
name: "identity transform on a date column",
|
||||
sourceType: iceberg.PrimitiveTypes.Date,
|
||||
transform: iceberg.IdentityTransform{},
|
||||
logicalType: "date",
|
||||
decoded: day,
|
||||
want: iceberg.Date(20737),
|
||||
},
|
||||
{
|
||||
name: "identity transform on a timestamp column",
|
||||
sourceType: iceberg.PrimitiveTypes.Timestamp,
|
||||
transform: iceberg.IdentityTransform{},
|
||||
logicalType: "timestamp-micros",
|
||||
decoded: day.Add(3 * time.Hour),
|
||||
want: iceberg.Timestamp(day.Add(3 * time.Hour).UnixMicro()),
|
||||
},
|
||||
{
|
||||
name: "identity transform on a time column",
|
||||
sourceType: iceberg.PrimitiveTypes.Time,
|
||||
transform: iceberg.IdentityTransform{},
|
||||
logicalType: "time-micros",
|
||||
decoded: 3 * time.Hour,
|
||||
want: iceberg.Time((3 * time.Hour).Microseconds()),
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
schema := iceberg.NewSchema(0,
|
||||
iceberg.NestedField{ID: 1, Name: "source", Type: c.sourceType, Required: true},
|
||||
)
|
||||
spec := iceberg.NewPartitionSpec(iceberg.PartitionField{
|
||||
SourceIDs: []int{1}, FieldID: 1000, Name: "part", Transform: c.transform,
|
||||
})
|
||||
specs := map[int]iceberg.PartitionSpec{spec.ID(): spec}
|
||||
|
||||
dfBuilder, err := iceberg.NewDataFileBuilder(spec, iceberg.EntryContentData, "data/file.parquet",
|
||||
iceberg.ParquetFile, map[int]any{1000: c.want}, nil, nil, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("build data file: %v", err)
|
||||
}
|
||||
snapshotID := int64(1)
|
||||
entry := iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapshotID, nil, nil, dfBuilder.Build())
|
||||
|
||||
foreignBytes, foreignManifest := s3tablestest.ForeignPartitionManifest(t, schema, spec, entry,
|
||||
"metadata/foreign-manifest.avro", c.logicalType, c.decoded, !c.nullFirst)
|
||||
|
||||
// iceberg-go reads the logical type off the union's last branch, so
|
||||
// the ordering decides whether the entry arrives as an Iceberg value
|
||||
// or as whatever the Avro decoder produced.
|
||||
rawWant := any(c.decoded)
|
||||
if c.nullFirst || c.rawIsNormalized {
|
||||
rawWant = c.want
|
||||
}
|
||||
raw, err := iceberg.ReadManifest(foreignManifest, bytes.NewReader(foreignBytes), true)
|
||||
if err != nil {
|
||||
t.Fatalf("read foreign manifest: %v", err)
|
||||
}
|
||||
if got := raw[0].DataFile().Partition()[1000]; got != rawWant {
|
||||
t.Fatalf("unnormalized partition value = %#v (%T), want %#v (%T)", got, got, rawWant, rawWant)
|
||||
}
|
||||
|
||||
entries, err := ReadManifest(foreignManifest, foreignBytes, true, specs, schema)
|
||||
if err != nil {
|
||||
t.Fatalf("read foreign manifest: %v", err)
|
||||
}
|
||||
if got := entries[0].DataFile().Partition()[1000]; got != c.want {
|
||||
t.Fatalf("normalized partition value = %#v (%T), want %#v (%T)", got, got, c.want, c.want)
|
||||
}
|
||||
|
||||
// The point of normalizing: the entry can be written out again.
|
||||
var buf bytes.Buffer
|
||||
if _, err := iceberg.WriteManifest("metadata/manifest.avro", &buf, 2, spec, schema, snapshotID, entries); err != nil {
|
||||
t.Fatalf("write normalized manifest: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A manifest written under a spec the table metadata no longer records is
|
||||
// returned as read rather than refused: the caller decides what to do with it.
|
||||
func TestReadManifestUnknownSpec(t *testing.T) {
|
||||
schema := iceberg.NewSchema(0,
|
||||
iceberg.NestedField{ID: 1, Name: "source", Type: iceberg.PrimitiveTypes.Date, Required: true},
|
||||
)
|
||||
spec := iceberg.NewPartitionSpec(iceberg.PartitionField{
|
||||
SourceIDs: []int{1}, FieldID: 1000, Name: "part", Transform: iceberg.IdentityTransform{},
|
||||
})
|
||||
dfBuilder, err := iceberg.NewDataFileBuilder(spec, iceberg.EntryContentData, "data/file.parquet",
|
||||
iceberg.ParquetFile, map[int]any{1000: iceberg.Date(20737)}, nil, nil, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("build data file: %v", err)
|
||||
}
|
||||
snapshotID := int64(1)
|
||||
entry := iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapshotID, nil, nil, dfBuilder.Build())
|
||||
foreignBytes, foreignManifest := s3tablestest.ForeignPartitionManifest(t, schema, spec, entry,
|
||||
"metadata/foreign-manifest.avro", "date", time.Date(2026, time.October, 11, 0, 0, 0, 0, time.UTC), true)
|
||||
|
||||
entries, err := ReadManifest(foreignManifest, foreignBytes, true, nil, schema)
|
||||
if err != nil {
|
||||
t.Fatalf("read foreign manifest: %v", err)
|
||||
}
|
||||
if _, ok := entries[0].DataFile().Partition()[1000].(time.Time); !ok {
|
||||
t.Fatalf("partition value = %T, want it returned as read", entries[0].DataFile().Partition()[1000])
|
||||
}
|
||||
}
|
||||
|
||||
// iceberg-go now converts day-transform partitions to iceberg.Date during read
|
||||
// (applyDayTransformDates), so an entry nobody looked at between reading and
|
||||
// writing still carries an Iceberg value and writes cleanly. This used to fail
|
||||
// because conversion was lazy and ManifestWriter.addEntry rebound the logical
|
||||
// types to the output manifest — whose day transform carries no date logical
|
||||
// type — before the first Partition() call, leaving a time.Time that the
|
||||
// encoder rejected. The shim keeps the round-trip working for the identity
|
||||
// transforms iceberg-go still does not convert (see TestReadManifestNormalizesForeignPartitions).
|
||||
func TestReadManifestConvertsBeforeTheWriterRebindsLogicalTypes(t *testing.T) {
|
||||
schema := iceberg.NewSchema(0,
|
||||
iceberg.NestedField{ID: 1, Name: "event_time", Type: iceberg.PrimitiveTypes.Timestamp, Required: true},
|
||||
)
|
||||
spec := iceberg.NewPartitionSpec(iceberg.PartitionField{
|
||||
SourceIDs: []int{1}, FieldID: 1000, Name: "event_time_day", Transform: iceberg.DayTransform{},
|
||||
})
|
||||
specs := map[int]iceberg.PartitionSpec{spec.ID(): spec}
|
||||
dfBuilder, err := iceberg.NewDataFileBuilder(spec, iceberg.EntryContentData, "data/file.parquet",
|
||||
iceberg.ParquetFile, map[int]any{1000: iceberg.Date(20737)}, nil, nil, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("build data file: %v", err)
|
||||
}
|
||||
snapshotID := int64(1)
|
||||
entry := iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapshotID, nil, nil, dfBuilder.Build())
|
||||
foreignBytes, foreignManifest := s3tablestest.ForeignPartitionManifest(t, schema, spec, entry,
|
||||
"metadata/foreign-manifest.avro", "date", time.Date(2026, time.October, 11, 0, 0, 0, 0, time.UTC), false)
|
||||
|
||||
untouched, err := iceberg.ReadManifest(foreignManifest, bytes.NewReader(foreignBytes), true)
|
||||
if err != nil {
|
||||
t.Fatalf("read foreign manifest: %v", err)
|
||||
}
|
||||
got := untouched[0].DataFile().Partition()[1000]
|
||||
if _, ok := got.(iceberg.Date); !ok {
|
||||
t.Fatalf("iceberg-go now converts day partitions on read; got %T, want iceberg.Date", got)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if _, err := iceberg.WriteManifest("metadata/manifest.avro", &buf, 2, spec, schema, snapshotID, untouched); err != nil {
|
||||
t.Fatalf("write manifest read directly through iceberg-go: %v", err)
|
||||
}
|
||||
|
||||
entries, err := ReadManifest(foreignManifest, foreignBytes, true, specs, schema)
|
||||
if err != nil {
|
||||
t.Fatalf("read foreign manifest: %v", err)
|
||||
}
|
||||
buf.Reset()
|
||||
if _, err := iceberg.WriteManifest("metadata/manifest.avro", &buf, 2, spec, schema, snapshotID, entries); err != nil {
|
||||
t.Fatalf("write manifest read through the shim: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// rebuildManifestEntry rebuilds the DataFile through NewDataFileBuilder, which
|
||||
// has no setter for the original block_size_in_bytes (the field is not exposed
|
||||
// on DataFile). The v1 manifest schema includes block_size_in_bytes with a
|
||||
// default of 64 MiB and the spec says "Always write default in v1", so a
|
||||
// rebuilt entry written as v1 must carry that default — not 0.
|
||||
func TestReadManifestRebuiltEntryPreservesV1BlockSizeDefault(t *testing.T) {
|
||||
schema := iceberg.NewSchema(0,
|
||||
iceberg.NestedField{ID: 1, Name: "event_date", Type: iceberg.PrimitiveTypes.Date, Required: true},
|
||||
)
|
||||
spec := iceberg.NewPartitionSpec(iceberg.PartitionField{
|
||||
SourceIDs: []int{1}, FieldID: 1000, Name: "part", Transform: iceberg.IdentityTransform{},
|
||||
})
|
||||
specs := map[int]iceberg.PartitionSpec{spec.ID(): spec}
|
||||
dfBuilder, err := iceberg.NewDataFileBuilder(spec, iceberg.EntryContentData, "data/file.parquet",
|
||||
iceberg.ParquetFile, map[int]any{1000: iceberg.Date(20737)}, nil, nil, 1, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("build data file: %v", err)
|
||||
}
|
||||
snapshotID := int64(1)
|
||||
entry := iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapshotID, nil, nil, dfBuilder.Build())
|
||||
// valueFirst=true spells the union [<type>, null], hiding the logical type
|
||||
// from iceberg-go so the partition arrives as a time.Time. That triggers
|
||||
// normalizePartitionValue → rebuildManifestEntry, which is the path that
|
||||
// sets block_size_in_bytes.
|
||||
foreignBytes, foreignManifest := s3tablestest.ForeignPartitionManifest(t, schema, spec, entry,
|
||||
"metadata/foreign-manifest.avro", "date", time.Date(2026, time.October, 11, 0, 0, 0, 0, time.UTC), true)
|
||||
|
||||
entries, err := ReadManifest(foreignManifest, foreignBytes, true, specs, schema)
|
||||
if err != nil {
|
||||
t.Fatalf("read foreign manifest: %v", err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if _, err := iceberg.WriteManifest("metadata/manifest.avro", &buf, 1, spec, schema, snapshotID, entries); err != nil {
|
||||
t.Fatalf("write v1 manifest: %v", err)
|
||||
}
|
||||
|
||||
reader, err := ocf.NewReader(bytes.NewReader(buf.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("open v1 manifest: %v", err)
|
||||
}
|
||||
var record map[string]any
|
||||
if err := reader.Decode(&record); err != nil {
|
||||
t.Fatalf("decode v1 manifest: %v", err)
|
||||
}
|
||||
if err := reader.Close(); err != nil {
|
||||
t.Fatalf("close v1 manifest: %v", err)
|
||||
}
|
||||
dataFile, ok := record["data_file"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("data_file = %T, want record", record["data_file"])
|
||||
}
|
||||
got, ok := dataFile["block_size_in_bytes"]
|
||||
if !ok {
|
||||
t.Fatal("block_size_in_bytes missing from v1 manifest")
|
||||
}
|
||||
const want = int64(64 * 1024 * 1024)
|
||||
if got != want {
|
||||
t.Fatalf("block_size_in_bytes = %v, want %d", got, want)
|
||||
}
|
||||
}
|
||||
139
weed/s3api/s3tables/manager.go
Normal file
139
weed/s3api/s3tables/manager.go
Normal file
@@ -0,0 +1,139 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
// Manager provides reusable S3 Tables operations for shell/admin without HTTP routing.
|
||||
type Manager struct {
|
||||
handler *S3TablesHandler
|
||||
}
|
||||
|
||||
// NewManager creates a new Manager.
|
||||
func NewManager() *Manager {
|
||||
m := &Manager{handler: NewS3TablesHandler()}
|
||||
// Default to allowing access when IAM is not configured
|
||||
m.handler.SetDefaultAllow(true)
|
||||
return m
|
||||
}
|
||||
|
||||
// SetRegion sets the AWS region for ARN generation.
|
||||
func (m *Manager) SetRegion(region string) {
|
||||
m.handler.SetRegion(region)
|
||||
}
|
||||
|
||||
// SetAccountID sets the AWS account ID for ARN generation.
|
||||
func (m *Manager) SetAccountID(accountID string) {
|
||||
m.handler.SetAccountID(accountID)
|
||||
}
|
||||
|
||||
// SetDefaultAllow sets whether to allow access by default.
|
||||
func (m *Manager) SetDefaultAllow(allow bool) {
|
||||
m.handler.SetDefaultAllow(allow)
|
||||
}
|
||||
|
||||
// SetTrusted lets trusted local tooling (shell, admin console) bypass authorization.
|
||||
func (m *Manager) SetTrusted(trusted bool) {
|
||||
m.handler.SetTrusted(trusted)
|
||||
}
|
||||
|
||||
// Execute runs an S3 Tables operation and decodes the response into resp (if provided).
|
||||
func (m *Manager) Execute(ctx context.Context, filerClient FilerClient, operation string, req interface{}, resp interface{}, identity string) error {
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
httpReq, err := newManagerRequest(ctx, operation, body, identity)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
m.handler.HandleRequest(recorder, httpReq, filerClient)
|
||||
return decodeS3TablesHTTPResponse(recorder, resp)
|
||||
}
|
||||
|
||||
// AuthorizeCreateTable checks that identity may create the table the request
|
||||
// describes, without creating anything. A deferred create (Iceberg
|
||||
// stage-create) writes into the table bucket long before it registers the
|
||||
// table, so it passes this gate first.
|
||||
func (m *Manager) AuthorizeCreateTable(ctx context.Context, filerClient FilerClient, req *CreateTableRequest, identity string) error {
|
||||
httpReq, err := newManagerRequest(ctx, "CreateTable", nil, identity)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
_, authErr := m.handler.authorizeCreateTable(recorder, httpReq, filerClient, req.TableBucketARN, req.Namespace, req.Name, req.Tags)
|
||||
if authErr == nil {
|
||||
return nil
|
||||
}
|
||||
if decoded := decodeS3TablesHTTPResponse(recorder, nil); decoded != nil {
|
||||
return decoded
|
||||
}
|
||||
return authErr
|
||||
}
|
||||
|
||||
func newManagerRequest(ctx context.Context, operation string, body []byte, identity string) (*http.Request, error) {
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, "/", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/x-amz-json-1.1")
|
||||
httpReq.Header.Set("X-Amz-Target", "S3Tables."+operation)
|
||||
if identity != "" {
|
||||
httpReq.Header.Set(s3_constants.AmzAccountId, identity)
|
||||
httpReq = httpReq.WithContext(s3_constants.SetIdentityNameInContext(httpReq.Context(), identity))
|
||||
}
|
||||
return httpReq, nil
|
||||
}
|
||||
|
||||
func decodeS3TablesHTTPResponse(recorder *httptest.ResponseRecorder, resp interface{}) error {
|
||||
result := recorder.Result()
|
||||
defer result.Body.Close()
|
||||
data, err := io.ReadAll(result.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if result.StatusCode >= http.StatusBadRequest {
|
||||
var errResp S3TablesError
|
||||
if len(data) > 0 {
|
||||
if jsonErr := json.Unmarshal(data, &errResp); jsonErr == nil && (errResp.Type != "" || errResp.Message != "") {
|
||||
return &errResp
|
||||
}
|
||||
}
|
||||
return &S3TablesError{Type: ErrCodeInternalError, Message: string(bytes.TrimSpace(data))}
|
||||
}
|
||||
if resp == nil || len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(data, resp); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ManagerClient adapts a SeaweedFilerClient to the FilerClient interface.
|
||||
type ManagerClient struct {
|
||||
client filer_pb.SeaweedFilerClient
|
||||
}
|
||||
|
||||
// NewManagerClient wraps a filer client.
|
||||
func NewManagerClient(client filer_pb.SeaweedFilerClient) *ManagerClient {
|
||||
return &ManagerClient{client: client}
|
||||
}
|
||||
|
||||
// WithFilerClient implements FilerClient.
|
||||
func (m *ManagerClient) WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
|
||||
if m.client == nil {
|
||||
return errors.New("nil filer client")
|
||||
}
|
||||
return fn(m.client)
|
||||
}
|
||||
99
weed/s3api/s3tables/manager_test.go
Normal file
99
weed/s3api/s3tables/manager_test.go
Normal file
@@ -0,0 +1,99 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var errFilerReached = errors.New("filer reached")
|
||||
|
||||
// recordingFilerClient reports whether the handler reached the filer. The
|
||||
// CreateTableBucket handler authorizes before touching the filer, so "filer
|
||||
// reached" is a proxy for "authorization passed".
|
||||
type recordingFilerClient struct {
|
||||
called bool
|
||||
}
|
||||
|
||||
func (c *recordingFilerClient) WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
|
||||
c.called = true
|
||||
return errFilerReached
|
||||
}
|
||||
|
||||
// The Manager path (used by the Iceberg catalog) enforces authorization for
|
||||
// authenticated callers and only falls open for trusted/zero-config access.
|
||||
func TestManagerCreateTableBucketAuthorization(t *testing.T) {
|
||||
lowPriv := &testIdentity{
|
||||
Name: "alice",
|
||||
Account: &testIdentityAccount{Id: s3_constants.AccountAdminId},
|
||||
Actions: []string{"Read"},
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
defaultAllow bool
|
||||
trusted bool
|
||||
ctx context.Context
|
||||
identity string
|
||||
wantFiler bool // true => authorization passed (filer reached)
|
||||
}{
|
||||
{
|
||||
name: "authenticated identity struct is enforced",
|
||||
defaultAllow: true,
|
||||
ctx: s3_constants.SetIdentityInContext(context.Background(), lowPriv),
|
||||
identity: "alice",
|
||||
wantFiler: false,
|
||||
},
|
||||
{
|
||||
name: "secured manager denies a name without struct",
|
||||
defaultAllow: false,
|
||||
ctx: context.Background(),
|
||||
identity: "alice",
|
||||
wantFiler: false,
|
||||
},
|
||||
{
|
||||
name: "untrusted name without struct is enforced",
|
||||
defaultAllow: true,
|
||||
ctx: context.Background(),
|
||||
identity: "alice",
|
||||
wantFiler: false,
|
||||
},
|
||||
{
|
||||
name: "trusted manager allows a name without struct",
|
||||
defaultAllow: true,
|
||||
trusted: true,
|
||||
ctx: context.Background(),
|
||||
identity: "alice",
|
||||
wantFiler: true,
|
||||
},
|
||||
{
|
||||
name: "admin principal is allowed",
|
||||
defaultAllow: false,
|
||||
ctx: context.Background(),
|
||||
identity: s3_constants.AccountAdminId,
|
||||
wantFiler: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
m := NewManager()
|
||||
m.SetDefaultAllow(tc.defaultAllow)
|
||||
m.SetTrusted(tc.trusted)
|
||||
fc := &recordingFilerClient{}
|
||||
err := m.Execute(tc.ctx, fc, "CreateTableBucket", &CreateTableBucketRequest{Name: "testbucket"}, nil, tc.identity)
|
||||
require.Error(t, err) // denied (403) or the filer sentinel (reached); never nil here
|
||||
assert.Equal(t, tc.wantFiler, fc.called, "filer reached should equal authorization passed")
|
||||
if !tc.wantFiler {
|
||||
var s3Err *S3TablesError
|
||||
require.ErrorAs(t, err, &s3Err)
|
||||
assert.Equal(t, ErrCodeAccessDenied, s3Err.Type)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
435
weed/s3api/s3tables/permissions.go
Normal file
435
weed/s3api/s3tables/permissions.go
Normal file
@@ -0,0 +1,435 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
|
||||
)
|
||||
|
||||
// Permission represents a specific action permission
|
||||
type Permission string
|
||||
|
||||
// IAM Policy structures for evaluation
|
||||
type PolicyDocument struct {
|
||||
Version string `json:"Version"`
|
||||
Statement []Statement `json:"Statement"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON handles both single statement object and array of statements
|
||||
// AWS allows {"Statement": {...}} or {"Statement": [{...}]}
|
||||
func (pd *PolicyDocument) UnmarshalJSON(data []byte) error {
|
||||
type Alias PolicyDocument
|
||||
aux := &struct {
|
||||
Statement interface{} `json:"Statement"`
|
||||
*Alias
|
||||
}{
|
||||
Alias: (*Alias)(pd),
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &aux); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Handle Statement as either a single object or array
|
||||
switch s := aux.Statement.(type) {
|
||||
case map[string]interface{}:
|
||||
// Single statement object - unmarshal to one Statement
|
||||
stmtData, err := json.Marshal(s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal single statement: %w", err)
|
||||
}
|
||||
var stmt Statement
|
||||
if err := json.Unmarshal(stmtData, &stmt); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal single statement: %w", err)
|
||||
}
|
||||
pd.Statement = []Statement{stmt}
|
||||
case []interface{}:
|
||||
// Array of statements - normal handling
|
||||
stmtData, err := json.Marshal(s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal statement array: %w", err)
|
||||
}
|
||||
if err := json.Unmarshal(stmtData, &pd.Statement); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal statement array: %w", err)
|
||||
}
|
||||
case nil:
|
||||
// No statements
|
||||
pd.Statement = []Statement{}
|
||||
default:
|
||||
return fmt.Errorf("Statement must be an object or array, got %T", aux.Statement)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type Statement struct {
|
||||
Effect string `json:"Effect"` // "Allow" or "Deny"
|
||||
Principal interface{} `json:"Principal"` // Can be string, []string, or map
|
||||
Action interface{} `json:"Action"` // Can be string or []string
|
||||
Resource interface{} `json:"Resource"` // Can be string or []string
|
||||
Condition map[string]map[string]interface{} `json:"Condition,omitempty"`
|
||||
}
|
||||
|
||||
type PolicyContext struct {
|
||||
Namespace string
|
||||
TableName string
|
||||
TableBucketName string
|
||||
IdentityActions []string
|
||||
RequestTags map[string]string
|
||||
ResourceTags map[string]string
|
||||
TableBucketTags map[string]string
|
||||
TagKeys []string
|
||||
SSEAlgorithm string
|
||||
KMSKeyArn string
|
||||
StorageClass string
|
||||
DefaultAllow bool
|
||||
}
|
||||
|
||||
// CheckPermissionWithContext checks permission with optional resource and condition context.
|
||||
func CheckPermissionWithContext(operation, principal, owner, resourcePolicy, resourceARN string, ctx *PolicyContext) bool {
|
||||
// Deny access if identities are empty
|
||||
if principal == "" || owner == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
// Admin always has permission.
|
||||
if principal == s3_constants.AccountAdminId {
|
||||
return true
|
||||
}
|
||||
|
||||
glog.V(2).Infof("S3Tables: CheckPermission operation=%s principal=%s owner=%s", operation, principal, owner)
|
||||
|
||||
return checkPermission(operation, principal, owner, resourcePolicy, resourceARN, ctx)
|
||||
}
|
||||
|
||||
func checkPermission(operation, principal, owner, resourcePolicy, resourceARN string, ctx *PolicyContext) bool {
|
||||
|
||||
// Owner always has permission
|
||||
if principal == owner {
|
||||
return true
|
||||
}
|
||||
|
||||
if hasIdentityPermission(operation, ctx) {
|
||||
return true
|
||||
}
|
||||
|
||||
// If no policy is provided, use default allow if enabled
|
||||
if resourcePolicy == "" {
|
||||
if ctx != nil && ctx.DefaultAllow {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Normalize operation to full IAM-style action name (e.g., "s3tables:CreateTableBucket")
|
||||
// if not already prefixed
|
||||
fullAction := operation
|
||||
if !strings.Contains(operation, ":") {
|
||||
fullAction = "s3tables:" + operation
|
||||
}
|
||||
|
||||
// Parse and evaluate policy
|
||||
var policy PolicyDocument
|
||||
if err := json.Unmarshal([]byte(resourcePolicy), &policy); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// Evaluate policy statements
|
||||
// Default is deny, so we need an explicit allow
|
||||
hasAllow := false
|
||||
|
||||
for _, stmt := range policy.Statement {
|
||||
// Check if principal matches
|
||||
if !matchesPrincipal(stmt.Principal, principal) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if action matches (using normalized full action name)
|
||||
if !matchesAction(stmt.Action, fullAction) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if resource matches (if resourceARN specified and Resource field exists)
|
||||
if resourceARN != "" && !matchesResource(stmt.Resource, resourceARN) {
|
||||
continue
|
||||
}
|
||||
|
||||
if !matchesConditions(stmt.Condition, ctx) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Statement matches - check effect
|
||||
switch stmt.Effect {
|
||||
case "Allow":
|
||||
hasAllow = true
|
||||
case "Deny":
|
||||
// Explicit deny always wins
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if hasAllow {
|
||||
return true
|
||||
}
|
||||
|
||||
// If no statement matched, use default allow if enabled
|
||||
if ctx != nil && ctx.DefaultAllow {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func hasIdentityPermission(operation string, ctx *PolicyContext) bool {
|
||||
if ctx == nil || len(ctx.IdentityActions) == 0 {
|
||||
return false
|
||||
}
|
||||
fullAction := operation
|
||||
if !strings.Contains(operation, ":") {
|
||||
fullAction = "s3tables:" + operation
|
||||
}
|
||||
if hasAdminAction(ctx.IdentityActions) {
|
||||
return true
|
||||
}
|
||||
candidates := []string{operation, fullAction}
|
||||
if ctx.TableBucketName != "" {
|
||||
candidates = append(candidates, operation+":"+ctx.TableBucketName, fullAction+":"+ctx.TableBucketName)
|
||||
}
|
||||
for _, action := range ctx.IdentityActions {
|
||||
for _, candidate := range candidates {
|
||||
if action == candidate {
|
||||
return true
|
||||
}
|
||||
if strings.ContainsAny(action, "*?") && wildcard.MatchesWildcard(action, candidate) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// hasAdminAction reports whether the action list grants blanket admin access.
|
||||
// Legacy static identities may use broad markers or s3 wildcards; s3:* is treated
|
||||
// as s3tables:* so shared admin policies still permit table access.
|
||||
func hasAdminAction(actions []string) bool {
|
||||
for _, action := range actions {
|
||||
if action == "*" || action == s3_constants.ACTION_ADMIN || action == "s3:*" || action == "s3tables:*" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// matchesPrincipal checks if the principal matches the statement's principal
|
||||
func matchesPrincipal(principalSpec interface{}, principal string) bool {
|
||||
if principalSpec == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
switch p := principalSpec.(type) {
|
||||
case string:
|
||||
// Direct string match or wildcard
|
||||
if p == "*" || p == principal {
|
||||
return true
|
||||
}
|
||||
// Support wildcard matching for principals (e.g., "arn:aws:iam::*:user/admin")
|
||||
return wildcard.MatchesWildcard(p, principal)
|
||||
case []interface{}:
|
||||
// Array of principals
|
||||
for _, item := range p {
|
||||
if str, ok := item.(string); ok {
|
||||
if str == "*" || str == principal {
|
||||
return true
|
||||
}
|
||||
// Support wildcard matching
|
||||
if wildcard.MatchesWildcard(str, principal) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
case map[string]interface{}:
|
||||
// AWS-style principal with service prefix, e.g., {"AWS": "arn:aws:iam::..."}
|
||||
// For S3 Tables, we primarily care about the AWS key
|
||||
if aws, ok := p["AWS"]; ok {
|
||||
return matchesPrincipal(aws, principal)
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// matchesAction checks if the action matches the statement's action
|
||||
func matchesAction(actionSpec interface{}, action string) bool {
|
||||
if actionSpec == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
switch a := actionSpec.(type) {
|
||||
case string:
|
||||
// Direct match or wildcard
|
||||
return matchesActionPattern(a, action)
|
||||
case []interface{}:
|
||||
// Array of actions
|
||||
for _, item := range a {
|
||||
if str, ok := item.(string); ok {
|
||||
if matchesActionPattern(str, action) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// matchesActionPattern checks if an action matches a pattern (supports wildcards)
|
||||
// This uses the policy_engine.MatchesWildcard function for full wildcard support,
|
||||
// including middle wildcards (e.g., "s3tables:Get*Table") for complete IAM compatibility.
|
||||
func matchesActionPattern(pattern, action string) bool {
|
||||
if pattern == "*" {
|
||||
return true
|
||||
}
|
||||
|
||||
// Exact match
|
||||
if pattern == action {
|
||||
return true
|
||||
}
|
||||
|
||||
// Wildcard match using policy engine's wildcard matcher
|
||||
// Supports both * (any sequence) and ? (single character) anywhere in the pattern
|
||||
return wildcard.MatchesWildcard(pattern, action)
|
||||
}
|
||||
|
||||
func matchesConditions(conditions map[string]map[string]interface{}, ctx *PolicyContext) bool {
|
||||
if len(conditions) == 0 {
|
||||
return true
|
||||
}
|
||||
if ctx == nil {
|
||||
return false
|
||||
}
|
||||
for operator, conditionValues := range conditions {
|
||||
if !matchesConditionOperator(operator, conditionValues, ctx) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func matchesConditionOperator(operator string, conditionValues map[string]interface{}, ctx *PolicyContext) bool {
|
||||
evaluator, err := policy_engine.GetConditionEvaluator(operator)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
for key, value := range conditionValues {
|
||||
contextVals := getConditionContextValues(key, ctx)
|
||||
if !evaluator.Evaluate(value, contextVals) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func getConditionContextValues(key string, ctx *PolicyContext) []string {
|
||||
switch key {
|
||||
case "s3tables:namespace":
|
||||
return []string{ctx.Namespace}
|
||||
case "s3tables:tableName":
|
||||
return []string{ctx.TableName}
|
||||
case "s3tables:tableBucketName":
|
||||
return []string{ctx.TableBucketName}
|
||||
case "s3tables:SSEAlgorithm":
|
||||
return []string{ctx.SSEAlgorithm}
|
||||
case "s3tables:KMSKeyArn":
|
||||
return []string{ctx.KMSKeyArn}
|
||||
case "s3tables:StorageClass":
|
||||
return []string{ctx.StorageClass}
|
||||
case "aws:TagKeys":
|
||||
return ctx.TagKeys
|
||||
}
|
||||
if strings.HasPrefix(key, "aws:RequestTag/") {
|
||||
tagKey := strings.TrimPrefix(key, "aws:RequestTag/")
|
||||
if val, ok := ctx.RequestTags[tagKey]; ok {
|
||||
return []string{val}
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(key, "aws:ResourceTag/") {
|
||||
tagKey := strings.TrimPrefix(key, "aws:ResourceTag/")
|
||||
if val, ok := ctx.ResourceTags[tagKey]; ok {
|
||||
return []string{val}
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(key, "s3tables:TableBucketTag/") {
|
||||
tagKey := strings.TrimPrefix(key, "s3tables:TableBucketTag/")
|
||||
if val, ok := ctx.TableBucketTags[tagKey]; ok {
|
||||
return []string{val}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// matchesResource checks if the resource ARN matches the statement's resource specification
|
||||
// Returns true if resource matches or if Resource is not specified (implicit match)
|
||||
func matchesResource(resourceSpec interface{}, resourceARN string) bool {
|
||||
// If no Resource is specified, match all resources (implicit *)
|
||||
if resourceSpec == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
switch r := resourceSpec.(type) {
|
||||
case string:
|
||||
// Direct match or wildcard
|
||||
return matchesResourcePattern(r, resourceARN)
|
||||
case []interface{}:
|
||||
// Array of resources - match if any matches
|
||||
for _, item := range r {
|
||||
if str, ok := item.(string); ok {
|
||||
if matchesResourcePattern(str, resourceARN) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// matchesResourcePattern checks if a resource ARN matches a pattern (supports wildcards)
|
||||
func matchesResourcePattern(pattern, resourceARN string) bool {
|
||||
if pattern == "*" {
|
||||
return true
|
||||
}
|
||||
|
||||
// Exact match
|
||||
if pattern == resourceARN {
|
||||
return true
|
||||
}
|
||||
|
||||
// Wildcard match using policy engine's wildcard matcher
|
||||
return wildcard.MatchesWildcard(pattern, resourceARN)
|
||||
}
|
||||
|
||||
// AuthError represents an authorization error
|
||||
type AuthError struct {
|
||||
Operation string
|
||||
Principal string
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *AuthError) Error() string {
|
||||
return "unauthorized: " + e.Principal + " is not permitted to perform " + e.Operation + ": " + e.Message
|
||||
}
|
||||
|
||||
// NewAuthError creates a new authorization error
|
||||
func NewAuthError(operation, principal, message string) *AuthError {
|
||||
return &AuthError{
|
||||
Operation: operation,
|
||||
Principal: principal,
|
||||
Message: message,
|
||||
}
|
||||
}
|
||||
253
weed/s3api/s3tables/permissions_test.go
Normal file
253
weed/s3api/s3tables/permissions_test.go
Normal file
@@ -0,0 +1,253 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMatchesActionPattern(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pattern string
|
||||
action string
|
||||
expected bool
|
||||
}{
|
||||
// Exact matches
|
||||
{"exact match", "GetTable", "GetTable", true},
|
||||
{"no match", "GetTable", "DeleteTable", false},
|
||||
|
||||
// Universal wildcard
|
||||
{"universal wildcard", "*", "anything", true},
|
||||
|
||||
// Suffix wildcards
|
||||
{"suffix wildcard match", "s3tables:*", "s3tables:GetTable", true},
|
||||
{"suffix wildcard no match", "s3tables:*", "iam:GetUser", false},
|
||||
|
||||
// Middle wildcards (new capability from policy_engine)
|
||||
{"middle wildcard Get*Table", "s3tables:Get*Table", "s3tables:GetTable", true},
|
||||
{"middle wildcard Get*Table no match GetTableBucket", "s3tables:Get*Table", "s3tables:GetTableBucket", false},
|
||||
{"middle wildcard Get*Table no match DeleteTable", "s3tables:Get*Table", "s3tables:DeleteTable", false},
|
||||
{"middle wildcard *Table*", "s3tables:*Table*", "s3tables:GetTableBucket", true},
|
||||
{"middle wildcard *Table* match CreateTable", "s3tables:*Table*", "s3tables:CreateTable", true},
|
||||
|
||||
// Question mark wildcards
|
||||
{"question mark single char", "GetTable?", "GetTableX", true},
|
||||
{"question mark no match", "GetTable?", "GetTableXY", false},
|
||||
|
||||
// Combined wildcards
|
||||
{"combined * and ? singular", "s3tables:Get?able*", "s3tables:GetTable", true},
|
||||
{"combined * and ? plural", "s3tables:Get?able*", "s3tables:GetTables", true},
|
||||
{"combined no match - ? needs 1 char", "s3tables:Get?able*", "s3tables:Getable", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := matchesActionPattern(tt.pattern, tt.action)
|
||||
if result != tt.expected {
|
||||
t.Errorf("matchesActionPattern(%q, %q) = %v, want %v", tt.pattern, tt.action, result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesPrincipal(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
principalSpec interface{}
|
||||
principal string
|
||||
expected bool
|
||||
}{
|
||||
// String principals
|
||||
{"exact match", "user123", "user123", true},
|
||||
{"no match", "user123", "user456", false},
|
||||
{"universal wildcard", "*", "anyone", true},
|
||||
|
||||
// Wildcard principals
|
||||
{"prefix wildcard", "arn:aws:iam::123456789012:user/*", "arn:aws:iam::123456789012:user/admin", true},
|
||||
{"prefix wildcard no match", "arn:aws:iam::123456789012:user/*", "arn:aws:iam::987654321098:user/admin", false},
|
||||
{"middle wildcard", "arn:aws:iam::*:user/admin", "arn:aws:iam::123456789012:user/admin", true},
|
||||
|
||||
// Array of principals
|
||||
{"array match first", []interface{}{"user1", "user2"}, "user1", true},
|
||||
{"array match second", []interface{}{"user1", "user2"}, "user2", true},
|
||||
{"array no match", []interface{}{"user1", "user2"}, "user3", false},
|
||||
{"array wildcard", []interface{}{"user1", "arn:aws:iam::*:user/admin"}, "arn:aws:iam::123:user/admin", true},
|
||||
|
||||
// Map-style AWS principals
|
||||
{"AWS map exact", map[string]interface{}{"AWS": "user123"}, "user123", true},
|
||||
{"AWS map wildcard", map[string]interface{}{"AWS": "arn:aws:iam::*:user/admin"}, "arn:aws:iam::123:user/admin", true},
|
||||
{"AWS map array", map[string]interface{}{"AWS": []interface{}{"user1", "user2"}}, "user1", true},
|
||||
|
||||
// Nil/empty cases
|
||||
{"nil principal", nil, "user123", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := matchesPrincipal(tt.principalSpec, tt.principal)
|
||||
if result != tt.expected {
|
||||
t.Errorf("matchesPrincipal(%v, %q) = %v, want %v", tt.principalSpec, tt.principal, result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluatePolicyWithConditions(t *testing.T) {
|
||||
policy := &PolicyDocument{
|
||||
Statement: []Statement{
|
||||
{
|
||||
Effect: "Allow",
|
||||
Principal: "*",
|
||||
Action: "s3tables:GetTable",
|
||||
Condition: map[string]map[string]interface{}{
|
||||
"StringEquals": {
|
||||
"s3tables:namespace": "default",
|
||||
},
|
||||
"StringLike": {
|
||||
"s3tables:tableName": "test_*",
|
||||
},
|
||||
"NumericGreaterThan": {
|
||||
"aws:RequestTag/priority": "10",
|
||||
},
|
||||
"Bool": {
|
||||
"aws:ResourceTag/is_public": "true",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
policyBytes, _ := json.Marshal(policy)
|
||||
policyStr := string(policyBytes)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
ctx *PolicyContext
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
"all conditions match",
|
||||
&PolicyContext{
|
||||
Namespace: "default",
|
||||
TableName: "test_table",
|
||||
RequestTags: map[string]string{
|
||||
"priority": "15",
|
||||
},
|
||||
ResourceTags: map[string]string{
|
||||
"is_public": "true",
|
||||
},
|
||||
},
|
||||
true,
|
||||
},
|
||||
{
|
||||
"namespace mismatch",
|
||||
&PolicyContext{
|
||||
Namespace: "other",
|
||||
TableName: "test_table",
|
||||
RequestTags: map[string]string{
|
||||
"priority": "15",
|
||||
},
|
||||
ResourceTags: map[string]string{
|
||||
"is_public": "true",
|
||||
},
|
||||
},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"table name mismatch",
|
||||
&PolicyContext{
|
||||
Namespace: "default",
|
||||
TableName: "other_table",
|
||||
RequestTags: map[string]string{
|
||||
"priority": "15",
|
||||
},
|
||||
ResourceTags: map[string]string{
|
||||
"is_public": "true",
|
||||
},
|
||||
},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"numeric condition failure",
|
||||
&PolicyContext{
|
||||
Namespace: "default",
|
||||
TableName: "test_table",
|
||||
RequestTags: map[string]string{
|
||||
"priority": "5",
|
||||
},
|
||||
ResourceTags: map[string]string{
|
||||
"is_public": "true",
|
||||
},
|
||||
},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"bool condition failure",
|
||||
&PolicyContext{
|
||||
Namespace: "default",
|
||||
TableName: "test_table",
|
||||
RequestTags: map[string]string{
|
||||
"priority": "15",
|
||||
},
|
||||
ResourceTags: map[string]string{
|
||||
"is_public": "false",
|
||||
},
|
||||
},
|
||||
false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// principal="user123", owner="owner123"
|
||||
result := CheckPermissionWithContext("s3tables:GetTable", "user123", "owner123", policyStr, "", tt.ctx)
|
||||
if result != tt.expected {
|
||||
t.Errorf("CheckPermissionWithContext() = %v, want %v", result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestCheckPermissionWithDefaultAllow(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
defaultAllow bool
|
||||
policy string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
"default deny (no policy, DefaultAllow=false)",
|
||||
false,
|
||||
"",
|
||||
false,
|
||||
},
|
||||
{
|
||||
"default allow (no policy, DefaultAllow=true)",
|
||||
true,
|
||||
"",
|
||||
true,
|
||||
},
|
||||
{
|
||||
"explicit deny overrides DefaultAllow=true",
|
||||
true,
|
||||
`{"Statement":[{"Effect":"Deny","Principal":"*","Action":"s3tables:GetTable"}]}`,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"explicit allow works with DefaultAllow=false",
|
||||
false,
|
||||
`{"Statement":[{"Effect":"Allow","Principal":"*","Action":"s3tables:GetTable"}]}`,
|
||||
true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctx := &PolicyContext{
|
||||
DefaultAllow: tt.defaultAllow,
|
||||
}
|
||||
result := CheckPermissionWithContext("s3tables:GetTable", "user123", "owner123", tt.policy, "", ctx)
|
||||
if result != tt.expected {
|
||||
t.Errorf("CheckPermissionWithContext() = %v, want %v (DefaultAllow=%v, Policy=%s)", result, tt.expected, tt.defaultAllow, tt.policy)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
150
weed/s3api/s3tables/s3tablestest/foreign_manifest.go
Normal file
150
weed/s3api/s3tables/s3tablestest/foreign_manifest.go
Normal file
@@ -0,0 +1,150 @@
|
||||
package s3tablestest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/apache/iceberg-go"
|
||||
"github.com/twmb/avro"
|
||||
"github.com/twmb/avro/ocf"
|
||||
)
|
||||
|
||||
// ForeignPartitionManifest rewrites a manifest iceberg-go wrote so its
|
||||
// partition field carries the Avro union a foreign writer emits: [<type>, null]
|
||||
// rather than [null, <type>]. Both are valid Avro, but iceberg-go reads a
|
||||
// partition field's logical type from the union's last branch, so the ordering
|
||||
// decides whether it hands back an Iceberg value or the rich Go value the Avro
|
||||
// decoder produced.
|
||||
//
|
||||
// logicalType is stamped onto the value branch, since iceberg-go writes a day
|
||||
// partition as a bare int, and decoded replaces the partition value in the
|
||||
// record, standing in for what the foreign writer encoded. valueFirst orders
|
||||
// the union [<type>, null] rather than the [null, <type>] Java and iceberg-rust
|
||||
// emit.
|
||||
func ForeignPartitionManifest(
|
||||
t *testing.T,
|
||||
schema *iceberg.Schema,
|
||||
spec iceberg.PartitionSpec,
|
||||
entry iceberg.ManifestEntry,
|
||||
manifestPath, logicalType string,
|
||||
decoded any,
|
||||
valueFirst bool,
|
||||
) ([]byte, iceberg.ManifestFile) {
|
||||
t.Helper()
|
||||
|
||||
var original bytes.Buffer
|
||||
manifest, err := iceberg.WriteManifest(manifestPath, &original, 2, spec, schema, 1, []iceberg.ManifestEntry{entry})
|
||||
if err != nil {
|
||||
t.Fatalf("write base manifest: %v", err)
|
||||
}
|
||||
|
||||
reader, err := ocf.NewReader(bytes.NewReader(original.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("open base manifest: %v", err)
|
||||
}
|
||||
metadata := reader.Metadata()
|
||||
var record map[string]any
|
||||
if err := reader.Decode(&record); err != nil {
|
||||
t.Fatalf("decode base manifest: %v", err)
|
||||
}
|
||||
if err := reader.Close(); err != nil {
|
||||
t.Fatalf("close base manifest: %v", err)
|
||||
}
|
||||
|
||||
fieldName := spec.Field(0).Name
|
||||
|
||||
var schemaDoc map[string]any
|
||||
if err := json.Unmarshal(metadata["avro.schema"], &schemaDoc); err != nil {
|
||||
t.Fatalf("decode manifest schema: %v", err)
|
||||
}
|
||||
dataFile := findAvroField(t, schemaDoc, "data_file")
|
||||
partition := findAvroField(t, dataFile, "partition")
|
||||
partitionFields, ok := partition["fields"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("partition schema fields have type %T", partition["fields"])
|
||||
}
|
||||
for _, rawField := range partitionFields {
|
||||
field, ok := rawField.(map[string]any)
|
||||
if !ok || field["name"] != fieldName {
|
||||
continue
|
||||
}
|
||||
fieldType, ok := field["type"].([]any)
|
||||
if !ok || len(fieldType) != 2 {
|
||||
t.Fatalf("%s schema type = %#v, want nullable union", fieldName, field["type"])
|
||||
}
|
||||
valueType, ok := fieldType[1].(map[string]any)
|
||||
if !ok {
|
||||
primitive, primitiveOK := fieldType[1].(string)
|
||||
if !primitiveOK {
|
||||
t.Fatalf("%s value branch = %T, want Avro type", fieldName, fieldType[1])
|
||||
}
|
||||
valueType = map[string]any{"type": primitive}
|
||||
}
|
||||
valueType["logicalType"] = logicalType
|
||||
if valueFirst {
|
||||
field["type"] = []any{valueType, "null"}
|
||||
} else {
|
||||
field["type"] = []any{"null", valueType}
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
dataFileRecord, ok := record["data_file"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("decoded data_file = %T, want record", record["data_file"])
|
||||
}
|
||||
partitionRecord, ok := dataFileRecord["partition"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("decoded partition = %T, want record", dataFileRecord["partition"])
|
||||
}
|
||||
partitionRecord[fieldName] = decoded
|
||||
|
||||
foreignSchemaJSON, err := json.Marshal(schemaDoc)
|
||||
if err != nil {
|
||||
t.Fatalf("encode foreign manifest schema: %v", err)
|
||||
}
|
||||
foreignSchema, err := avro.Parse(string(foreignSchemaJSON))
|
||||
if err != nil {
|
||||
t.Fatalf("parse foreign manifest schema: %v", err)
|
||||
}
|
||||
userMetadata := make(map[string][]byte)
|
||||
for key, value := range metadata {
|
||||
if key != "avro.schema" && key != "avro.codec" {
|
||||
userMetadata[key] = value
|
||||
}
|
||||
}
|
||||
var foreign bytes.Buffer
|
||||
writer, err := ocf.NewWriter(&foreign, foreignSchema, ocf.WithSchema(string(foreignSchemaJSON)), ocf.WithMetadata(userMetadata))
|
||||
if err != nil {
|
||||
t.Fatalf("create foreign manifest writer: %v", err)
|
||||
}
|
||||
if err := writer.Encode(record); err != nil {
|
||||
t.Fatalf("encode foreign manifest: %v", err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("close foreign manifest writer: %v", err)
|
||||
}
|
||||
|
||||
return foreign.Bytes(), iceberg.NewManifestFile(2, manifest.FilePath(), int64(foreign.Len()), manifest.PartitionSpecID(), 1).AddedFiles(1).SequenceNum(1, 1).Build()
|
||||
}
|
||||
|
||||
func findAvroField(t *testing.T, record map[string]any, name string) map[string]any {
|
||||
t.Helper()
|
||||
fields, ok := record["fields"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("Avro record fields have type %T", record["fields"])
|
||||
}
|
||||
for _, rawField := range fields {
|
||||
field, ok := rawField.(map[string]any)
|
||||
if ok && field["name"] == name {
|
||||
fieldType, ok := field["type"].(map[string]any)
|
||||
if ok {
|
||||
return fieldType
|
||||
}
|
||||
t.Fatalf("Avro field %q type has type %T", name, field["type"])
|
||||
}
|
||||
}
|
||||
t.Fatalf("Avro field %q not found", name)
|
||||
return nil
|
||||
}
|
||||
242
weed/s3api/s3tables/s3tablestest/memfiler.go
Normal file
242
weed/s3api/s3tables/s3tablestest/memfiler.go
Normal file
@@ -0,0 +1,242 @@
|
||||
// Package s3tablestest provides an in-memory filer for driving S3 Tables and
|
||||
// Lance namespace operations end-to-end without a live cluster.
|
||||
package s3tablestest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// MemFiler is an in-memory filer used to drive Manager operations
|
||||
// end-to-end without a live cluster.
|
||||
type MemFiler struct {
|
||||
filer_pb.UnimplementedSeaweedFilerServer
|
||||
// mu guards entries. The real filer serves concurrent RPCs, and a test that
|
||||
// races writers against each other - the point of an exclusive create - hits
|
||||
// this map from several goroutines at once.
|
||||
mu sync.RWMutex
|
||||
entries map[string]map[string]*filer_pb.Entry // dir -> name -> entry
|
||||
Client filer_pb.SeaweedFilerClient
|
||||
// BeforeUpdate runs once, at the start of the next UpdateEntry, so a test
|
||||
// can land a competing write in a handler's read-to-write window.
|
||||
BeforeUpdate func()
|
||||
// RejectDelete returns the reason DeleteEntry refuses a path, answered the
|
||||
// way the filer answers one: no transport error, the reason in the response.
|
||||
RejectDelete func(dir, name string) string
|
||||
}
|
||||
|
||||
func newMemFiler() *MemFiler {
|
||||
return &MemFiler{entries: make(map[string]map[string]*filer_pb.Entry)}
|
||||
}
|
||||
|
||||
func (f *MemFiler) Get(dir, name string) *filer_pb.Entry {
|
||||
f.mu.RLock()
|
||||
defer f.mu.RUnlock()
|
||||
if d, ok := f.entries[dir]; ok {
|
||||
return d[name]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *MemFiler) Put(dir, name string, extended map[string][]byte) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if _, ok := f.entries[dir]; !ok {
|
||||
f.entries[dir] = make(map[string]*filer_pb.Entry)
|
||||
}
|
||||
f.entries[dir][name] = &filer_pb.Entry{Name: name, IsDirectory: true, Extended: extended}
|
||||
}
|
||||
|
||||
// PutFile adds a file entry with an explicit modification time, which callers
|
||||
// that age entries out (orphan cleanup, expiry) need in order to see them.
|
||||
func (f *MemFiler) PutFile(dir, name string, mtime time.Time) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if _, ok := f.entries[dir]; !ok {
|
||||
f.entries[dir] = make(map[string]*filer_pb.Entry)
|
||||
}
|
||||
f.entries[dir][name] = &filer_pb.Entry{
|
||||
Name: name,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: mtime.Unix(), Crtime: mtime.Unix()},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *MemFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
||||
if e := f.Get(req.Directory, req.Name); e != nil {
|
||||
return &filer_pb.LookupDirectoryEntryResponse{Entry: e}, nil
|
||||
}
|
||||
// Carry the sentinel text so filer_pb.LookupEntry maps it to ErrNotFound.
|
||||
return nil, status.Errorf(codes.NotFound, "%s: %s/%s", filer_pb.ErrNotFound.Error(), req.Directory, req.Name)
|
||||
}
|
||||
|
||||
// ListEntries honours prefix, start-from and limit the way the real filer does.
|
||||
// A harness that ignores them makes a paginating caller re-read the first page
|
||||
// forever, which looks like duplicated entries rather than a broken listing.
|
||||
func (f *MemFiler) ListEntries(req *filer_pb.ListEntriesRequest, stream grpc.ServerStreamingServer[filer_pb.ListEntriesResponse]) error {
|
||||
f.mu.RLock()
|
||||
d, ok := f.entries[req.Directory]
|
||||
names := make([]string, 0, len(d))
|
||||
snapshot := make(map[string]*filer_pb.Entry, len(d))
|
||||
for name, entry := range d {
|
||||
names = append(names, name)
|
||||
snapshot[name] = entry
|
||||
}
|
||||
f.mu.RUnlock()
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
sent := uint32(0)
|
||||
for _, name := range names {
|
||||
if req.Prefix != "" && !strings.HasPrefix(name, req.Prefix) {
|
||||
continue
|
||||
}
|
||||
if req.StartFromFileName != "" {
|
||||
if name < req.StartFromFileName {
|
||||
continue
|
||||
}
|
||||
if name == req.StartFromFileName && !req.InclusiveStartFrom {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := stream.Send(&filer_pb.ListEntriesResponse{Entry: snapshot[name]}); err != nil {
|
||||
return err
|
||||
}
|
||||
sent++
|
||||
if req.Limit > 0 && sent >= req.Limit {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *MemFiler) CreateEntry(_ context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if _, ok := f.entries[req.Directory]; !ok {
|
||||
f.entries[req.Directory] = make(map[string]*filer_pb.Entry)
|
||||
}
|
||||
// O_EXCL is the filer's put-if-not-exists. Ignoring it here would let a test
|
||||
// that races two writers see both of them win.
|
||||
if _, exists := f.entries[req.Directory][req.Entry.Name]; exists && req.OExcl {
|
||||
return &filer_pb.CreateEntryResponse{ErrorCode: filer_pb.FilerError_ENTRY_ALREADY_EXISTS}, nil
|
||||
}
|
||||
f.entries[req.Directory][req.Entry.Name] = req.Entry
|
||||
return &filer_pb.CreateEntryResponse{}, nil
|
||||
}
|
||||
|
||||
func (f *MemFiler) UpdateEntry(_ context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
|
||||
// The hook runs before the lock is taken: its whole purpose is to land a
|
||||
// competing write in the read-to-write window, and that write needs the lock.
|
||||
if hook := f.BeforeUpdate; hook != nil {
|
||||
f.BeforeUpdate = nil
|
||||
hook()
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
// The real filer validates ExpectedExtended under the per-path lock; without
|
||||
// it here a lost update would look like a success.
|
||||
for key, expected := range req.ExpectedExtended {
|
||||
var actual []byte
|
||||
if d, ok := f.entries[req.Directory]; ok {
|
||||
if existing := d[req.Entry.Name]; existing != nil {
|
||||
actual = existing.Extended[key]
|
||||
}
|
||||
}
|
||||
if !bytes.Equal(actual, expected) {
|
||||
return nil, status.Errorf(codes.FailedPrecondition, "extended attribute %q changed", key)
|
||||
}
|
||||
}
|
||||
if _, ok := f.entries[req.Directory]; !ok {
|
||||
f.entries[req.Directory] = make(map[string]*filer_pb.Entry)
|
||||
}
|
||||
f.entries[req.Directory][req.Entry.Name] = req.Entry
|
||||
return &filer_pb.UpdateEntryResponse{}, nil
|
||||
}
|
||||
|
||||
func (f *MemFiler) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) {
|
||||
if reject := f.RejectDelete; reject != nil {
|
||||
if reason := reject(req.Directory, req.Name); reason != "" {
|
||||
return &filer_pb.DeleteEntryResponse{Error: reason}, nil
|
||||
}
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if d, ok := f.entries[req.Directory]; ok {
|
||||
delete(d, req.Name)
|
||||
}
|
||||
// Honor recursive data deletion so a regression that wipes the table directory
|
||||
// also drops its metadata/ and data/ children (the data-loss this guards against).
|
||||
if req.IsRecursive && req.IsDeleteData {
|
||||
child := path.Join(req.Directory, req.Name)
|
||||
for dir := range f.entries {
|
||||
if dir == child || strings.HasPrefix(dir, child+"/") {
|
||||
delete(f.entries, dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
return &filer_pb.DeleteEntryResponse{}, nil
|
||||
}
|
||||
|
||||
// GetFilerConfiguration answers with the defaults, so operations that resolve
|
||||
// the buckets directory before touching an entry work against this filer.
|
||||
func (f *MemFiler) GetFilerConfiguration(_ context.Context, _ *filer_pb.GetFilerConfigurationRequest) (*filer_pb.GetFilerConfigurationResponse, error) {
|
||||
return &filer_pb.GetFilerConfigurationResponse{DirBuckets: s3_constants.DefaultBucketsPath}, nil
|
||||
}
|
||||
|
||||
func (f *MemFiler) Ping(_ context.Context, _ *filer_pb.PingRequest) (*filer_pb.PingResponse, error) {
|
||||
now := time.Now().UnixNano()
|
||||
return &filer_pb.PingResponse{StartTimeNs: now, RemoteTimeNs: now, StopTimeNs: now}, nil
|
||||
}
|
||||
|
||||
func Start(t *testing.T) *MemFiler {
|
||||
t.Helper()
|
||||
fs := newMemFiler()
|
||||
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("start filer: %v", err)
|
||||
}
|
||||
|
||||
server := grpc.NewServer()
|
||||
filer_pb.RegisterSeaweedFilerServer(server, fs)
|
||||
go func() { _ = server.Serve(listener) }()
|
||||
t.Cleanup(server.GracefulStop)
|
||||
|
||||
conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
t.Fatalf("start filer: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
|
||||
fs.Client = filer_pb.NewSeaweedFilerClient(conn)
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
pingCtx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
_, err := fs.Client.Ping(pingCtx, &filer_pb.PingRequest{})
|
||||
cancel()
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("filer not ready: %v", err)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
return fs
|
||||
}
|
||||
37
weed/s3api/s3tables/table_data_dir_test.go
Normal file
37
weed/s3api/s3tables/table_data_dir_test.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"path"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTableDataDirFromMetadataLocation(t *testing.T) {
|
||||
cases := []struct {
|
||||
loc string
|
||||
want string
|
||||
}{
|
||||
{"s3://warehouse/sales/orders/metadata/v1.metadata.json", path.Join(TablesPath, "warehouse/sales/orders")},
|
||||
{"s3://warehouse/sales/orders/metadata/00003-9f1c.metadata.json", path.Join(TablesPath, "warehouse/sales/orders")},
|
||||
{"s3://warehouse/ns/tbl", path.Join(TablesPath, "warehouse/ns/tbl")},
|
||||
{"", ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := TableDataDirFromMetadataLocation(c.loc); got != c.want {
|
||||
t.Errorf("TableDataDirFromMetadataLocation(%q) = %q, want %q", c.loc, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataVersionFromLocationUniqueSuffix(t *testing.T) {
|
||||
cases := map[string]int{
|
||||
"s3://bkt/ns/t/metadata/v4.metadata.json": 4,
|
||||
"s3://bkt/ns/t/metadata/v4-0a1b2c3d-4e5f-6789-abcd-ef0123456789.metadata.json": 4,
|
||||
"s3://bkt/ns/t/metadata/00007-0a1b2c3d.metadata.json": 7,
|
||||
"s3://bkt/ns/t/metadata/whatever.metadata.json": 1,
|
||||
}
|
||||
for location, want := range cases {
|
||||
if got := metadataVersionFromLocation(location); got != want {
|
||||
t.Errorf("metadataVersionFromLocation(%q) = %d, want %d", location, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
626
weed/s3api/s3tables/types.go
Normal file
626
weed/s3api/s3tables/types.go
Normal file
@@ -0,0 +1,626 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Table bucket types
|
||||
|
||||
type TableBucket struct {
|
||||
ARN string `json:"arn"`
|
||||
Name string `json:"name"`
|
||||
OwnerAccountID string `json:"ownerAccountId"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
Format string `json:"format,omitempty"`
|
||||
}
|
||||
|
||||
type CreateTableBucketRequest struct {
|
||||
Name string `json:"name"`
|
||||
Tags map[string]string `json:"tags,omitempty"`
|
||||
// Format is the table format this bucket holds. A bucket is a catalog and a
|
||||
// catalog serves one protocol, so declaring it here is what lets a caller be
|
||||
// told where to connect. Empty means ICEBERG, which is what AWS S3 Tables
|
||||
// serves and therefore what an SDK that has never heard of this field means.
|
||||
Format string `json:"format,omitempty"`
|
||||
}
|
||||
|
||||
type CreateTableBucketResponse struct {
|
||||
ARN string `json:"arn"`
|
||||
}
|
||||
|
||||
type GetTableBucketRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
}
|
||||
|
||||
type GetTableBucketResponse struct {
|
||||
ARN string `json:"arn"`
|
||||
Name string `json:"name"`
|
||||
OwnerAccountID string `json:"ownerAccountId"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
// Format is empty for a bucket created before formats were declared. Such a
|
||||
// bucket accepts any format, which is what it did when it was made.
|
||||
Format string `json:"format,omitempty"`
|
||||
}
|
||||
|
||||
type ListTableBucketsRequest struct {
|
||||
Prefix string `json:"prefix,omitempty"`
|
||||
ContinuationToken string `json:"continuationToken,omitempty"`
|
||||
MaxBuckets int `json:"maxBuckets,omitempty"`
|
||||
}
|
||||
|
||||
type TableBucketSummary struct {
|
||||
ARN string `json:"arn"`
|
||||
Name string `json:"name"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
Format string `json:"format,omitempty"`
|
||||
}
|
||||
|
||||
type ListTableBucketsResponse struct {
|
||||
TableBuckets []TableBucketSummary `json:"tableBuckets"`
|
||||
ContinuationToken string `json:"continuationToken,omitempty"`
|
||||
}
|
||||
|
||||
type DeleteTableBucketRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
}
|
||||
|
||||
// Table bucket policy types
|
||||
|
||||
type PutTableBucketPolicyRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
ResourcePolicy string `json:"resourcePolicy"`
|
||||
}
|
||||
|
||||
type GetTableBucketPolicyRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
}
|
||||
|
||||
type GetTableBucketPolicyResponse struct {
|
||||
ResourcePolicy string `json:"resourcePolicy"`
|
||||
}
|
||||
|
||||
type DeleteTableBucketPolicyRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
}
|
||||
|
||||
// Namespace types
|
||||
|
||||
type Namespace struct {
|
||||
Namespace []string `json:"namespace"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
OwnerAccountID string `json:"ownerAccountId"`
|
||||
Properties map[string]string `json:"properties,omitempty"`
|
||||
}
|
||||
|
||||
type CreateNamespaceRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Properties map[string]string `json:"properties,omitempty"`
|
||||
}
|
||||
|
||||
type CreateNamespaceResponse struct {
|
||||
Namespace []string `json:"namespace"`
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Properties map[string]string `json:"properties,omitempty"`
|
||||
}
|
||||
|
||||
type GetNamespaceRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
}
|
||||
|
||||
type UpdateNamespaceRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Properties map[string]string `json:"properties,omitempty"`
|
||||
}
|
||||
|
||||
type UpdateNamespaceResponse struct {
|
||||
Namespace []string `json:"namespace"`
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Properties map[string]string `json:"properties,omitempty"`
|
||||
}
|
||||
|
||||
type GetNamespaceResponse struct {
|
||||
Namespace []string `json:"namespace"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
OwnerAccountID string `json:"ownerAccountId"`
|
||||
Properties map[string]string `json:"properties,omitempty"`
|
||||
}
|
||||
|
||||
type ListNamespacesRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Prefix string `json:"prefix,omitempty"`
|
||||
ContinuationToken string `json:"continuationToken,omitempty"`
|
||||
MaxNamespaces int `json:"maxNamespaces,omitempty"`
|
||||
}
|
||||
|
||||
type NamespaceSummary struct {
|
||||
Namespace []string `json:"namespace"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
type ListNamespacesResponse struct {
|
||||
Namespaces []NamespaceSummary `json:"namespaces"`
|
||||
ContinuationToken string `json:"continuationToken,omitempty"`
|
||||
}
|
||||
|
||||
type DeleteNamespaceRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
}
|
||||
|
||||
// Table types
|
||||
|
||||
type IcebergSchemaField struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Required bool `json:"required,omitempty"`
|
||||
}
|
||||
|
||||
type IcebergSchema struct {
|
||||
Fields []IcebergSchemaField `json:"fields"`
|
||||
}
|
||||
|
||||
type IcebergMetadata struct {
|
||||
Schema IcebergSchema `json:"schema"`
|
||||
TableUUID string `json:"tableUuid,omitempty"`
|
||||
}
|
||||
|
||||
type TableMetadata struct {
|
||||
Iceberg *IcebergMetadata `json:"iceberg,omitempty"`
|
||||
FullMetadata json.RawMessage `json:"fullMetadata,omitempty"`
|
||||
}
|
||||
|
||||
type Table struct {
|
||||
Name string `json:"name"`
|
||||
TableARN string `json:"tableARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Format string `json:"format"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
ModifiedAt time.Time `json:"modifiedAt"`
|
||||
OwnerAccountID string `json:"ownerAccountId"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
Metadata *TableMetadata `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type CreateTableRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
Format string `json:"format"`
|
||||
Metadata *TableMetadata `json:"metadata,omitempty"`
|
||||
MetadataVersion int `json:"metadataVersion,omitempty"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
Tags map[string]string `json:"tags,omitempty"`
|
||||
}
|
||||
|
||||
type CreateTableResponse struct {
|
||||
TableARN string `json:"tableARN"`
|
||||
VersionToken string `json:"versionToken"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
}
|
||||
|
||||
type RegisterTableRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
MetadataLocation string `json:"metadataLocation"`
|
||||
}
|
||||
|
||||
type RegisterTableResponse struct {
|
||||
TableARN string `json:"tableARN"`
|
||||
VersionToken string `json:"versionToken"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
}
|
||||
|
||||
type GetTableRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN,omitempty"`
|
||||
Namespace []string `json:"namespace,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
TableARN string `json:"tableARN,omitempty"`
|
||||
}
|
||||
|
||||
type GetTableResponse struct {
|
||||
Name string `json:"name"`
|
||||
TableARN string `json:"tableARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Format string `json:"format"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
ModifiedAt time.Time `json:"modifiedAt"`
|
||||
OwnerAccountID string `json:"ownerAccountId"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
VersionToken string `json:"versionToken"`
|
||||
MetadataVersion int `json:"metadataVersion"`
|
||||
Metadata *TableMetadata `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type ListTablesRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace,omitempty"`
|
||||
Prefix string `json:"prefix,omitempty"`
|
||||
ContinuationToken string `json:"continuationToken,omitempty"`
|
||||
MaxTables int `json:"maxTables,omitempty"`
|
||||
}
|
||||
|
||||
type TableSummary struct {
|
||||
Name string `json:"name"`
|
||||
TableARN string `json:"tableARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
// Format lets a caller tell an Iceberg table from a catalog-only one without
|
||||
// a GetTable per row. AWS omits it; listing a mixed catalog needs it.
|
||||
Format string `json:"format,omitempty"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
ModifiedAt time.Time `json:"modifiedAt"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
}
|
||||
|
||||
type ListTablesResponse struct {
|
||||
Tables []TableSummary `json:"tables"`
|
||||
ContinuationToken string `json:"continuationToken,omitempty"`
|
||||
}
|
||||
|
||||
type DeleteTableRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
VersionToken string `json:"versionToken,omitempty"`
|
||||
}
|
||||
|
||||
type UpdateTableRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
VersionToken string `json:"versionToken,omitempty"`
|
||||
Metadata *TableMetadata `json:"metadata,omitempty"`
|
||||
MetadataVersion int `json:"metadataVersion,omitempty"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
}
|
||||
|
||||
type UpdateTableResponse struct {
|
||||
TableARN string `json:"tableARN"`
|
||||
VersionToken string `json:"versionToken"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
}
|
||||
|
||||
type RenameTableRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
SourceNamespace []string `json:"sourceNamespace"`
|
||||
SourceName string `json:"sourceName"`
|
||||
DestNamespace []string `json:"destNamespace"`
|
||||
DestName string `json:"destName"`
|
||||
}
|
||||
|
||||
type RenameTableResponse struct {
|
||||
TableARN string `json:"tableARN"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
}
|
||||
|
||||
// View types
|
||||
//
|
||||
// Views are stored exactly like tables (a filer directory carrying a metadata
|
||||
// pointer xattr) but tagged with ExtendedKeyEntryType="view". They reuse
|
||||
// TableMetadata for the metadata pointer.
|
||||
|
||||
type CreateViewRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
Metadata *TableMetadata `json:"metadata,omitempty"`
|
||||
MetadataVersion int `json:"metadataVersion,omitempty"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
}
|
||||
|
||||
type CreateViewResponse struct {
|
||||
ViewARN string `json:"viewARN"`
|
||||
VersionToken string `json:"versionToken"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
}
|
||||
|
||||
type GetViewRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type GetViewResponse struct {
|
||||
Name string `json:"name"`
|
||||
ViewARN string `json:"viewARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
ModifiedAt time.Time `json:"modifiedAt"`
|
||||
OwnerAccountID string `json:"ownerAccountId"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
VersionToken string `json:"versionToken"`
|
||||
MetadataVersion int `json:"metadataVersion"`
|
||||
Metadata *TableMetadata `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type ListViewsRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace,omitempty"`
|
||||
Prefix string `json:"prefix,omitempty"`
|
||||
ContinuationToken string `json:"continuationToken,omitempty"`
|
||||
MaxViews int `json:"maxViews,omitempty"`
|
||||
}
|
||||
|
||||
type ViewSummary struct {
|
||||
Name string `json:"name"`
|
||||
ViewARN string `json:"viewARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
ModifiedAt time.Time `json:"modifiedAt"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
}
|
||||
|
||||
type ListViewsResponse struct {
|
||||
Views []ViewSummary `json:"views"`
|
||||
ContinuationToken string `json:"continuationToken,omitempty"`
|
||||
}
|
||||
|
||||
type UpdateViewRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
VersionToken string `json:"versionToken,omitempty"`
|
||||
Metadata *TableMetadata `json:"metadata,omitempty"`
|
||||
MetadataVersion int `json:"metadataVersion,omitempty"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
}
|
||||
|
||||
type UpdateViewResponse struct {
|
||||
ViewARN string `json:"viewARN"`
|
||||
VersionToken string `json:"versionToken"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
}
|
||||
|
||||
type DeleteViewRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
VersionToken string `json:"versionToken,omitempty"`
|
||||
}
|
||||
|
||||
// Table policy types
|
||||
|
||||
type PutTablePolicyRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
ResourcePolicy string `json:"resourcePolicy"`
|
||||
}
|
||||
|
||||
type GetTablePolicyRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type GetTablePolicyResponse struct {
|
||||
ResourcePolicy string `json:"resourcePolicy"`
|
||||
}
|
||||
|
||||
type DeleteTablePolicyRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// Maintenance configuration types
|
||||
|
||||
const (
|
||||
MaintenanceTypeIcebergCompaction = "icebergCompaction"
|
||||
MaintenanceTypeIcebergSnapshotManagement = "icebergSnapshotManagement"
|
||||
MaintenanceTypeIcebergUnreferencedFileRemoval = "icebergUnreferencedFileRemoval"
|
||||
|
||||
MaintenanceStatusEnabled = "enabled"
|
||||
MaintenanceStatusDisabled = "disabled"
|
||||
)
|
||||
|
||||
type IcebergCompactionSettings struct {
|
||||
Strategy string `json:"strategy,omitempty"`
|
||||
TargetFileSizeMB *int64 `json:"targetFileSizeMB,omitempty"`
|
||||
}
|
||||
|
||||
// Compaction strategies. AWS also defines z-order, which the maintenance
|
||||
// worker cannot produce, so it is rejected rather than silently binpacked.
|
||||
const (
|
||||
CompactionStrategyAuto = "auto"
|
||||
CompactionStrategyBinpack = "binpack"
|
||||
CompactionStrategySort = "sort"
|
||||
CompactionStrategyZOrder = "z-order"
|
||||
)
|
||||
|
||||
type IcebergSnapshotManagementSettings struct {
|
||||
MinSnapshotsToKeep *int64 `json:"minSnapshotsToKeep,omitempty"`
|
||||
MaxSnapshotAgeHours *int64 `json:"maxSnapshotAgeHours,omitempty"`
|
||||
}
|
||||
|
||||
type IcebergUnreferencedFileRemovalSettings struct {
|
||||
UnreferencedDays *int64 `json:"unreferencedDays,omitempty"`
|
||||
NonCurrentDays *int64 `json:"nonCurrentDays,omitempty"`
|
||||
}
|
||||
|
||||
type MaintenanceSettings struct {
|
||||
IcebergCompaction *IcebergCompactionSettings `json:"icebergCompaction,omitempty"`
|
||||
IcebergSnapshotManagement *IcebergSnapshotManagementSettings `json:"icebergSnapshotManagement,omitempty"`
|
||||
IcebergUnreferencedFileRemoval *IcebergUnreferencedFileRemovalSettings `json:"icebergUnreferencedFileRemoval,omitempty"`
|
||||
}
|
||||
|
||||
type MaintenanceConfigurationValue struct {
|
||||
Status string `json:"status,omitempty"`
|
||||
Settings *MaintenanceSettings `json:"settings,omitempty"`
|
||||
}
|
||||
|
||||
// MaintenanceConfiguration maps a maintenance type to its configuration. It is
|
||||
// stored verbatim as the maintenance extended attribute so Get is a passthrough.
|
||||
type MaintenanceConfiguration map[string]*MaintenanceConfigurationValue
|
||||
|
||||
// MergeMaintenanceConfiguration overlays a table's configuration on its
|
||||
// bucket's. Unreferenced file removal is only configurable on the bucket, so
|
||||
// anything reading a table's effective configuration has to consult both.
|
||||
func MergeMaintenanceConfiguration(bucket, table MaintenanceConfiguration) MaintenanceConfiguration {
|
||||
if len(bucket) == 0 {
|
||||
return table
|
||||
}
|
||||
if len(table) == 0 {
|
||||
return bucket
|
||||
}
|
||||
|
||||
merged := make(MaintenanceConfiguration, len(bucket)+len(table))
|
||||
for k, v := range bucket {
|
||||
merged[k] = v
|
||||
}
|
||||
for k, v := range table {
|
||||
merged[k] = v
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
type PutTableBucketMaintenanceConfigurationRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Type string `json:"type"`
|
||||
Value *MaintenanceConfigurationValue `json:"value"`
|
||||
}
|
||||
|
||||
type GetTableBucketMaintenanceConfigurationRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
}
|
||||
|
||||
type GetTableBucketMaintenanceConfigurationResponse struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Configuration MaintenanceConfiguration `json:"configuration"`
|
||||
}
|
||||
|
||||
type PutTableMaintenanceConfigurationRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Value *MaintenanceConfigurationValue `json:"value"`
|
||||
}
|
||||
|
||||
type GetTableMaintenanceConfigurationRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type GetTableMaintenanceConfigurationResponse struct {
|
||||
TableARN string `json:"tableARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
Configuration MaintenanceConfiguration `json:"configuration"`
|
||||
}
|
||||
|
||||
// Maintenance job status types
|
||||
|
||||
const (
|
||||
MaintenanceJobStatusNotYetRun = "Not_Yet_Run"
|
||||
MaintenanceJobStatusSuccessful = "Successful"
|
||||
MaintenanceJobStatusFailed = "Failed"
|
||||
MaintenanceJobStatusDisabled = "Disabled"
|
||||
)
|
||||
|
||||
type MaintenanceJobStatusValue struct {
|
||||
Status string `json:"status"`
|
||||
LastRunTimestamp *time.Time `json:"lastRunTimestamp,omitempty"`
|
||||
FailureMessage string `json:"failureMessage,omitempty"`
|
||||
}
|
||||
|
||||
// MaintenanceJobStatus maps a maintenance type to the outcome of its last run.
|
||||
// The worker writes it; the control plane only reads it.
|
||||
type MaintenanceJobStatus map[string]*MaintenanceJobStatusValue
|
||||
|
||||
type GetTableMaintenanceJobStatusRequest struct {
|
||||
TableBucketARN string `json:"tableBucketARN"`
|
||||
Namespace []string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type GetTableMaintenanceJobStatusResponse struct {
|
||||
TableARN string `json:"tableARN"`
|
||||
Status MaintenanceJobStatus `json:"status"`
|
||||
}
|
||||
|
||||
// Tagging types
|
||||
|
||||
type TagResourceRequest struct {
|
||||
ResourceARN string `json:"resourceArn"`
|
||||
Tags map[string]string `json:"tags"`
|
||||
}
|
||||
|
||||
type ListTagsForResourceRequest struct {
|
||||
ResourceARN string `json:"resourceArn"`
|
||||
}
|
||||
|
||||
type ListTagsForResourceResponse struct {
|
||||
Tags map[string]string `json:"tags"`
|
||||
}
|
||||
|
||||
type UntagResourceRequest struct {
|
||||
ResourceARN string `json:"resourceArn"`
|
||||
TagKeys []string `json:"tagKeys"`
|
||||
}
|
||||
|
||||
// Error types
|
||||
|
||||
type S3TablesError struct {
|
||||
Type string `json:"__type"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
func (e *S3TablesError) Error() string {
|
||||
return e.Message
|
||||
}
|
||||
|
||||
// Table formats a catalog entry may declare.
|
||||
//
|
||||
// ICEBERG tables carry metadata the catalog maintains and the maintenance
|
||||
// worker rewrites. LANCE is catalog-only: the entry records a name and the
|
||||
// dataset root in MetadataLocation, and the Lance client owns every byte under
|
||||
// it. Nothing in this package interprets a catalog-only table's files.
|
||||
const (
|
||||
FormatIceberg = "ICEBERG"
|
||||
FormatLance = "LANCE"
|
||||
)
|
||||
|
||||
// IsCatalogOnlyFormat reports whether the catalog only records where a table of
|
||||
// this format lives, without understanding its files.
|
||||
func IsCatalogOnlyFormat(format string) bool {
|
||||
return format == FormatLance
|
||||
}
|
||||
|
||||
// NormalizeFormat folds a caller's spelling onto the canonical one and reports
|
||||
// whether it names a format this catalog serves.
|
||||
func NormalizeFormat(format string) (string, bool) {
|
||||
switch strings.ToUpper(strings.TrimSpace(format)) {
|
||||
case FormatIceberg:
|
||||
return FormatIceberg, true
|
||||
case FormatLance:
|
||||
return FormatLance, true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// Error codes
|
||||
const (
|
||||
ErrCodeBucketAlreadyExists = "BucketAlreadyExists"
|
||||
ErrCodeBucketNotEmpty = "BucketNotEmpty"
|
||||
ErrCodeNoSuchBucket = "NoSuchBucket"
|
||||
ErrCodeNoSuchNamespace = "NoSuchNamespace"
|
||||
ErrCodeNoSuchTable = "NoSuchTable"
|
||||
ErrCodeNamespaceAlreadyExists = "NamespaceAlreadyExists"
|
||||
ErrCodeNamespaceNotEmpty = "NamespaceNotEmpty"
|
||||
ErrCodeTableAlreadyExists = "TableAlreadyExists"
|
||||
ErrCodeNoSuchView = "NoSuchView"
|
||||
ErrCodeViewAlreadyExists = "ViewAlreadyExists"
|
||||
ErrCodeAccessDenied = "AccessDenied"
|
||||
ErrCodeInvalidRequest = "InvalidRequest"
|
||||
ErrCodeInternalError = "InternalError"
|
||||
ErrCodeNoSuchPolicy = "NoSuchPolicy"
|
||||
ErrCodeConflict = "Conflict"
|
||||
)
|
||||
535
weed/s3api/s3tables/utils.go
Normal file
535
weed/s3api/s3tables/utils.go
Normal file
@@ -0,0 +1,535 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
const (
|
||||
bucketNamePatternStr = `[a-z0-9-]+`
|
||||
tableNamespacePatternStr = `[a-z0-9_.-]+`
|
||||
tableNamePatternStr = `[a-z0-9_-]+`
|
||||
)
|
||||
|
||||
const (
|
||||
tableObjectRootDirName = ".objects"
|
||||
)
|
||||
|
||||
// ARNPartitionPatternStr matches any AWS partition, not just the commercial
|
||||
// one: aws-cn and aws-us-gov ARNs are valid and must reach the handlers.
|
||||
const ARNPartitionPatternStr = `aws[-a-z0-9]*`
|
||||
|
||||
// ARNPrefixPatternStr is the leading, partition-tolerant part of every S3
|
||||
// Tables ARN. The HTTP router shares it so routes and parsing agree.
|
||||
const ARNPrefixPatternStr = `arn:` + ARNPartitionPatternStr + `:s3tables`
|
||||
|
||||
var (
|
||||
bucketARNPattern = regexp.MustCompile(`^` + ARNPrefixPatternStr + `:[^:]*:[^:]*:bucket/(` + bucketNamePatternStr + `)$`)
|
||||
tableARNPattern = regexp.MustCompile(`^` + ARNPrefixPatternStr + `:[^:]*:[^:]*:bucket/(` + bucketNamePatternStr + `)/table/(` + tableNamespacePatternStr + `)/(` + tableNamePatternStr + `)$`)
|
||||
tagPattern = regexp.MustCompile(`^([\p{L}\p{Z}\p{N}_.:/=+\-@]*)$`)
|
||||
)
|
||||
|
||||
// ARN parsing functions
|
||||
|
||||
// parseBucketNameFromARN extracts bucket name from table bucket ARN
|
||||
// ARN format: arn:aws:s3tables:{region}:{account}:bucket/{bucket-name}
|
||||
func parseBucketNameFromARN(arn string) (string, error) {
|
||||
matches := bucketARNPattern.FindStringSubmatch(arn)
|
||||
if len(matches) != 2 {
|
||||
return "", fmt.Errorf("invalid bucket ARN: %s", arn)
|
||||
}
|
||||
bucketName := matches[1]
|
||||
if !isValidBucketName(bucketName) {
|
||||
return "", fmt.Errorf("invalid bucket name in ARN: %s", bucketName)
|
||||
}
|
||||
return bucketName, nil
|
||||
}
|
||||
|
||||
// ParseBucketNameFromARN is a wrapper to validate bucket ARN for other packages.
|
||||
func ParseBucketNameFromARN(arn string) (string, error) {
|
||||
return parseBucketNameFromARN(arn)
|
||||
}
|
||||
|
||||
// IsValidBucketName is a wrapper to validate a table bucket name for other packages.
|
||||
func IsValidBucketName(name string) bool {
|
||||
return isValidBucketName(name)
|
||||
}
|
||||
|
||||
// parseTableFromARN extracts bucket name, namespace, and table name from ARN
|
||||
// ARN format: arn:aws:s3tables:{region}:{account}:bucket/{bucket-name}/table/{namespace}/{table-name}
|
||||
func parseTableFromARN(arn string) (bucketName, namespace, tableName string, err error) {
|
||||
matches := tableARNPattern.FindStringSubmatch(arn)
|
||||
if len(matches) != 4 {
|
||||
return "", "", "", fmt.Errorf("invalid table ARN: %s", arn)
|
||||
}
|
||||
|
||||
// Validate bucket name
|
||||
bucketName = matches[1]
|
||||
if err := validateBucketName(bucketName); err != nil {
|
||||
return "", "", "", fmt.Errorf("invalid bucket name in ARN: %v", err)
|
||||
}
|
||||
|
||||
namespace, err = validateNamespace([]string{matches[2]})
|
||||
if err != nil {
|
||||
return "", "", "", fmt.Errorf("invalid namespace in ARN: %v", err)
|
||||
}
|
||||
|
||||
// URL decode and validate the table name from the ARN path component
|
||||
tableNameUnescaped, err := url.PathUnescape(matches[3])
|
||||
if err != nil {
|
||||
return "", "", "", fmt.Errorf("invalid table name encoding in ARN: %v", err)
|
||||
}
|
||||
if _, err := validateTableName(tableNameUnescaped); err != nil {
|
||||
return "", "", "", fmt.Errorf("invalid table name in ARN: %v", err)
|
||||
}
|
||||
return bucketName, namespace, tableNameUnescaped, nil
|
||||
}
|
||||
|
||||
// Path helpers
|
||||
|
||||
// GetTableBucketPath returns the filer path for a table bucket
|
||||
func GetTableBucketPath(bucketName string) string {
|
||||
return path.Join(TablesPath, bucketName)
|
||||
}
|
||||
|
||||
// GetNamespacePath returns the filer path for a namespace
|
||||
func GetNamespacePath(bucketName, namespace string) string {
|
||||
return path.Join(TablesPath, bucketName, namespace)
|
||||
}
|
||||
|
||||
// GetTablePath returns the filer path for a table
|
||||
func GetTablePath(bucketName, namespace, tableName string) string {
|
||||
return path.Join(TablesPath, bucketName, namespace, tableName)
|
||||
}
|
||||
|
||||
// TableDataDirFromMetadataLocation maps a table's s3:// metadata location to the
|
||||
// filer directory holding its data. A renamed table is catalog-only, so its data
|
||||
// stays at the original location while its catalog entry moves; this lets a drop
|
||||
// purge the real data instead of the now-empty catalog path.
|
||||
func TableDataDirFromMetadataLocation(metadataLocation string) string {
|
||||
loc := strings.TrimSuffix(metadataLocation, "/")
|
||||
if idx := strings.LastIndex(loc, "/metadata/"); idx != -1 {
|
||||
loc = loc[:idx]
|
||||
}
|
||||
loc = strings.TrimPrefix(loc, "s3://")
|
||||
if loc == "" {
|
||||
return ""
|
||||
}
|
||||
return path.Join(TablesPath, loc)
|
||||
}
|
||||
|
||||
// ValidateMetadataLocation checks that an s3:// metadata location stays within
|
||||
// the authorized table bucket and rejects traversal segments that path.Join
|
||||
// would collapse to escape the bucket directory. Empty locations are allowed
|
||||
// (the catalog derives one). A non-empty location must include a table path so
|
||||
// its metadata directory is table-specific, not shared at the bucket level.
|
||||
func ValidateMetadataLocation(metadataLocation, bucketName string) error {
|
||||
if metadataLocation == "" {
|
||||
return nil
|
||||
}
|
||||
bucket, tablePath, err := parseS3Location(metadataLocation)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if bucket != bucketName {
|
||||
return fmt.Errorf("metadata location must be within bucket %s", bucketName)
|
||||
}
|
||||
hasSegment := false
|
||||
for _, segment := range strings.Split(tablePath, "/") {
|
||||
if segment == "" {
|
||||
continue
|
||||
}
|
||||
if segment == "." || segment == ".." || strings.ContainsAny(segment, "\\\x00") {
|
||||
return fmt.Errorf("invalid metadata location path")
|
||||
}
|
||||
hasSegment = true
|
||||
}
|
||||
if !hasSegment {
|
||||
return fmt.Errorf("metadata location must include a table path")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseS3Location(location string) (bucket, tablePath string, err error) {
|
||||
if !strings.HasPrefix(location, "s3://") {
|
||||
return "", "", fmt.Errorf("unsupported location: %s", location)
|
||||
}
|
||||
trimmed := strings.TrimPrefix(location, "s3://")
|
||||
trimmed = strings.TrimSuffix(trimmed, "/")
|
||||
if trimmed == "" {
|
||||
return "", "", fmt.Errorf("invalid location: %s", location)
|
||||
}
|
||||
parts := strings.SplitN(trimmed, "/", 2)
|
||||
bucket = parts[0]
|
||||
if bucket == "" {
|
||||
return "", "", fmt.Errorf("invalid location bucket: %s", location)
|
||||
}
|
||||
if len(parts) == 2 {
|
||||
tablePath = parts[1]
|
||||
}
|
||||
return bucket, tablePath, nil
|
||||
}
|
||||
|
||||
// GetTableObjectRootDir returns the root path for table bucket object storage
|
||||
func GetTableObjectRootDir() string {
|
||||
return path.Join(TablesPath, tableObjectRootDirName)
|
||||
}
|
||||
|
||||
// GetTableObjectBucketPath returns the filer path for table bucket object storage
|
||||
func GetTableObjectBucketPath(bucketName string) string {
|
||||
return path.Join(GetTableObjectRootDir(), bucketName)
|
||||
}
|
||||
|
||||
// Metadata structures
|
||||
|
||||
type tableBucketMetadata struct {
|
||||
Name string `json:"name"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
OwnerAccountID string `json:"ownerAccountId"`
|
||||
Format string `json:"format,omitempty"`
|
||||
}
|
||||
|
||||
// namespaceMetadata stores metadata for a namespace
|
||||
type namespaceMetadata struct {
|
||||
Namespace []string `json:"namespace"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
OwnerAccountID string `json:"ownerAccountId"`
|
||||
Properties map[string]string `json:"properties,omitempty"`
|
||||
}
|
||||
|
||||
// tableMetadataInternal stores metadata for a table
|
||||
type tableMetadataInternal struct {
|
||||
Name string `json:"name"`
|
||||
Namespace string `json:"namespace"`
|
||||
Format string `json:"format"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
ModifiedAt time.Time `json:"modifiedAt"`
|
||||
OwnerAccountID string `json:"ownerAccountId"`
|
||||
VersionToken string `json:"versionToken"`
|
||||
MetadataVersion int `json:"metadataVersion"`
|
||||
MetadataLocation string `json:"metadataLocation,omitempty"`
|
||||
Metadata *TableMetadata `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
// IsTableBucketEntry returns true when the entry is marked as a table bucket.
|
||||
func IsTableBucketEntry(entry *filer_pb.Entry) bool {
|
||||
if entry == nil || entry.Extended == nil {
|
||||
return false
|
||||
}
|
||||
_, ok := entry.Extended[ExtendedKeyTableBucket]
|
||||
return ok
|
||||
}
|
||||
|
||||
// EntryType returns the entry-type marker for a catalog entry. Tables and views
|
||||
// share the same on-disk layout; the marker distinguishes them. An absent marker
|
||||
// means table for back-compat.
|
||||
func EntryType(extended map[string][]byte) string {
|
||||
if extended == nil {
|
||||
return EntryTypeTable
|
||||
}
|
||||
if v, ok := extended[ExtendedKeyEntryType]; ok && len(v) > 0 {
|
||||
return string(v)
|
||||
}
|
||||
return EntryTypeTable
|
||||
}
|
||||
|
||||
// Utility functions
|
||||
|
||||
// validateBucketName validates bucket name and returns an error if invalid.
|
||||
// Bucket names must contain only lowercase letters, numbers, and hyphens.
|
||||
// Length must be between 3 and 63 characters.
|
||||
// Must start and end with a letter or digit.
|
||||
// Reserved prefixes/suffixes are rejected.
|
||||
func validateBucketName(name string) error {
|
||||
if name == "" {
|
||||
return fmt.Errorf("bucket name is required")
|
||||
}
|
||||
|
||||
if len(name) < 3 || len(name) > 63 {
|
||||
return fmt.Errorf("bucket name must be between 3 and 63 characters")
|
||||
}
|
||||
|
||||
// Must start and end with a letter or digit
|
||||
start := name[0]
|
||||
end := name[len(name)-1]
|
||||
if !((start >= 'a' && start <= 'z') || (start >= '0' && start <= '9')) {
|
||||
return fmt.Errorf("bucket name must start with a letter or digit")
|
||||
}
|
||||
if !((end >= 'a' && end <= 'z') || (end >= '0' && end <= '9')) {
|
||||
return fmt.Errorf("bucket name must end with a letter or digit")
|
||||
}
|
||||
|
||||
// Allowed characters: a-z, 0-9, -
|
||||
for i := 0; i < len(name); i++ {
|
||||
ch := name[i]
|
||||
if (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '-' {
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("bucket name can only contain lowercase letters, numbers, and hyphens")
|
||||
}
|
||||
|
||||
// Reserved prefixes
|
||||
reservedPrefixes := []string{"xn--", "sthree-", "amzn-s3-demo-", "aws"}
|
||||
for _, p := range reservedPrefixes {
|
||||
if strings.HasPrefix(name, p) {
|
||||
return fmt.Errorf("bucket name cannot start with reserved prefix: %s", p)
|
||||
}
|
||||
}
|
||||
|
||||
// Reserved suffixes
|
||||
reservedSuffixes := []string{"-s3alias", "--ol-s3", "--x-s3", "--table-s3"}
|
||||
for _, s := range reservedSuffixes {
|
||||
if strings.HasSuffix(name, s) {
|
||||
return fmt.Errorf("bucket name cannot end with reserved suffix: %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// BuildBucketARN builds a bucket ARN with the provided region and account ID.
|
||||
// If region is empty, the ARN will omit the region field.
|
||||
func BuildBucketARN(region, accountID, bucketName string) (string, error) {
|
||||
if bucketName == "" {
|
||||
return "", fmt.Errorf("bucket name is required")
|
||||
}
|
||||
if err := validateBucketName(bucketName); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if accountID == "" {
|
||||
accountID = DefaultAccountID
|
||||
}
|
||||
return buildARN(region, accountID, fmt.Sprintf("bucket/%s", bucketName)), nil
|
||||
}
|
||||
|
||||
// BuildTableARN builds a table ARN with the provided region and account ID.
|
||||
func BuildTableARN(region, accountID, bucketName, namespace, tableName string) (string, error) {
|
||||
if bucketName == "" {
|
||||
return "", fmt.Errorf("bucket name is required")
|
||||
}
|
||||
if err := validateBucketName(bucketName); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if namespace == "" {
|
||||
return "", fmt.Errorf("namespace is required")
|
||||
}
|
||||
normalizedNamespace, err := validateNamespace([]string{namespace})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if tableName == "" {
|
||||
return "", fmt.Errorf("table name is required")
|
||||
}
|
||||
normalizedTable, err := validateTableName(tableName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if accountID == "" {
|
||||
accountID = DefaultAccountID
|
||||
}
|
||||
return buildARN(region, accountID, fmt.Sprintf("bucket/%s/table/%s/%s", bucketName, normalizedNamespace, normalizedTable)), nil
|
||||
}
|
||||
|
||||
func buildARN(region, accountID, resourcePath string) string {
|
||||
return fmt.Sprintf("arn:%s:s3tables:%s:%s:%s", arnPartitionForRegion(region), region, accountID, resourcePath)
|
||||
}
|
||||
|
||||
// arnPartitionForRegion returns the ARN partition a region belongs to, so an
|
||||
// ARN this handler emits round-trips through a client in that partition.
|
||||
func arnPartitionForRegion(region string) string {
|
||||
switch {
|
||||
case strings.HasPrefix(region, "cn-"):
|
||||
return "aws-cn"
|
||||
case strings.HasPrefix(region, "us-gov-"):
|
||||
return "aws-us-gov"
|
||||
case strings.HasPrefix(region, "us-iso-"):
|
||||
return "aws-iso"
|
||||
case strings.HasPrefix(region, "us-isob-"):
|
||||
return "aws-iso-b"
|
||||
case strings.HasPrefix(region, "eu-isoe-"):
|
||||
return "aws-iso-e"
|
||||
case strings.HasPrefix(region, "us-isof-"):
|
||||
return "aws-iso-f"
|
||||
case strings.HasPrefix(region, "eusc-"):
|
||||
return "aws-eusc"
|
||||
default:
|
||||
return "aws"
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateTags validates tags for S3 Tables.
|
||||
func ValidateTags(tags map[string]string) error {
|
||||
if len(tags) > 10 {
|
||||
return fmt.Errorf("validate tags: %d tags more than 10", len(tags))
|
||||
}
|
||||
for k, v := range tags {
|
||||
if len(k) > 128 {
|
||||
return fmt.Errorf("validate tags: tag key longer than 128")
|
||||
}
|
||||
if !tagPattern.MatchString(k) {
|
||||
return fmt.Errorf("validate tags key %s error, incorrect key", k)
|
||||
}
|
||||
if len(v) > 256 {
|
||||
return fmt.Errorf("validate tags: tag value longer than 256")
|
||||
}
|
||||
if !tagPattern.MatchString(v) {
|
||||
return fmt.Errorf("validate tags value %s error, incorrect value", v)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isValidBucketName validates bucket name characters (kept for compatibility)
|
||||
// Deprecated: use validateBucketName instead
|
||||
func isValidBucketName(name string) bool {
|
||||
return validateBucketName(name) == nil
|
||||
}
|
||||
|
||||
// generateVersionToken generates a unique, unpredictable version token
|
||||
func generateVersionToken() string {
|
||||
b := make([]byte, 16)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
// Fallback to timestamp if crypto/rand fails
|
||||
return fmt.Sprintf("%x", time.Now().UnixNano())
|
||||
}
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
// splitPath splits a path into directory and name components using stdlib
|
||||
func splitPath(p string) (dir, name string) {
|
||||
dir = path.Dir(p)
|
||||
name = path.Base(p)
|
||||
return
|
||||
}
|
||||
|
||||
func validateNamespacePart(name string) error {
|
||||
if len(name) < 1 || len(name) > 255 {
|
||||
return fmt.Errorf("namespace name must be between 1 and 255 characters")
|
||||
}
|
||||
|
||||
// Prevent path traversal and multi-segment paths
|
||||
if name == "." || name == ".." {
|
||||
return fmt.Errorf("namespace name cannot be '.' or '..'")
|
||||
}
|
||||
if strings.Contains(name, "/") {
|
||||
return fmt.Errorf("namespace name cannot contain '/'")
|
||||
}
|
||||
|
||||
// Must start and end with a letter or digit
|
||||
start := name[0]
|
||||
end := name[len(name)-1]
|
||||
if !((start >= 'a' && start <= 'z') || (start >= '0' && start <= '9')) {
|
||||
return fmt.Errorf("namespace name must start with a letter or digit")
|
||||
}
|
||||
if !((end >= 'a' && end <= 'z') || (end >= '0' && end <= '9')) {
|
||||
return fmt.Errorf("namespace name must end with a letter or digit")
|
||||
}
|
||||
|
||||
// Allowed characters: a-z, 0-9, _, - (hyphen interior; start/end checked above)
|
||||
for _, ch := range name {
|
||||
if (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '-' {
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("invalid namespace name: only 'a-z', '0-9', '_', and '-' are allowed")
|
||||
}
|
||||
|
||||
// Reserved prefix
|
||||
if strings.HasPrefix(name, "aws") {
|
||||
return fmt.Errorf("namespace name cannot start with reserved prefix 'aws'")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeNamespace(namespace []string) ([]string, error) {
|
||||
if len(namespace) == 0 {
|
||||
return nil, fmt.Errorf("namespace is required")
|
||||
}
|
||||
|
||||
parts := namespace
|
||||
if len(namespace) == 1 {
|
||||
parts = strings.Split(namespace[0], ".")
|
||||
}
|
||||
|
||||
normalized := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
if err := validateNamespacePart(part); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
normalized = append(normalized, part)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
// validateNamespace validates namespace identifiers and returns an internal namespace key.
|
||||
// A single dotted namespace value is interpreted as multi-level namespace for compatibility
|
||||
// with path-style APIs, for example "analytics.daily" => ["analytics", "daily"].
|
||||
func validateNamespace(namespace []string) (string, error) {
|
||||
parts, err := normalizeNamespace(namespace)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return flattenNamespace(parts), nil
|
||||
}
|
||||
|
||||
// ParseNamespace parses a namespace string into namespace parts.
|
||||
func ParseNamespace(namespace string) ([]string, error) {
|
||||
return normalizeNamespace([]string{namespace})
|
||||
}
|
||||
|
||||
// validateTableName validates a table name
|
||||
func validateTableName(name string) (string, error) {
|
||||
if len(name) < 1 || len(name) > 255 {
|
||||
return "", fmt.Errorf("table name must be between 1 and 255 characters")
|
||||
}
|
||||
if name == "." || name == ".." || strings.Contains(name, "/") {
|
||||
return "", fmt.Errorf("invalid table name: cannot be '.', '..' or contain '/'")
|
||||
}
|
||||
|
||||
// First character must be a letter or digit
|
||||
start := name[0]
|
||||
if !((start >= 'a' && start <= 'z') || (start >= '0' && start <= '9')) {
|
||||
return "", fmt.Errorf("table name must start with a letter or digit")
|
||||
}
|
||||
|
||||
// Allowed characters: a-z, 0-9, _, - (start checked above)
|
||||
for _, ch := range name {
|
||||
if (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '-' {
|
||||
continue
|
||||
}
|
||||
return "", fmt.Errorf("invalid table name: only 'a-z', '0-9', '_', and '-' are allowed")
|
||||
}
|
||||
return name, nil
|
||||
}
|
||||
|
||||
// ValidateTableName is a wrapper to validate table name for other packages.
|
||||
func ValidateTableName(name string) (string, error) {
|
||||
return validateTableName(name)
|
||||
}
|
||||
|
||||
// flattenNamespace joins namespace elements into a single string (using dots as per AWS S3 Tables)
|
||||
func flattenNamespace(namespace []string) string {
|
||||
if len(namespace) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(namespace, ".")
|
||||
}
|
||||
|
||||
func expandNamespace(namespace string) []string {
|
||||
if namespace == "" {
|
||||
return nil
|
||||
}
|
||||
parts, err := ParseNamespace(namespace)
|
||||
if err != nil {
|
||||
return []string{namespace}
|
||||
}
|
||||
return parts
|
||||
}
|
||||
33
weed/s3api/s3tables/utils_location_test.go
Normal file
33
weed/s3api/s3tables/utils_location_test.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package s3tables
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestValidateMetadataLocation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
loc string
|
||||
bucket string
|
||||
wantErr bool
|
||||
}{
|
||||
{"empty allowed", "", "bkt", false},
|
||||
{"same bucket", "s3://bkt/ns/t/metadata/v1.metadata.json", "bkt", false},
|
||||
{"cross bucket rejected", "s3://other/ns/t/metadata/v1.metadata.json", "bkt", true},
|
||||
{"bucket only rejected", "s3://bkt", "bkt", true},
|
||||
{"bucket with trailing slash rejected", "s3://bkt/", "bkt", true},
|
||||
{"slash-only path rejected", "s3://bkt///", "bkt", true},
|
||||
{"dotdot in path rejected", "s3://bkt/../victim/metadata/v1.metadata.json", "bkt", true},
|
||||
{"dotdot mid path rejected", "s3://bkt/ns/../../victim/metadata/v1.metadata.json", "bkt", true},
|
||||
{"dot segment rejected", "s3://bkt/ns/./t/metadata/v1.metadata.json", "bkt", true},
|
||||
{"backslash segment rejected", "s3://bkt/ns/\\t/metadata/v1.metadata.json", "bkt", true},
|
||||
{"non-s3 scheme rejected", "file:///bkt/ns/t", "bkt", true},
|
||||
{"empty location string", "s3://", "bkt", true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := ValidateMetadataLocation(tt.loc, tt.bucket)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("ValidateMetadataLocation(%q, %q) err = %v, wantErr = %v", tt.loc, tt.bucket, err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
186
weed/s3api/s3tables/utils_namespace_test.go
Normal file
186
weed/s3api/s3tables/utils_namespace_test.go
Normal file
@@ -0,0 +1,186 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateNamespaceSupportsMultiLevel(t *testing.T) {
|
||||
got, err := validateNamespace([]string{"analytics", "daily"})
|
||||
if err != nil {
|
||||
t.Fatalf("validateNamespace returned error: %v", err)
|
||||
}
|
||||
if got != "analytics.daily" {
|
||||
t.Fatalf("validateNamespace = %q, want %q", got, "analytics.daily")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNamespaceSupportsDottedInput(t *testing.T) {
|
||||
got, err := validateNamespace([]string{"analytics.daily"})
|
||||
if err != nil {
|
||||
t.Fatalf("validateNamespace returned error: %v", err)
|
||||
}
|
||||
if got != "analytics.daily" {
|
||||
t.Fatalf("validateNamespace = %q, want %q", got, "analytics.daily")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNamespaceRejectsEmptyDottedSegment(t *testing.T) {
|
||||
_, err := validateNamespace([]string{"analytics..daily"})
|
||||
if err == nil {
|
||||
t.Fatalf("expected validateNamespace to fail for empty dotted segment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNamespace(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
namespace string
|
||||
want []string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "single level",
|
||||
namespace: "analytics",
|
||||
want: []string{"analytics"},
|
||||
},
|
||||
{
|
||||
name: "multi level dotted",
|
||||
namespace: "analytics.daily",
|
||||
want: []string{"analytics", "daily"},
|
||||
},
|
||||
{
|
||||
name: "invalid reserved prefix",
|
||||
namespace: "analytics.awsprod",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid empty segment",
|
||||
namespace: "analytics..daily",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := ParseNamespace(tt.namespace)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("ParseNamespace(%q) expected error", tt.namespace)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("ParseNamespace(%q) unexpected error: %v", tt.namespace, err)
|
||||
}
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("ParseNamespace(%q) = %v, want %v", tt.namespace, got, tt.want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != tt.want[i] {
|
||||
t.Fatalf("ParseNamespace(%q) = %v, want %v", tt.namespace, got, tt.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTableFromARNWithMultiLevelNamespace(t *testing.T) {
|
||||
arn := "arn:aws:s3tables:us-east-1:123456789012:bucket/testbucket/table/analytics.daily/events"
|
||||
bucket, namespace, table, err := parseTableFromARN(arn)
|
||||
if err != nil {
|
||||
t.Fatalf("parseTableFromARN returned error: %v", err)
|
||||
}
|
||||
if bucket != "testbucket" {
|
||||
t.Fatalf("bucket = %q, want %q", bucket, "testbucket")
|
||||
}
|
||||
if namespace != "analytics.daily" {
|
||||
t.Fatalf("namespace = %q, want %q", namespace, "analytics.daily")
|
||||
}
|
||||
if table != "events" {
|
||||
t.Fatalf("table = %q, want %q", table, "events")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildTableARNWithDottedNamespace(t *testing.T) {
|
||||
arn, err := BuildTableARN("us-east-1", "123456789012", "testbucket", "analytics.daily", "events")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildTableARN returned error: %v", err)
|
||||
}
|
||||
if !strings.Contains(arn, "/table/analytics.daily/events") {
|
||||
t.Fatalf("BuildTableARN returned %q, missing normalized namespace/table path", arn)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandNamespace(t *testing.T) {
|
||||
got := expandNamespace("analytics.daily")
|
||||
want := []string{"analytics", "daily"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("expandNamespace = %v, want %v", got, want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("expandNamespace = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamespaceMetadataPropertiesRoundTrip(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
metadata namespaceMetadata
|
||||
}{
|
||||
{
|
||||
name: "with properties",
|
||||
metadata: namespaceMetadata{
|
||||
Namespace: []string{"analytics"},
|
||||
Properties: map[string]string{"owner": "finance"},
|
||||
OwnerAccountID: "123456789012",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "nil properties",
|
||||
metadata: namespaceMetadata{
|
||||
Namespace: []string{"analytics"},
|
||||
Properties: nil,
|
||||
OwnerAccountID: "123456789012",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty properties",
|
||||
metadata: namespaceMetadata{
|
||||
Namespace: []string{"analytics"},
|
||||
Properties: map[string]string{},
|
||||
OwnerAccountID: "123456789012",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
data, err := json.Marshal(tc.metadata)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(metadata) returned error: %v", err)
|
||||
}
|
||||
|
||||
var decoded namespaceMetadata
|
||||
if err := json.Unmarshal(data, &decoded); err != nil {
|
||||
t.Fatalf("json.Unmarshal(data) returned error: %v", err)
|
||||
}
|
||||
|
||||
// Due to `omitempty`, nil and empty maps are unmarshaled as nil.
|
||||
if len(tc.metadata.Properties) == 0 {
|
||||
if decoded.Properties != nil {
|
||||
t.Fatalf("expected nil properties for empty/nil input, got %v", decoded.Properties)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(decoded.Properties, tc.metadata.Properties) {
|
||||
t.Fatalf("decoded.Properties = %v, want %v", decoded.Properties, tc.metadata.Properties)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user