snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
100
weed/s3api/lance/errors.go
Normal file
100
weed/s3api/lance/errors.go
Normal file
@@ -0,0 +1,100 @@
|
||||
package lance
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
// Lance Namespace error codes. The spec numbers them, so a client switches on
|
||||
// the code rather than parsing the message.
|
||||
const (
|
||||
codeUnsupported = 0
|
||||
codeNamespaceNotFound = 1
|
||||
codeNamespaceAlreadyExists = 2
|
||||
codeNamespaceNotEmpty = 3
|
||||
codeTableNotFound = 4
|
||||
codeTableAlreadyExists = 5
|
||||
codeTableVersionNotFound = 11
|
||||
codeInvalidInput = 13
|
||||
codeConcurrentModification = 14
|
||||
codePermissionDenied = 15
|
||||
codeUnauthenticated = 16
|
||||
codeInternal = 18
|
||||
)
|
||||
|
||||
type errorResponse struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
Code int `json:"code"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
Instance string `json:"instance,omitempty"`
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, body interface{}) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
if body == nil {
|
||||
return
|
||||
}
|
||||
if err := json.NewEncoder(w).Encode(body); err != nil {
|
||||
glog.Warningf("lance: failed to encode response: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, r *http.Request, status, code int, message string) {
|
||||
instance := ""
|
||||
if r != nil {
|
||||
instance = r.URL.Path
|
||||
}
|
||||
writeJSON(w, status, errorResponse{Error: message, Code: code, Instance: instance})
|
||||
}
|
||||
|
||||
// writeStorageError translates an S3 Tables storage error into the Lance error
|
||||
// model. A table bucket is the first namespace level here, so a missing bucket
|
||||
// is a missing namespace rather than a missing catalog.
|
||||
func writeStorageError(w http.ResponseWriter, r *http.Request, err error) {
|
||||
var storageErr *s3tables.S3TablesError
|
||||
if !errors.As(err, &storageErr) {
|
||||
writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
status, code := http.StatusInternalServerError, codeInternal
|
||||
switch storageErr.Type {
|
||||
case s3tables.ErrCodeNoSuchBucket, s3tables.ErrCodeNoSuchNamespace:
|
||||
status, code = http.StatusNotFound, codeNamespaceNotFound
|
||||
case s3tables.ErrCodeNoSuchTable, s3tables.ErrCodeNoSuchView:
|
||||
status, code = http.StatusNotFound, codeTableNotFound
|
||||
case s3tables.ErrCodeBucketAlreadyExists, s3tables.ErrCodeNamespaceAlreadyExists:
|
||||
status, code = http.StatusConflict, codeNamespaceAlreadyExists
|
||||
case s3tables.ErrCodeTableAlreadyExists, s3tables.ErrCodeViewAlreadyExists:
|
||||
status, code = http.StatusConflict, codeTableAlreadyExists
|
||||
case s3tables.ErrCodeBucketNotEmpty, s3tables.ErrCodeNamespaceNotEmpty:
|
||||
status, code = http.StatusConflict, codeNamespaceNotEmpty
|
||||
case s3tables.ErrCodeConflict:
|
||||
status, code = http.StatusConflict, codeConcurrentModification
|
||||
case s3tables.ErrCodeAccessDenied:
|
||||
status, code = http.StatusForbidden, codePermissionDenied
|
||||
case s3tables.ErrCodeInvalidRequest, s3tables.ErrCodeInvalidIcebergLayout:
|
||||
status, code = http.StatusBadRequest, codeInvalidInput
|
||||
}
|
||||
writeError(w, r, status, code, storageErr.Message)
|
||||
}
|
||||
|
||||
// isNotFound reports whether a storage error means the object is absent, so
|
||||
// exists-style handlers can answer without a second lookup.
|
||||
func isNotFound(err error) bool {
|
||||
var storageErr *s3tables.S3TablesError
|
||||
if !errors.As(err, &storageErr) {
|
||||
return false
|
||||
}
|
||||
switch storageErr.Type {
|
||||
case s3tables.ErrCodeNoSuchBucket, s3tables.ErrCodeNoSuchNamespace,
|
||||
s3tables.ErrCodeNoSuchTable, s3tables.ErrCodeNoSuchView:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
341
weed/s3api/lance/handlers_namespace.go
Normal file
341
weed/s3api/lance/handlers_namespace.go
Normal file
@@ -0,0 +1,341 @@
|
||||
package lance
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
// handleCreateNamespace creates a table bucket for a one-part identifier and a
|
||||
// namespace inside one for anything deeper. The root cannot be created.
|
||||
func (s *Server) handleCreateNamespace(w http.ResponseWriter, r *http.Request) {
|
||||
id, _, ok := routeIdentifier(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
var req CreateNamespaceRequest
|
||||
if err := decodeBody(r, &req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
if len(id) == 0 && len(req.ID) > 0 {
|
||||
id = req.ID
|
||||
}
|
||||
if !checkBodyIdentifier(w, r, id, req.ID) {
|
||||
return
|
||||
}
|
||||
if len(id) == 0 {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, "the root namespace always exists and cannot be created")
|
||||
return
|
||||
}
|
||||
|
||||
mode := normalizeMode(req.Mode, modeCreate)
|
||||
switch mode {
|
||||
case modeCreate, modeExistOk, modeOverwrite:
|
||||
default:
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, "mode must be Create, ExistOk or Overwrite")
|
||||
return
|
||||
}
|
||||
|
||||
bucket, ns := id.namespace()
|
||||
exists, err := s.namespaceExists(r, bucket, ns)
|
||||
if err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
if exists {
|
||||
switch mode {
|
||||
case modeExistOk:
|
||||
writeJSON(w, http.StatusOK, CreateNamespaceResponse{Properties: req.Properties})
|
||||
return
|
||||
case modeCreate:
|
||||
writeError(w, r, http.StatusConflict, codeNamespaceAlreadyExists, "namespace already exists")
|
||||
return
|
||||
}
|
||||
// Overwrite replaces the namespace with an empty one, so the drop has to
|
||||
// succeed first. A namespace holding tables refuses, which is the point.
|
||||
if err := s.dropNamespace(r, bucket, ns); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.createNamespace(r, bucket, ns, req.Properties); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, CreateNamespaceResponse{Properties: normalizeProperties(req.Properties)})
|
||||
}
|
||||
|
||||
// handleListNamespaces lists the children of a namespace: table buckets at the
|
||||
// root, and the next path component below that.
|
||||
func (s *Server) handleListNamespaces(w http.ResponseWriter, r *http.Request) {
|
||||
id, _, ok := routeIdentifier(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
bucket, ns := id.namespace()
|
||||
|
||||
if bucket == "" {
|
||||
var resp s3tables.ListTableBucketsResponse
|
||||
req := &s3tables.ListTableBucketsRequest{
|
||||
ContinuationToken: r.URL.Query().Get("page_token"),
|
||||
MaxBuckets: pageSize(r),
|
||||
}
|
||||
if err := s.execute(r, "ListTableBuckets", req, &resp); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
names := make([]string, 0, len(resp.TableBuckets))
|
||||
for _, b := range resp.TableBuckets {
|
||||
names = append(names, b.Name)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, ListNamespacesResponse{Namespaces: names, PageToken: resp.ContinuationToken})
|
||||
return
|
||||
}
|
||||
|
||||
var resp s3tables.ListNamespacesResponse
|
||||
req := &s3tables.ListNamespacesRequest{
|
||||
TableBucketARN: bucketARN(bucket),
|
||||
ContinuationToken: r.URL.Query().Get("page_token"),
|
||||
MaxNamespaces: pageSize(r),
|
||||
}
|
||||
if len(ns) > 0 {
|
||||
req.Prefix = strings.Join(ns, ".") + "."
|
||||
}
|
||||
if err := s.execute(r, "ListNamespaces", req, &resp); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Storage namespaces are full paths; the spec wants the child name relative
|
||||
// to the parent, so take the next component and drop repeats.
|
||||
children := make([]string, 0, len(resp.Namespaces))
|
||||
seen := make(map[string]struct{}, len(resp.Namespaces))
|
||||
for _, summary := range resp.Namespaces {
|
||||
if len(summary.Namespace) <= len(ns) {
|
||||
continue
|
||||
}
|
||||
child := summary.Namespace[len(ns)]
|
||||
if _, done := seen[child]; done {
|
||||
continue
|
||||
}
|
||||
seen[child] = struct{}{}
|
||||
children = append(children, child)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, ListNamespacesResponse{Namespaces: children, PageToken: resp.ContinuationToken})
|
||||
}
|
||||
|
||||
// handleDescribeNamespace returns a namespace's properties.
|
||||
func (s *Server) handleDescribeNamespace(w http.ResponseWriter, r *http.Request) {
|
||||
id, _, ok := routeIdentifier(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var req DescribeNamespaceRequest
|
||||
if err := decodeBody(r, &req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
if len(id) == 0 && len(req.ID) > 0 {
|
||||
id = req.ID
|
||||
}
|
||||
if !checkBodyIdentifier(w, r, id, req.ID) {
|
||||
return
|
||||
}
|
||||
|
||||
bucket, ns := id.namespace()
|
||||
if bucket == "" {
|
||||
writeJSON(w, http.StatusOK, DescribeNamespaceResponse{Properties: map[string]string{}})
|
||||
return
|
||||
}
|
||||
if len(ns) == 0 {
|
||||
var resp s3tables.GetTableBucketResponse
|
||||
if err := s.execute(r, "GetTableBucket", &s3tables.GetTableBucketRequest{TableBucketARN: bucketARN(bucket)}, &resp); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, DescribeNamespaceResponse{Properties: map[string]string{}})
|
||||
return
|
||||
}
|
||||
|
||||
var resp s3tables.GetNamespaceResponse
|
||||
req2 := &s3tables.GetNamespaceRequest{TableBucketARN: bucketARN(bucket), Namespace: ns}
|
||||
if err := s.execute(r, "GetNamespace", req2, &resp); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, DescribeNamespaceResponse{Properties: normalizeProperties(resp.Properties)})
|
||||
}
|
||||
|
||||
// handleDropNamespace removes a namespace or table bucket.
|
||||
func (s *Server) handleDropNamespace(w http.ResponseWriter, r *http.Request) {
|
||||
id, _, ok := routeIdentifier(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var req DropNamespaceRequest
|
||||
if err := decodeBody(r, &req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
if len(id) == 0 && len(req.ID) > 0 {
|
||||
id = req.ID
|
||||
}
|
||||
if !checkBodyIdentifier(w, r, id, req.ID) {
|
||||
return
|
||||
}
|
||||
if len(id) == 0 {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, "the root namespace cannot be dropped")
|
||||
return
|
||||
}
|
||||
|
||||
if normalizeMode(req.Behavior, behaviorRestrict) == behaviorCascade {
|
||||
writeError(w, r, http.StatusNotImplemented, codeUnsupported,
|
||||
"cascade drop is not supported; drop the tables in the namespace first")
|
||||
return
|
||||
}
|
||||
|
||||
bucket, ns := id.namespace()
|
||||
err := s.dropNamespace(r, bucket, ns)
|
||||
if err == nil {
|
||||
writeJSON(w, http.StatusOK, DropNamespaceResponse{})
|
||||
return
|
||||
}
|
||||
if isNotFound(err) {
|
||||
// Skip reports success on a missing namespace; Fail reports 400 rather
|
||||
// than 404, which is what the spec asks for on this operation alone.
|
||||
if normalizeMode(req.Mode, modeFail) == modeSkip {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
writeError(w, r, http.StatusBadRequest, codeNamespaceNotFound, "namespace does not exist")
|
||||
return
|
||||
}
|
||||
writeStorageError(w, r, err)
|
||||
}
|
||||
|
||||
// handleNamespaceExists answers with the status code and no body.
|
||||
func (s *Server) handleNamespaceExists(w http.ResponseWriter, r *http.Request) {
|
||||
id, _, ok := routeIdentifier(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var req NamespaceExistsRequest
|
||||
if err := decodeBody(r, &req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
if len(id) == 0 && len(req.ID) > 0 {
|
||||
id = req.ID
|
||||
}
|
||||
if !checkBodyIdentifier(w, r, id, req.ID) {
|
||||
return
|
||||
}
|
||||
|
||||
bucket, ns := id.namespace()
|
||||
if bucket == "" {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
exists, err := s.namespaceExists(r, bucket, ns)
|
||||
if err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
if !exists {
|
||||
writeError(w, r, http.StatusNotFound, codeNamespaceNotFound, "namespace does not exist")
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (s *Server) namespaceExists(r *http.Request, bucket string, ns []string) (bool, error) {
|
||||
var err error
|
||||
if len(ns) == 0 {
|
||||
var resp s3tables.GetTableBucketResponse
|
||||
err = s.execute(r, "GetTableBucket", &s3tables.GetTableBucketRequest{TableBucketARN: bucketARN(bucket)}, &resp)
|
||||
} else {
|
||||
var resp s3tables.GetNamespaceResponse
|
||||
err = s.execute(r, "GetNamespace", &s3tables.GetNamespaceRequest{TableBucketARN: bucketARN(bucket), Namespace: ns}, &resp)
|
||||
}
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if isNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
func (s *Server) createNamespace(r *http.Request, bucket string, ns []string, properties map[string]string) error {
|
||||
if len(ns) == 0 {
|
||||
// A bucket made through this surface holds Lance tables. Saying so is
|
||||
// what stops it from being described to a client as an Iceberg catalog.
|
||||
var resp s3tables.CreateTableBucketResponse
|
||||
return s.execute(r, "CreateTableBucket", &s3tables.CreateTableBucketRequest{
|
||||
Name: bucket,
|
||||
Format: s3tables.FormatLance,
|
||||
}, &resp)
|
||||
}
|
||||
// A table bucket is a tenant resource with its own policy and lifecycle, so
|
||||
// it is created deliberately, never as a side effect of naming a namespace
|
||||
// inside it.
|
||||
// The immediate parent has to exist too. Storage keeps a namespace's parts
|
||||
// flattened, so creating "a.b" without "a" leaves an intermediate that
|
||||
// listing derives from the name and describe then denies exists. The spec
|
||||
// asks for NamespaceNotFound here, which also keeps the two consistent.
|
||||
if len(ns) > 1 {
|
||||
parent := ns[:len(ns)-1]
|
||||
if exists, err := s.namespaceExists(r, bucket, parent); err != nil {
|
||||
return err
|
||||
} else if !exists {
|
||||
return &s3tables.S3TablesError{
|
||||
Type: s3tables.ErrCodeNoSuchNamespace,
|
||||
Message: "parent namespace " + strings.Join(parent, ".") + " does not exist",
|
||||
}
|
||||
}
|
||||
}
|
||||
if exists, err := s.namespaceExists(r, bucket, nil); err != nil {
|
||||
return err
|
||||
} else if !exists {
|
||||
return &s3tables.S3TablesError{
|
||||
Type: s3tables.ErrCodeNoSuchBucket,
|
||||
Message: "table bucket " + bucket + " does not exist",
|
||||
}
|
||||
}
|
||||
var resp s3tables.CreateNamespaceResponse
|
||||
req := &s3tables.CreateNamespaceRequest{
|
||||
TableBucketARN: bucketARN(bucket),
|
||||
Namespace: ns,
|
||||
Properties: properties,
|
||||
}
|
||||
return s.execute(r, "CreateNamespace", req, &resp)
|
||||
}
|
||||
|
||||
func (s *Server) dropNamespace(r *http.Request, bucket string, ns []string) error {
|
||||
if len(ns) == 0 {
|
||||
return s.execute(r, "DeleteTableBucket", &s3tables.DeleteTableBucketRequest{TableBucketARN: bucketARN(bucket)}, nil)
|
||||
}
|
||||
req := &s3tables.DeleteNamespaceRequest{TableBucketARN: bucketARN(bucket), Namespace: ns}
|
||||
return s.execute(r, "DeleteNamespace", req, nil)
|
||||
}
|
||||
|
||||
// normalizeMode folds a spec mode or behavior value, which is case-insensitive
|
||||
// and spelled either PascalCase or snake_case, onto its lowercase form.
|
||||
func normalizeMode(value, fallback string) string {
|
||||
value = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(value), "_", ""))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func normalizeProperties(properties map[string]string) map[string]string {
|
||||
if properties == nil {
|
||||
return map[string]string{}
|
||||
}
|
||||
return properties
|
||||
}
|
||||
573
weed/s3api/lance/handlers_table.go
Normal file
573
weed/s3api/lance/handlers_table.go
Normal file
@@ -0,0 +1,573 @@
|
||||
package lance
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
// handleListTables lists the Lance tables under a namespace. The spec asks for
|
||||
// full string identifiers, not bare names, so a recursive listing stays
|
||||
// unambiguous.
|
||||
func (s *Server) handleListTables(w http.ResponseWriter, r *http.Request) {
|
||||
id, delimiter, ok := routeIdentifier(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
bucket, ns := id.namespace()
|
||||
if bucket == "" {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, "listing tables needs a namespace, not the root")
|
||||
return
|
||||
}
|
||||
|
||||
req := &s3tables.ListTablesRequest{
|
||||
TableBucketARN: bucketARN(bucket),
|
||||
Namespace: ns,
|
||||
ContinuationToken: r.URL.Query().Get("page_token"),
|
||||
MaxTables: pageSize(r),
|
||||
}
|
||||
var resp s3tables.ListTablesResponse
|
||||
if err := s.execute(r, "ListTables", req, &resp); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
includeDeclared := true
|
||||
if raw := r.URL.Query().Get("include_declared"); raw != "" {
|
||||
includeDeclared = boolQuery(r, "include_declared")
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, ListTablesResponse{
|
||||
Tables: s.lanceTableIDs(r, bucket, resp.Tables, delimiter, includeDeclared),
|
||||
PageToken: resp.ContinuationToken,
|
||||
})
|
||||
}
|
||||
|
||||
// handleListAllTables lists every Lance table the caller can see, across every
|
||||
// table bucket.
|
||||
func (s *Server) handleListAllTables(w http.ResponseWriter, r *http.Request) {
|
||||
delimiter := requestDelimiter(r)
|
||||
|
||||
var buckets s3tables.ListTableBucketsResponse
|
||||
if err := s.execute(r, "ListTableBuckets", &s3tables.ListTableBucketsRequest{MaxBuckets: pageSize(r)}, &buckets); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// The spec marks `tables` required, so an empty catalog answers with an
|
||||
// empty list rather than null.
|
||||
all := []string{}
|
||||
for _, bucket := range buckets.TableBuckets {
|
||||
var tables s3tables.ListTablesResponse
|
||||
req := &s3tables.ListTablesRequest{TableBucketARN: bucketARN(bucket.Name), MaxTables: pageSize(r)}
|
||||
if err := s.execute(r, "ListTables", req, &tables); err != nil {
|
||||
// One unreadable bucket must not hide the rest; a caller with access
|
||||
// to some buckets still gets those.
|
||||
glog.V(2).Infof("lance: skipping bucket %s in ListAllTables: %v", bucket.Name, err)
|
||||
continue
|
||||
}
|
||||
all = append(all, s.lanceTableIDs(r, bucket.Name, tables.Tables, delimiter, true)...)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, ListTablesResponse{Tables: all})
|
||||
}
|
||||
|
||||
// lanceTableIDs keeps the Lance tables out of a listing that also carries
|
||||
// Iceberg tables, and drops deregistered ones because they are meant to be
|
||||
// invisible until re-registered.
|
||||
func (s *Server) lanceTableIDs(r *http.Request, bucket string, summaries []s3tables.TableSummary, delimiter string, includeDeclared bool) []string {
|
||||
ids := make([]string, 0, len(summaries))
|
||||
for _, summary := range summaries {
|
||||
if summary.Format != s3tables.FormatLance {
|
||||
continue
|
||||
}
|
||||
location := summary.MetadataLocation
|
||||
if location == "" {
|
||||
location = tableLocation(bucket, summary.Namespace, summary.Name)
|
||||
}
|
||||
deregistered, hasData, err := s.datasetState(r, location)
|
||||
if err != nil {
|
||||
glog.V(2).Infof("lance: cannot read dataset state for %s: %v", location, err)
|
||||
continue
|
||||
}
|
||||
if deregistered {
|
||||
continue
|
||||
}
|
||||
if !hasData && !includeDeclared {
|
||||
continue
|
||||
}
|
||||
parts := append([]string{bucket}, summary.Namespace...)
|
||||
ids = append(ids, identifier(append(parts, summary.Name)).String(delimiter))
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// handleDeclareTable records a table that does not exist on storage yet. This
|
||||
// is what a Lance client calls on CREATE TABLE, before it writes any data.
|
||||
func (s *Server) handleDeclareTable(w http.ResponseWriter, r *http.Request) {
|
||||
id, _, ok := routeIdentifier(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var req DeclareTableRequest
|
||||
if err := decodeBody(r, &req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
if len(id) == 0 && len(req.ID) > 0 {
|
||||
id = req.ID
|
||||
}
|
||||
if !checkBodyIdentifier(w, r, id, req.ID) {
|
||||
return
|
||||
}
|
||||
bucket, ns, name, err := id.table()
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
location := strings.TrimSuffix(req.Location, "/")
|
||||
if location == "" {
|
||||
location = tableLocation(bucket, ns, name)
|
||||
} else if err := confineLocation(bucket, location); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.createTable(r, bucket, ns, name, location); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
if err := s.writeMarker(r, location, reservedMarker); err != nil {
|
||||
// The catalog entry is the authority; the marker only mirrors it for
|
||||
// clients that read the storage prefix directly.
|
||||
glog.V(1).Infof("lance: could not write %s for %s: %v", reservedMarker, location, err)
|
||||
}
|
||||
// Declaring a name that was deregistered brings it back, the same way
|
||||
// registering it does.
|
||||
if err := s.removeMarker(r, location, deregisteredMarker); err != nil {
|
||||
glog.V(2).Infof("lance: could not clear %s for %s: %v", deregisteredMarker, location, err)
|
||||
}
|
||||
|
||||
options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials))
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
|
||||
return
|
||||
}
|
||||
// Properties are not persisted for a table, so neither response carries
|
||||
// them: null says "this catalog does not keep them", where echoing the
|
||||
// request back or answering {} would say they were stored and are empty.
|
||||
writeJSON(w, http.StatusOK, DeclareTableResponse{
|
||||
Location: location,
|
||||
StorageOptions: options,
|
||||
})
|
||||
}
|
||||
|
||||
// handleDescribeTable resolves a table to a location, and to credentials when
|
||||
// the caller asks for them.
|
||||
func (s *Server) handleDescribeTable(w http.ResponseWriter, r *http.Request) {
|
||||
id, _, ok := routeIdentifier(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var req DescribeTableRequest
|
||||
if err := decodeBody(r, &req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
if len(id) == 0 && len(req.ID) > 0 {
|
||||
id = req.ID
|
||||
}
|
||||
if !checkBodyIdentifier(w, r, id, req.ID) {
|
||||
return
|
||||
}
|
||||
bucket, ns, name, err := id.table()
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
if req.Version != nil || req.Tag != "" || req.Branch != "" {
|
||||
writeError(w, r, http.StatusNotImplemented, codeUnsupported,
|
||||
"this namespace does not resolve versions, tags or branches; the dataset owns them")
|
||||
return
|
||||
}
|
||||
|
||||
table, err := s.loadLanceTable(r, bucket, ns, name)
|
||||
if err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
location := table.location
|
||||
|
||||
deregistered, hasData, err := s.datasetState(r, location)
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
|
||||
return
|
||||
}
|
||||
if deregistered {
|
||||
writeError(w, r, http.StatusNotFound, codeTableNotFound, "table is deregistered")
|
||||
return
|
||||
}
|
||||
|
||||
options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials))
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
resp := DescribeTableResponse{
|
||||
Location: location,
|
||||
}
|
||||
if len(options) > 0 {
|
||||
resp.StorageOptions = options
|
||||
}
|
||||
if wants(r, "with_table_uri", req.WithTableURI) {
|
||||
resp.TableURI = location
|
||||
}
|
||||
if wants(r, "check_declared", req.CheckDeclared) {
|
||||
onlyDeclared := !hasData
|
||||
resp.IsOnlyDeclared = &onlyDeclared
|
||||
}
|
||||
if wants(r, "load_detailed_metadata", req.LoadDetailedMetadata) {
|
||||
resp.Table = name
|
||||
resp.Namespace = ns
|
||||
}
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// handleTableExists answers with the status code and no body.
|
||||
func (s *Server) handleTableExists(w http.ResponseWriter, r *http.Request) {
|
||||
id, _, ok := routeIdentifier(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var req TableExistsRequest
|
||||
if err := decodeBody(r, &req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
if len(id) == 0 && len(req.ID) > 0 {
|
||||
id = req.ID
|
||||
}
|
||||
if !checkBodyIdentifier(w, r, id, req.ID) {
|
||||
return
|
||||
}
|
||||
bucket, ns, name, err := id.table()
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
table, err := s.loadLanceTable(r, bucket, ns, name)
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
writeError(w, r, http.StatusNotFound, codeTableNotFound, "table does not exist")
|
||||
return
|
||||
}
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
deregistered, _, err := s.datasetState(r, table.location)
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
|
||||
return
|
||||
}
|
||||
if deregistered {
|
||||
writeError(w, r, http.StatusNotFound, codeTableNotFound, "table is deregistered")
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
// handleRegisterTable points a table name at an existing dataset.
|
||||
func (s *Server) handleRegisterTable(w http.ResponseWriter, r *http.Request) {
|
||||
id, _, ok := routeIdentifier(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var req RegisterTableRequest
|
||||
if err := decodeBody(r, &req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
if len(id) == 0 && len(req.ID) > 0 {
|
||||
id = req.ID
|
||||
}
|
||||
if !checkBodyIdentifier(w, r, id, req.ID) {
|
||||
return
|
||||
}
|
||||
bucket, ns, name, err := id.table()
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
location := strings.TrimSuffix(req.Location, "/")
|
||||
if location == "" {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, "location is required")
|
||||
return
|
||||
}
|
||||
if err := confineLocation(bucket, location); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
mode := normalizeMode(req.Mode, modeCreate)
|
||||
if mode != modeCreate && mode != modeOverwrite {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, "mode must be Create or Overwrite")
|
||||
return
|
||||
}
|
||||
|
||||
existing, err := s.loadLanceTable(r, bucket, ns, name)
|
||||
switch {
|
||||
case err != nil && !isNotFound(err):
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
case err == nil:
|
||||
// A deregistered table is absent as far as the spec is concerned, so
|
||||
// registering over it is a re-registration rather than a conflict.
|
||||
deregistered, _, stateErr := s.datasetState(r, existing.location)
|
||||
if stateErr != nil {
|
||||
writeError(w, r, http.StatusInternalServerError, codeInternal, stateErr.Error())
|
||||
return
|
||||
}
|
||||
if !deregistered && mode == modeCreate {
|
||||
writeError(w, r, http.StatusConflict, codeTableAlreadyExists, "table already exists")
|
||||
return
|
||||
}
|
||||
if existing.location != location {
|
||||
// Repointing the name at another dataset is an update. Dropping and
|
||||
// recreating the entry would take the old dataset's files with it,
|
||||
// because the entry is the directory holding them.
|
||||
if err := s.repointTable(r, bucket, ns, name, location, existing.versionToken); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
default:
|
||||
if err := s.createTable(r, bucket, ns, name, location); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Registering a deregistered dataset brings it back.
|
||||
if err := s.removeMarker(r, location, deregisteredMarker); err != nil {
|
||||
glog.V(2).Infof("lance: could not clear %s for %s: %v", deregisteredMarker, location, err)
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, RegisterTableResponse{
|
||||
Location: location,
|
||||
Properties: normalizeProperties(req.Properties),
|
||||
})
|
||||
}
|
||||
|
||||
// handleDeregisterTable forgets a table without touching its data.
|
||||
func (s *Server) handleDeregisterTable(w http.ResponseWriter, r *http.Request) {
|
||||
id, _, ok := routeIdentifier(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var req DeregisterTableRequest
|
||||
if err := decodeBody(r, &req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
if len(id) == 0 && len(req.ID) > 0 {
|
||||
id = req.ID
|
||||
}
|
||||
if !checkBodyIdentifier(w, r, id, req.ID) {
|
||||
return
|
||||
}
|
||||
bucket, ns, name, err := id.table()
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
table, err := s.loadLanceTable(r, bucket, ns, name)
|
||||
if err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Deregistering is a state, not a deletion. Dropping the catalog entry would
|
||||
// take the dataset with it, because the entry is the dataset directory and
|
||||
// DeleteTable purges what it holds. The marker is what hides the table, and
|
||||
// it is also what a directory-catalog client reads.
|
||||
if err := s.writeMarker(r, table.location, deregisteredMarker); err != nil {
|
||||
writeError(w, r, http.StatusInternalServerError, codeInternal,
|
||||
"could not mark the table deregistered: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, DeregisterTableResponse{
|
||||
ID: append(append([]string{bucket}, ns...), name),
|
||||
Location: table.location,
|
||||
Properties: map[string]string{},
|
||||
})
|
||||
}
|
||||
|
||||
// handleDropTable removes the table and its data.
|
||||
func (s *Server) handleDropTable(w http.ResponseWriter, r *http.Request) {
|
||||
id, _, ok := routeIdentifier(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var req DropTableRequest
|
||||
if err := decodeBody(r, &req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
if len(id) == 0 && len(req.ID) > 0 {
|
||||
id = req.ID
|
||||
}
|
||||
if !checkBodyIdentifier(w, r, id, req.ID) {
|
||||
return
|
||||
}
|
||||
bucket, ns, name, err := id.table()
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
table, err := s.loadLanceTable(r, bucket, ns, name)
|
||||
if err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
if err := s.dropTableEntry(r, bucket, ns, name); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, DropTableResponse{
|
||||
ID: append(append([]string{bucket}, ns...), name),
|
||||
Location: table.location,
|
||||
Properties: map[string]string{},
|
||||
})
|
||||
}
|
||||
|
||||
// handleRenameTable moves a table's catalog entry. The dataset stays put, which
|
||||
// is what the storage layer already does for a renamed Iceberg table.
|
||||
func (s *Server) handleRenameTable(w http.ResponseWriter, r *http.Request) {
|
||||
id, _, ok := routeIdentifier(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var req RenameTableRequest
|
||||
if err := decodeBody(r, &req); err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
if len(id) == 0 && len(req.ID) > 0 {
|
||||
id = req.ID
|
||||
}
|
||||
if !checkBodyIdentifier(w, r, id, req.ID) {
|
||||
return
|
||||
}
|
||||
bucket, ns, name, err := id.table()
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return
|
||||
}
|
||||
destBucket, destNS, destName, err := identifier(req.NewID).table()
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, "new_id: "+err.Error())
|
||||
return
|
||||
}
|
||||
if destBucket != bucket {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput,
|
||||
"a table cannot move between table buckets")
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := s.loadLanceTable(r, bucket, ns, name); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
renameReq := &s3tables.RenameTableRequest{
|
||||
TableBucketARN: bucketARN(bucket),
|
||||
SourceNamespace: ns,
|
||||
SourceName: name,
|
||||
DestNamespace: destNS,
|
||||
DestName: destName,
|
||||
}
|
||||
var renameResp s3tables.RenameTableResponse
|
||||
if err := s.execute(r, "RenameTable", renameReq, &renameResp); err != nil {
|
||||
writeStorageError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, RenameTableResponse{})
|
||||
}
|
||||
|
||||
// lanceTable is the catalog's record of one Lance table.
|
||||
type lanceTable struct {
|
||||
location string
|
||||
versionToken string
|
||||
}
|
||||
|
||||
// loadLanceTable reads a table and refuses one that is not a Lance table, so a
|
||||
// Lance client never resolves an Iceberg table's location and writes over it.
|
||||
func (s *Server) loadLanceTable(r *http.Request, bucket string, ns []string, name string) (*lanceTable, error) {
|
||||
req := &s3tables.GetTableRequest{
|
||||
TableBucketARN: bucketARN(bucket),
|
||||
Namespace: ns,
|
||||
Name: name,
|
||||
}
|
||||
var resp s3tables.GetTableResponse
|
||||
if err := s.execute(r, "GetTable", req, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.Format != s3tables.FormatLance {
|
||||
return nil, &s3tables.S3TablesError{
|
||||
Type: s3tables.ErrCodeNoSuchTable,
|
||||
Message: "table " + name + " is not a lance table",
|
||||
}
|
||||
}
|
||||
location := resp.MetadataLocation
|
||||
if location == "" {
|
||||
location = tableLocation(bucket, resp.Namespace, resp.Name)
|
||||
}
|
||||
return &lanceTable{location: location, versionToken: resp.VersionToken}, nil
|
||||
}
|
||||
|
||||
func (s *Server) createTable(r *http.Request, bucket string, ns []string, name, location string) error {
|
||||
req := &s3tables.CreateTableRequest{
|
||||
TableBucketARN: bucketARN(bucket),
|
||||
Namespace: ns,
|
||||
Name: name,
|
||||
Format: s3tables.FormatLance,
|
||||
// A Lance table has no metadata file, so the location the catalog stores
|
||||
// is the dataset root itself.
|
||||
MetadataLocation: location,
|
||||
}
|
||||
var resp s3tables.CreateTableResponse
|
||||
return s.execute(r, "CreateTable", req, &resp)
|
||||
}
|
||||
|
||||
// repointTable moves an existing entry to another dataset location without
|
||||
// touching either dataset's files.
|
||||
func (s *Server) repointTable(r *http.Request, bucket string, ns []string, name, location, versionToken string) error {
|
||||
req := &s3tables.UpdateTableRequest{
|
||||
TableBucketARN: bucketARN(bucket),
|
||||
Namespace: ns,
|
||||
Name: name,
|
||||
VersionToken: versionToken,
|
||||
MetadataLocation: location,
|
||||
}
|
||||
var resp s3tables.UpdateTableResponse
|
||||
return s.execute(r, "UpdateTable", req, &resp)
|
||||
}
|
||||
|
||||
// dropTableEntry removes the catalog entry and the dataset under it. Only
|
||||
// DropTable wants this; deregistering and repointing must leave the files.
|
||||
func (s *Server) dropTableEntry(r *http.Request, bucket string, ns []string, name string) error {
|
||||
req := &s3tables.DeleteTableRequest{
|
||||
TableBucketARN: bucketARN(bucket),
|
||||
Namespace: ns,
|
||||
Name: name,
|
||||
}
|
||||
return s.execute(r, "DeleteTable", req, nil)
|
||||
}
|
||||
417
weed/s3api/lance/handlers_test.go
Normal file
417
weed/s3api/lance/handlers_test.go
Normal file
@@ -0,0 +1,417 @@
|
||||
package lance
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
|
||||
)
|
||||
|
||||
// openAuthenticator stands in for a gateway with no IAM configured, which is the
|
||||
// mode the namespace falls open in.
|
||||
type openAuthenticator struct{}
|
||||
|
||||
func (openAuthenticator) AuthenticateRequest(*http.Request) (string, interface{}, s3err.ErrorCode) {
|
||||
return s3tables.DefaultAccountID, nil, s3err.ErrNone
|
||||
}
|
||||
|
||||
func (openAuthenticator) DefaultAllow() bool { return true }
|
||||
|
||||
type testHarness struct {
|
||||
router *mux.Router
|
||||
filer *s3tablestest.MemFiler
|
||||
admin *s3tables.Manager
|
||||
server *Server
|
||||
}
|
||||
|
||||
func newTestHarness(t *testing.T) *testHarness {
|
||||
t.Helper()
|
||||
filer := s3tablestest.Start(t)
|
||||
server := NewServer(s3tables.NewManagerClient(filer.Client), openAuthenticator{})
|
||||
server.SetS3Endpoint("http://127.0.0.1:8333")
|
||||
server.SetS3Region(s3tables.DefaultRegion)
|
||||
router := mux.NewRouter().SkipClean(true)
|
||||
server.RegisterRoutes(router)
|
||||
|
||||
// Table buckets are created by an operator, not by a namespace client, so
|
||||
// the harness seeds them the way the shell and admin console would.
|
||||
admin := s3tables.NewManager()
|
||||
admin.SetTrusted(true)
|
||||
return &testHarness{router: router, filer: filer, admin: admin, server: server}
|
||||
}
|
||||
|
||||
// createBucket seeds a table bucket the Lance namespace can then be pointed at.
|
||||
// An empty format strips the declaration afterwards, which is the only way to
|
||||
// get the shape a bucket made before formats existed has: one that still
|
||||
// accepts either format.
|
||||
func (h *testHarness) createBucket(t *testing.T, name, format string) {
|
||||
t.Helper()
|
||||
var resp s3tables.CreateTableBucketResponse
|
||||
declared := format
|
||||
if declared == "" {
|
||||
declared = s3tables.FormatIceberg
|
||||
}
|
||||
err := h.admin.Execute(t.Context(), s3tables.NewManagerClient(h.filer.Client), "CreateTableBucket",
|
||||
&s3tables.CreateTableBucketRequest{Name: name, Format: declared}, &resp, "")
|
||||
if err != nil {
|
||||
t.Fatalf("create table bucket %s: %v", name, err)
|
||||
}
|
||||
if format == "" {
|
||||
h.undeclareBucket(t, name)
|
||||
}
|
||||
}
|
||||
|
||||
// undeclareBucket removes a bucket's format from its metadata, ageing it back to
|
||||
// what the filer holds for a bucket created before the field existed.
|
||||
func (h *testHarness) undeclareBucket(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
entry := h.filer.Get(s3tables.TablesPath, name)
|
||||
if entry == nil {
|
||||
t.Fatalf("table bucket %s is not in the filer", name)
|
||||
}
|
||||
var metadata map[string]any
|
||||
if err := json.Unmarshal(entry.Extended[s3tables.ExtendedKeyMetadata], &metadata); err != nil {
|
||||
t.Fatalf("read bucket metadata: %v", err)
|
||||
}
|
||||
delete(metadata, "format")
|
||||
updated, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
t.Fatalf("write bucket metadata: %v", err)
|
||||
}
|
||||
extended := map[string][]byte{}
|
||||
for key, value := range entry.Extended {
|
||||
extended[key] = value
|
||||
}
|
||||
extended[s3tables.ExtendedKeyMetadata] = updated
|
||||
h.filer.Put(s3tables.TablesPath, name, extended)
|
||||
}
|
||||
|
||||
func (h *testHarness) bucketARN(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
arn, err := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3tables.DefaultAccountID, name)
|
||||
if err != nil {
|
||||
t.Fatalf("build arn: %v", err)
|
||||
}
|
||||
return arn
|
||||
}
|
||||
|
||||
func (h *testHarness) do(t *testing.T, method, target, body string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
var reader *strings.Reader
|
||||
if body == "" {
|
||||
reader = strings.NewReader("")
|
||||
} else {
|
||||
reader = strings.NewReader(body)
|
||||
}
|
||||
req := httptest.NewRequest(method, target, reader)
|
||||
recorder := httptest.NewRecorder()
|
||||
h.router.ServeHTTP(recorder, req)
|
||||
return recorder
|
||||
}
|
||||
|
||||
func (h *testHarness) mustDo(t *testing.T, method, target, body string, want int) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
recorder := h.do(t, method, target, body)
|
||||
if recorder.Code != want {
|
||||
t.Fatalf("%s %s = %d (%s), want %d", method, target, recorder.Code, recorder.Body.String(), want)
|
||||
}
|
||||
return recorder
|
||||
}
|
||||
|
||||
func decode[T any](t *testing.T, recorder *httptest.ResponseRecorder) T {
|
||||
t.Helper()
|
||||
var out T
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &out); err != nil {
|
||||
t.Fatalf("decode %s: %v", recorder.Body.String(), err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The lifecycle a Lance client drives: create the namespace, declare the table
|
||||
// before any data exists, resolve it to a location, then deregister and bring it
|
||||
// back by registering the same location.
|
||||
func TestTableLifecycle(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
|
||||
h.createBucket(t, "analytics", s3tables.FormatLance)
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
|
||||
|
||||
declared := decode[DeclareTableResponse](t,
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/declare", `{}`, http.StatusOK))
|
||||
if declared.Location != "s3://analytics/sales/orders" {
|
||||
t.Fatalf("declared location = %q", declared.Location)
|
||||
}
|
||||
// The endpoint is plaintext, so a client that does not get allow_http fails
|
||||
// with what looks like a credential error.
|
||||
if declared.StorageOptions["aws_endpoint"] != "http://127.0.0.1:8333" ||
|
||||
declared.StorageOptions["allow_http"] != "true" {
|
||||
t.Fatalf("storage options = %v", declared.StorageOptions)
|
||||
}
|
||||
|
||||
described := decode[DescribeTableResponse](t,
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/describe?check_declared=true", `{}`, http.StatusOK))
|
||||
if described.Location != declared.Location {
|
||||
t.Fatalf("described location = %q, want %q", described.Location, declared.Location)
|
||||
}
|
||||
if described.IsOnlyDeclared == nil || !*described.IsOnlyDeclared {
|
||||
t.Fatalf("is_only_declared = %v, want true for a table with no data", described.IsOnlyDeclared)
|
||||
}
|
||||
|
||||
listed := decode[ListTablesResponse](t,
|
||||
h.mustDo(t, http.MethodGet, "/v1/namespace/analytics$sales/table/list", "", http.StatusOK))
|
||||
if len(listed.Tables) != 1 || listed.Tables[0] != "analytics$sales$orders" {
|
||||
t.Fatalf("listed tables = %v, want the full identifier", listed.Tables)
|
||||
}
|
||||
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/exists", `{}`, http.StatusOK)
|
||||
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/deregister", `{}`, http.StatusOK)
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/exists", `{}`, http.StatusNotFound)
|
||||
afterDrop := decode[ListTablesResponse](t,
|
||||
h.mustDo(t, http.MethodGet, "/v1/namespace/analytics$sales/table/list", "", http.StatusOK))
|
||||
if len(afterDrop.Tables) != 0 {
|
||||
t.Fatalf("deregistered table still listed: %v", afterDrop.Tables)
|
||||
}
|
||||
// Deregistering preserves the data. The catalog entry is the dataset
|
||||
// directory, so dropping it would take the dataset with it.
|
||||
if h.filer.Get(s3tables.GetNamespacePath("analytics", "sales"), "orders") == nil {
|
||||
t.Fatal("deregister deleted the dataset directory")
|
||||
}
|
||||
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/register",
|
||||
`{"location":"s3://analytics/sales/orders"}`, http.StatusOK)
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/exists", `{}`, http.StatusOK)
|
||||
|
||||
// Dropping is the operation that does remove the data.
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/drop", `{}`, http.StatusOK)
|
||||
if h.filer.Get(s3tables.GetNamespacePath("analytics", "sales"), "orders") != nil {
|
||||
t.Fatal("drop left the dataset directory behind")
|
||||
}
|
||||
}
|
||||
|
||||
// Repointing a registered name at another dataset must not take the dataset it
|
||||
// used to name with it.
|
||||
func TestRegisterOverwriteKeepsTheOldDataset(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
h.createBucket(t, "analytics", s3tables.FormatLance)
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/declare", `{}`, http.StatusOK)
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$archive/declare", `{}`, http.StatusOK)
|
||||
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/register",
|
||||
`{"location":"s3://analytics/sales/archive","mode":"Overwrite"}`, http.StatusOK)
|
||||
|
||||
described := decode[DescribeTableResponse](t,
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/describe", `{}`, http.StatusOK))
|
||||
if described.Location != "s3://analytics/sales/archive" {
|
||||
t.Fatalf("location after repointing = %q", described.Location)
|
||||
}
|
||||
if h.filer.Get(s3tables.GetNamespacePath("analytics", "sales"), "orders") == nil {
|
||||
t.Fatal("repointing deleted the dataset the name used to hold")
|
||||
}
|
||||
}
|
||||
|
||||
// A required list field answers empty rather than null, which a generated
|
||||
// client may decode differently.
|
||||
func TestListAllTablesIsNeverNull(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
h.createBucket(t, "analytics", s3tables.FormatLance)
|
||||
|
||||
body := h.mustDo(t, http.MethodGet, "/v1/table", "", http.StatusOK).Body.String()
|
||||
if strings.Contains(body, `"tables":null`) {
|
||||
t.Fatalf("ListAllTables returned null for a required field: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNamespaceListing(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
h.createBucket(t, "analytics", s3tables.FormatLance)
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$finance/create", `{}`, http.StatusOK)
|
||||
|
||||
roots := decode[ListNamespacesResponse](t,
|
||||
h.mustDo(t, http.MethodGet, "/v1/namespace/$/list", "", http.StatusOK))
|
||||
if len(roots.Namespaces) != 1 || roots.Namespaces[0] != "analytics" {
|
||||
t.Fatalf("root listing = %v, want the table buckets", roots.Namespaces)
|
||||
}
|
||||
|
||||
children := decode[ListNamespacesResponse](t,
|
||||
h.mustDo(t, http.MethodGet, "/v1/namespace/analytics/list", "", http.StatusOK))
|
||||
if len(children.Namespaces) != 2 {
|
||||
t.Fatalf("bucket listing = %v, want two namespaces", children.Namespaces)
|
||||
}
|
||||
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/exists", `{}`, http.StatusOK)
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$nope/exists", `{}`, http.StatusNotFound)
|
||||
|
||||
// Fail mode reports a missing namespace as 400 on this operation, which is
|
||||
// what the spec asks for; Skip reports success.
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$nope/drop", `{}`, http.StatusBadRequest)
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$nope/drop", `{"mode":"Skip"}`, http.StatusNoContent)
|
||||
}
|
||||
|
||||
// Storage flattens a namespace's parts, so creating a$b without a would leave
|
||||
// an intermediate that listing derives from the name and describe denies
|
||||
// exists. The spec asks for NamespaceNotFound, which keeps them consistent.
|
||||
func TestCreateNamespaceRequiresItsParent(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
h.createBucket(t, "analytics", s3tables.FormatLance)
|
||||
|
||||
recorder := h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$missing$child/create", `{}`, http.StatusNotFound)
|
||||
if got := decode[errorResponse](t, recorder); got.Code != codeNamespaceNotFound {
|
||||
t.Fatalf("error code = %d, want %d", got.Code, codeNamespaceNotFound)
|
||||
}
|
||||
|
||||
// With the parent in place the child is fine, and both are then listed and
|
||||
// describable.
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$parent/create", `{}`, http.StatusOK)
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$parent$child/create", `{}`, http.StatusOK)
|
||||
|
||||
listed := decode[ListNamespacesResponse](t,
|
||||
h.mustDo(t, http.MethodGet, "/v1/namespace/analytics/list", "", http.StatusOK))
|
||||
for _, name := range listed.Namespaces {
|
||||
path := "/v1/namespace/analytics$" + name
|
||||
h.mustDo(t, http.MethodPost, path+"/exists", `{}`, http.StatusOK)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateNamespaceModes(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
h.createBucket(t, "analytics", s3tables.FormatLance)
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
|
||||
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusConflict)
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{"mode":"ExistOk"}`, http.StatusOK)
|
||||
// snake_case is the other spelling the spec accepts.
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{"mode":"exist_ok"}`, http.StatusOK)
|
||||
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/$/create", `{}`, http.StatusBadRequest)
|
||||
}
|
||||
|
||||
// A Lance client must never resolve an Iceberg table's location, or it would
|
||||
// write a dataset over a table another engine owns.
|
||||
func TestIcebergTablesAreInvisible(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
// Undeclared, because a bucket that declares one format cannot hold the
|
||||
// other - and mixing is exactly what this test needs to prove is hidden.
|
||||
h.createBucket(t, "analytics", "")
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$vectors/declare", `{}`, http.StatusOK)
|
||||
|
||||
manager := h.admin
|
||||
var created s3tables.CreateTableResponse
|
||||
err := manager.Execute(t.Context(), s3tables.NewManagerClient(h.filer.Client), "CreateTable", &s3tables.CreateTableRequest{
|
||||
TableBucketARN: h.bucketARN(t, "analytics"),
|
||||
Namespace: []string{"sales"},
|
||||
Name: "ledger",
|
||||
Format: s3tables.FormatIceberg,
|
||||
MetadataLocation: "s3://analytics/sales/ledger/metadata/v1.metadata.json",
|
||||
}, &created, "")
|
||||
if err != nil {
|
||||
t.Fatalf("create iceberg table: %v", err)
|
||||
}
|
||||
|
||||
listed := decode[ListTablesResponse](t,
|
||||
h.mustDo(t, http.MethodGet, "/v1/namespace/analytics$sales/table/list", "", http.StatusOK))
|
||||
if len(listed.Tables) != 1 || listed.Tables[0] != "analytics$sales$vectors" {
|
||||
t.Fatalf("listing = %v, want only the lance table", listed.Tables)
|
||||
}
|
||||
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$ledger/describe", `{}`, http.StatusNotFound)
|
||||
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$ledger/exists", `{}`, http.StatusNotFound)
|
||||
|
||||
// Declaring over the Iceberg table must not quietly succeed and hand the
|
||||
// Lance client a directory another format owns.
|
||||
recorder := h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$ledger/declare", `{}`, http.StatusConflict)
|
||||
if got := decode[errorResponse](t, recorder); got.Code != codeTableAlreadyExists {
|
||||
t.Fatalf("error code = %d, want %d", got.Code, codeTableAlreadyExists)
|
||||
}
|
||||
}
|
||||
|
||||
// The route and the body naming different objects is a bad request, not a silent
|
||||
// preference for one of them.
|
||||
func TestRouteAndBodyMustAgree(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
h.createBucket(t, "analytics", s3tables.FormatLance)
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
|
||||
|
||||
recorder := h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/declare",
|
||||
`{"id":["analytics","sales","other"]}`, http.StatusBadRequest)
|
||||
if got := decode[errorResponse](t, recorder); got.Code != codeInvalidInput {
|
||||
t.Fatalf("error code = %d, want %d", got.Code, codeInvalidInput)
|
||||
}
|
||||
}
|
||||
|
||||
// A client-supplied location must resolve inside its own bucket. Without this,
|
||||
// a "../"-laden or cross-bucket location escapes when the marker path is joined
|
||||
// under /buckets, letting the request reach another tenant's namespace, or
|
||||
// outside /buckets entirely, and auto-create the parent directories on the way.
|
||||
func TestLocationOutsideBucketIsRejected(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
location string
|
||||
}{
|
||||
{"another bucket", "s3://victim/secret"},
|
||||
{"traversal into another bucket", "s3://analytics/../victim/secret"},
|
||||
{"traversal above the buckets root", "s3://analytics/../../etc/cron.d"},
|
||||
{"not an s3 uri", "../../etc/cron.d"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dir := s3tables.TableDataDirFromMetadataLocation(tc.location)
|
||||
body := `{"location":"` + tc.location + `"}`
|
||||
|
||||
// declare writes .lance-reserved at the location: rejection must leave
|
||||
// nothing behind at the escaped directory.
|
||||
t.Run("declare", func(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
h.createBucket(t, "analytics", s3tables.FormatLance)
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
|
||||
|
||||
recorder := h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/declare", body, http.StatusBadRequest)
|
||||
if got := decode[errorResponse](t, recorder); got.Code != codeInvalidInput {
|
||||
t.Fatalf("error code = %d, want %d", got.Code, codeInvalidInput)
|
||||
}
|
||||
if dir != "" && h.filer.Get(dir, reservedMarker) != nil {
|
||||
t.Fatalf("declare wrote %s into %s despite rejection", reservedMarker, dir)
|
||||
}
|
||||
})
|
||||
|
||||
// register removes .lance-deregistered at the location: rejection must
|
||||
// leave a victim's marker in place rather than un-hiding their table.
|
||||
t.Run("register", func(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
h.createBucket(t, "analytics", s3tables.FormatLance)
|
||||
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
|
||||
if dir != "" {
|
||||
h.filer.PutFile(dir, deregisteredMarker, time.Unix(1, 0))
|
||||
}
|
||||
|
||||
recorder := h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/register", body, http.StatusBadRequest)
|
||||
if got := decode[errorResponse](t, recorder); got.Code != codeInvalidInput {
|
||||
t.Fatalf("error code = %d, want %d", got.Code, codeInvalidInput)
|
||||
}
|
||||
if dir != "" && h.filer.Get(dir, deregisteredMarker) == nil {
|
||||
t.Fatalf("register removed %s from %s despite rejection", deregisteredMarker, dir)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The data plane needs Lance format support that does not exist in Go, so it
|
||||
// answers with the spec's own Unsupported code rather than a bare 404.
|
||||
func TestDataPlaneIsUnsupported(t *testing.T) {
|
||||
h := newTestHarness(t)
|
||||
recorder := h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/query", `{}`, http.StatusNotImplemented)
|
||||
if got := decode[errorResponse](t, recorder); got.Code != codeUnsupported {
|
||||
t.Fatalf("error code = %d, want %d", got.Code, codeUnsupported)
|
||||
}
|
||||
}
|
||||
84
weed/s3api/lance/identifier.go
Normal file
84
weed/s3api/lance/identifier.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package lance
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// defaultDelimiter joins the parts of a Lance string identifier when the caller
|
||||
// does not pass ?delimiter=. An id equal to the delimiter is the root namespace,
|
||||
// so /v1/namespace/$/list lists the root's children.
|
||||
const defaultDelimiter = "$"
|
||||
|
||||
// identifier is a Lance object identifier: zero parts is the root namespace, one
|
||||
// part names a table bucket, and the rest are namespace parts with a table name
|
||||
// last. The mapping onto storage is bucket / namespace / table, which is the
|
||||
// three-level shape Lance clients already use.
|
||||
type identifier []string
|
||||
|
||||
func requestDelimiter(r *http.Request) string {
|
||||
if d := r.URL.Query().Get("delimiter"); d != "" {
|
||||
return d
|
||||
}
|
||||
return defaultDelimiter
|
||||
}
|
||||
|
||||
// parseIdentifier decodes the {id} route variable. Empty parts are rejected
|
||||
// rather than dropped so that "a$$b" cannot silently resolve to "a$b".
|
||||
func parseIdentifier(encoded, delimiter string) (identifier, error) {
|
||||
if encoded == "" || encoded == delimiter {
|
||||
return nil, nil
|
||||
}
|
||||
parts := strings.Split(encoded, delimiter)
|
||||
for _, part := range parts {
|
||||
if part == "" {
|
||||
return nil, fmt.Errorf("identifier %q has an empty part", encoded)
|
||||
}
|
||||
}
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
func (id identifier) String(delimiter string) string {
|
||||
if len(id) == 0 {
|
||||
return delimiter
|
||||
}
|
||||
return strings.Join(id, delimiter)
|
||||
}
|
||||
|
||||
// namespace splits a namespace identifier into the table bucket and the parts of
|
||||
// the namespace inside it. The root and a bare bucket both return an empty
|
||||
// namespace, so callers check the bucket to tell them apart.
|
||||
func (id identifier) namespace() (bucket string, ns []string) {
|
||||
if len(id) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return id[0], id[1:]
|
||||
}
|
||||
|
||||
// table splits a table identifier. A table needs a bucket, at least one
|
||||
// namespace part and a name, because storage has no unnamespaced tables.
|
||||
func (id identifier) table() (bucket string, ns []string, name string, err error) {
|
||||
if len(id) < 3 {
|
||||
return "", nil, "", fmt.Errorf("table identifier needs a bucket, a namespace and a name")
|
||||
}
|
||||
return id[0], id[1 : len(id)-1], id[len(id)-1], nil
|
||||
}
|
||||
|
||||
// matchesBody reports whether an identifier carried in the request body agrees
|
||||
// with the one in the route. The spec requires 400 when both are present and
|
||||
// differ, and requires the route to win when the body omits it.
|
||||
func (id identifier) matchesBody(body []string) bool {
|
||||
if len(body) == 0 {
|
||||
return true
|
||||
}
|
||||
if len(body) != len(id) {
|
||||
return false
|
||||
}
|
||||
for i := range body {
|
||||
if body[i] != id[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
104
weed/s3api/lance/identifier_test.go
Normal file
104
weed/s3api/lance/identifier_test.go
Normal file
@@ -0,0 +1,104 @@
|
||||
package lance
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseIdentifier(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
encoded string
|
||||
delimiter string
|
||||
want []string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "root is the delimiter", encoded: "$", delimiter: "$"},
|
||||
{name: "empty is the root", encoded: "", delimiter: "$"},
|
||||
{name: "bucket", encoded: "analytics", delimiter: "$", want: []string{"analytics"}},
|
||||
{name: "namespace", encoded: "analytics$sales", delimiter: "$", want: []string{"analytics", "sales"}},
|
||||
{name: "table", encoded: "analytics$sales$orders", delimiter: "$", want: []string{"analytics", "sales", "orders"}},
|
||||
{name: "custom delimiter", encoded: "a.b.c", delimiter: ".", want: []string{"a", "b", "c"}},
|
||||
{name: "empty part is rejected", encoded: "a$$b", delimiter: "$", wantErr: true},
|
||||
{name: "trailing delimiter is rejected", encoded: "a$", delimiter: "$", wantErr: true},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got, err := parseIdentifier(c.encoded, c.delimiter)
|
||||
if c.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("parseIdentifier(%q) error = nil, want an error", c.encoded)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("parseIdentifier(%q) error = %v", c.encoded, err)
|
||||
}
|
||||
if len(got) != len(c.want) {
|
||||
t.Fatalf("parseIdentifier(%q) = %v, want %v", c.encoded, got, c.want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != c.want[i] {
|
||||
t.Fatalf("parseIdentifier(%q) = %v, want %v", c.encoded, got, c.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdentifierNamespace(t *testing.T) {
|
||||
bucket, ns := identifier(nil).namespace()
|
||||
if bucket != "" || len(ns) != 0 {
|
||||
t.Fatalf("root namespace() = %q %v, want empty", bucket, ns)
|
||||
}
|
||||
|
||||
bucket, ns = identifier{"analytics"}.namespace()
|
||||
if bucket != "analytics" || len(ns) != 0 {
|
||||
t.Fatalf("bucket namespace() = %q %v", bucket, ns)
|
||||
}
|
||||
|
||||
bucket, ns = identifier{"analytics", "sales", "eu"}.namespace()
|
||||
if bucket != "analytics" || len(ns) != 2 || ns[0] != "sales" || ns[1] != "eu" {
|
||||
t.Fatalf("nested namespace() = %q %v", bucket, ns)
|
||||
}
|
||||
}
|
||||
|
||||
// A table always needs a bucket, a namespace and a name: storage has no
|
||||
// unnamespaced tables, so a two-part identifier is a client error rather than a
|
||||
// table at the top of a bucket.
|
||||
func TestIdentifierTable(t *testing.T) {
|
||||
if _, _, _, err := (identifier{"analytics", "orders"}).table(); err == nil {
|
||||
t.Fatal("table() on a two-part identifier error = nil, want an error")
|
||||
}
|
||||
|
||||
bucket, ns, name, err := identifier{"analytics", "sales", "eu", "orders"}.table()
|
||||
if err != nil {
|
||||
t.Fatalf("table() error = %v", err)
|
||||
}
|
||||
if bucket != "analytics" || name != "orders" || len(ns) != 2 || ns[0] != "sales" || ns[1] != "eu" {
|
||||
t.Fatalf("table() = %q %v %q", bucket, ns, name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdentifierMatchesBody(t *testing.T) {
|
||||
id := identifier{"a", "b", "c"}
|
||||
if !id.matchesBody(nil) {
|
||||
t.Fatal("an absent body identifier must defer to the route")
|
||||
}
|
||||
if !id.matchesBody([]string{"a", "b", "c"}) {
|
||||
t.Fatal("an equal body identifier must match")
|
||||
}
|
||||
if id.matchesBody([]string{"a", "b"}) {
|
||||
t.Fatal("a shorter body identifier must not match")
|
||||
}
|
||||
if id.matchesBody([]string{"a", "b", "d"}) {
|
||||
t.Fatal("a different body identifier must not match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdentifierString(t *testing.T) {
|
||||
if got := identifier(nil).String("$"); got != "$" {
|
||||
t.Fatalf("root String() = %q, want %q", got, "$")
|
||||
}
|
||||
if got := (identifier{"a", "b"}).String("$"); got != "a$b" {
|
||||
t.Fatalf("String() = %q, want %q", got, "a$b")
|
||||
}
|
||||
}
|
||||
93
weed/s3api/lance/markers.go
Normal file
93
weed/s3api/lance/markers.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package lance
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
// Marker files the Lance Directory Catalog reads. Writing them beside the
|
||||
// dataset keeps a client that bypasses this namespace, and lists the storage
|
||||
// prefix directly, seeing the same table states the catalog reports.
|
||||
const (
|
||||
reservedMarker = ".lance-reserved"
|
||||
deregisteredMarker = ".lance-deregistered"
|
||||
versionsDir = "_versions"
|
||||
)
|
||||
|
||||
// datasetDir maps a table location onto the filer directory holding it.
|
||||
func datasetDir(location string) string {
|
||||
return s3tables.TableDataDirFromMetadataLocation(location)
|
||||
}
|
||||
|
||||
func (s *Server) entryExists(r *http.Request, dir, name string) (bool, error) {
|
||||
found := false
|
||||
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
_, lookupErr := filer_pb.LookupEntry(r.Context(), client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
if lookupErr == filer_pb.ErrNotFound {
|
||||
return nil
|
||||
}
|
||||
if lookupErr != nil {
|
||||
return lookupErr
|
||||
}
|
||||
found = true
|
||||
return nil
|
||||
})
|
||||
return found, err
|
||||
}
|
||||
|
||||
// writeMarker drops a zero-length marker into the dataset directory. The
|
||||
// directory has to exist already: a table declared through this namespace has
|
||||
// one, and a table registered at a location that does not yet exist has no
|
||||
// storage to mark.
|
||||
func (s *Server) writeMarker(r *http.Request, location, name string) error {
|
||||
dir := datasetDir(location)
|
||||
if dir == "" {
|
||||
return nil
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
return s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return filer_pb.CreateEntry(r.Context(), client, &filer_pb.CreateEntryRequest{
|
||||
Directory: dir,
|
||||
Entry: &filer_pb.Entry{
|
||||
Name: name,
|
||||
Attributes: &filer_pb.FuseAttributes{
|
||||
Mtime: now,
|
||||
Crtime: now,
|
||||
FileMode: uint32(0644),
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) removeMarker(r *http.Request, location, name string) error {
|
||||
dir := datasetDir(location)
|
||||
if dir == "" {
|
||||
return nil
|
||||
}
|
||||
return s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return filer_pb.DoRemove(r.Context(), client, dir, name, true, false, true, false, nil)
|
||||
})
|
||||
}
|
||||
|
||||
// datasetState reports the two things the spec asks about a table's storage:
|
||||
// whether it has been deregistered, and whether it holds data yet. A dataset
|
||||
// holds data once it has a version manifest, which is how a directory catalog
|
||||
// decides the same question.
|
||||
func (s *Server) datasetState(r *http.Request, location string) (deregistered, hasData bool, err error) {
|
||||
dir := datasetDir(location)
|
||||
if dir == "" {
|
||||
return false, false, nil
|
||||
}
|
||||
if deregistered, err = s.entryExists(r, dir, deregisteredMarker); err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
hasData, err = s.entryExists(r, dir, versionsDir)
|
||||
return deregistered, hasData, err
|
||||
}
|
||||
188
weed/s3api/lance/server.go
Normal file
188
weed/s3api/lance/server.go
Normal file
@@ -0,0 +1,188 @@
|
||||
package lance
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
// FilerClient provides access to the filer for storage operations.
|
||||
type FilerClient interface {
|
||||
WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error
|
||||
}
|
||||
|
||||
type S3Authenticator interface {
|
||||
AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode)
|
||||
DefaultAllow() bool
|
||||
}
|
||||
|
||||
// VendedCredentials are short-lived S3 credentials scoped to one table.
|
||||
type VendedCredentials struct {
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
SessionToken string
|
||||
Expiration time.Time
|
||||
}
|
||||
|
||||
// CredentialVendor mints credentials limited to a single table's prefix for a
|
||||
// caller the catalog has already authenticated and authorized. A nil result with
|
||||
// no error means the deployment has vending switched off.
|
||||
type CredentialVendor interface {
|
||||
VendTableCredentials(ctx context.Context, principal, bucket, prefix string) (*VendedCredentials, error)
|
||||
}
|
||||
|
||||
// Server implements the Lance Namespace REST spec.
|
||||
type Server struct {
|
||||
filerClient FilerClient
|
||||
tablesManager *s3tables.Manager
|
||||
authenticator S3Authenticator
|
||||
credentialVendor CredentialVendor
|
||||
s3Endpoint string
|
||||
s3Region string
|
||||
}
|
||||
|
||||
// NewServer creates a Lance namespace server over the given filer.
|
||||
func NewServer(filerClient FilerClient, authenticator S3Authenticator) *Server {
|
||||
manager := s3tables.NewManager()
|
||||
// Mirror the S3 port: fall open by default only when the gateway itself is
|
||||
// open, so an authenticated caller still passes the normal permission check.
|
||||
if authenticator != nil {
|
||||
manager.SetDefaultAllow(authenticator.DefaultAllow())
|
||||
}
|
||||
return &Server{
|
||||
filerClient: filerClient,
|
||||
tablesManager: manager,
|
||||
authenticator: authenticator,
|
||||
}
|
||||
}
|
||||
|
||||
// SetCredentialVendor enables storage_options credential vending for clients
|
||||
// that ask for it with vend_credentials.
|
||||
func (s *Server) SetCredentialVendor(vendor CredentialVendor) {
|
||||
s.credentialVendor = vendor
|
||||
}
|
||||
|
||||
// SetS3Endpoint configures the S3 endpoint advertised in storage_options so a
|
||||
// client can reach the dataset without separately discovering the S3 address.
|
||||
func (s *Server) SetS3Endpoint(endpoint string) {
|
||||
s.s3Endpoint = endpoint
|
||||
}
|
||||
|
||||
// SetS3Region configures the region advertised in storage_options.
|
||||
func (s *Server) SetS3Region(region string) {
|
||||
s.s3Region = region
|
||||
}
|
||||
|
||||
// RegisterRoutes registers the Lance Namespace REST routes.
|
||||
//
|
||||
// The spec puts the identifier in the path rather than the body so a reverse
|
||||
// proxy can route and authorize without deserializing the request.
|
||||
func (s *Server) RegisterRoutes(router *mux.Router) {
|
||||
router.Use(loggingMiddleware)
|
||||
|
||||
router.HandleFunc("/v1/namespace/{id}/create", s.Auth(s.handleCreateNamespace)).Methods(http.MethodPost)
|
||||
router.HandleFunc("/v1/namespace/{id}/list", s.Auth(s.handleListNamespaces)).Methods(http.MethodGet)
|
||||
router.HandleFunc("/v1/namespace/{id}/describe", s.Auth(s.handleDescribeNamespace)).Methods(http.MethodPost)
|
||||
router.HandleFunc("/v1/namespace/{id}/drop", s.Auth(s.handleDropNamespace)).Methods(http.MethodPost)
|
||||
router.HandleFunc("/v1/namespace/{id}/exists", s.Auth(s.handleNamespaceExists)).Methods(http.MethodPost)
|
||||
router.HandleFunc("/v1/namespace/{id}/table/list", s.Auth(s.handleListTables)).Methods(http.MethodGet)
|
||||
|
||||
router.HandleFunc("/v1/table", s.Auth(s.handleListAllTables)).Methods(http.MethodGet)
|
||||
router.HandleFunc("/v1/table/{id}/declare", s.Auth(s.handleDeclareTable)).Methods(http.MethodPost)
|
||||
router.HandleFunc("/v1/table/{id}/describe", s.Auth(s.handleDescribeTable)).Methods(http.MethodPost)
|
||||
router.HandleFunc("/v1/table/{id}/exists", s.Auth(s.handleTableExists)).Methods(http.MethodPost)
|
||||
router.HandleFunc("/v1/table/{id}/register", s.Auth(s.handleRegisterTable)).Methods(http.MethodPost)
|
||||
router.HandleFunc("/v1/table/{id}/deregister", s.Auth(s.handleDeregisterTable)).Methods(http.MethodPost)
|
||||
router.HandleFunc("/v1/table/{id}/drop", s.Auth(s.handleDropTable)).Methods(http.MethodPost)
|
||||
router.HandleFunc("/v1/table/{id}/rename", s.Auth(s.handleRenameTable)).Methods(http.MethodPost)
|
||||
|
||||
// The data plane needs Lance format support that does not exist in Go. Say
|
||||
// so with the spec's own code instead of returning a bare 404.
|
||||
for _, action := range []string{"create", "insert", "merge_insert", "update", "delete",
|
||||
"query", "count_rows", "explain_plan", "analyze_plan", "restore",
|
||||
"add_columns", "alter_columns", "drop_columns", "backfill_column",
|
||||
"create_index", "create_scalar_index", "stats", "schema_metadata/update"} {
|
||||
router.HandleFunc("/v1/table/{id}/"+action, s.Auth(s.handleUnsupported)).Methods(http.MethodPost)
|
||||
}
|
||||
// Version ops exist in the spec for stores that cannot order commits
|
||||
// themselves. Ours can: a Lance commit is a put-if-not-exists, and this S3
|
||||
// evaluates that precondition at the object's owner filer under a per-path
|
||||
// lock, so the dataset keeps its own version history and the catalog stays
|
||||
// out of the commit path.
|
||||
for _, action := range []string{
|
||||
"version/create", "version/list", "version/describe", "version/delete",
|
||||
"index/list", "tags/list", "tags/version", "tags/create", "tags/delete", "tags/update",
|
||||
"branches/list", "branches/create", "branches/delete"} {
|
||||
router.HandleFunc("/v1/table/{id}/"+action, s.Auth(s.handleUnsupported)).Methods(http.MethodPost)
|
||||
}
|
||||
|
||||
router.PathPrefix("/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
glog.V(2).Infof("lance: no route for %s %s", r.Method, r.RequestURI)
|
||||
writeError(w, r, http.StatusNotFound, codeUnsupported, "no such operation")
|
||||
})
|
||||
|
||||
glog.V(2).Infof("Registered Lance Namespace routes")
|
||||
}
|
||||
|
||||
func (s *Server) handleUnsupported(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, r, http.StatusNotImplemented, codeUnsupported,
|
||||
"this namespace records table metadata only; run data operations through a Lance client against the table location")
|
||||
}
|
||||
|
||||
func loggingMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
glog.V(2).Infof("lance request: %s %s from %s", r.Method, r.RequestURI, r.RemoteAddr)
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// Auth authenticates the caller and puts the identity in the request context.
|
||||
// The Lance spec maps identity onto the same headers the S3 authenticator
|
||||
// already understands, so SigV4 and bearer tokens both keep working.
|
||||
func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.authenticator == nil {
|
||||
writeError(w, r, http.StatusUnauthorized, codeUnauthenticated, "authentication required")
|
||||
return
|
||||
}
|
||||
|
||||
identityName, identity, errCode := s.authenticator.AuthenticateRequest(r)
|
||||
if errCode != s3err.ErrNone {
|
||||
if !s.authenticator.DefaultAllow() {
|
||||
apiErr := s3err.GetAPIError(errCode)
|
||||
code := codeInternal
|
||||
switch apiErr.HTTPStatusCode {
|
||||
case http.StatusForbidden:
|
||||
code = codePermissionDenied
|
||||
case http.StatusUnauthorized:
|
||||
code = codeUnauthenticated
|
||||
case http.StatusBadRequest:
|
||||
code = codeInvalidInput
|
||||
}
|
||||
writeError(w, r, apiErr.HTTPStatusCode, code, apiErr.Description)
|
||||
return
|
||||
}
|
||||
glog.V(2).Infof("lance: authentication failed (%v) but the gateway is open, proceeding", errCode)
|
||||
}
|
||||
|
||||
if identityName != "" || identity != nil {
|
||||
ctx := r.Context()
|
||||
if identityName != "" {
|
||||
ctx = s3_constants.SetIdentityNameInContext(ctx, identityName)
|
||||
}
|
||||
if identity != nil {
|
||||
ctx = s3_constants.SetIdentityInContext(ctx, identity)
|
||||
}
|
||||
r = r.WithContext(ctx)
|
||||
}
|
||||
|
||||
handler(w, r)
|
||||
}
|
||||
}
|
||||
183
weed/s3api/lance/storage.go
Normal file
183
weed/s3api/lance/storage.go
Normal file
@@ -0,0 +1,183 @@
|
||||
package lance
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
const defaultPageSize = 1000
|
||||
|
||||
// execute runs one S3 Tables operation as the request's authenticated caller.
|
||||
func (s *Server) execute(r *http.Request, operation string, req, resp interface{}) error {
|
||||
identityName := s3_constants.GetIdentityNameFromContext(r)
|
||||
return s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return s.tablesManager.Execute(r.Context(), s3tables.NewManagerClient(client), operation, req, resp, identityName)
|
||||
})
|
||||
}
|
||||
|
||||
func bucketARN(bucket string) string {
|
||||
arn, _ := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3_constants.AccountAdminId, bucket)
|
||||
return arn
|
||||
}
|
||||
|
||||
// maxRequestBody bounds what one call can make the catalog hold. Every request
|
||||
// this surface takes is a small JSON envelope; the largest carries a set of
|
||||
// properties, not data.
|
||||
const maxRequestBody = 4 << 20
|
||||
|
||||
// decodeBody reads an optional JSON request body. Every Lance operation carries
|
||||
// one, but the fields that matter are also in the route, so an empty body is
|
||||
// not an error.
|
||||
func decodeBody(r *http.Request, into interface{}) error {
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, maxRequestBody+1))
|
||||
if err != nil {
|
||||
return fmt.Errorf("read request body: %w", err)
|
||||
}
|
||||
if len(body) > maxRequestBody {
|
||||
return fmt.Errorf("request body is larger than %d bytes", maxRequestBody)
|
||||
}
|
||||
if len(strings.TrimSpace(string(body))) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(body, into); err != nil {
|
||||
return fmt.Errorf("invalid request body: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// routeIdentifier parses the {id} route variable, writing the error response
|
||||
// itself when the identifier is malformed.
|
||||
func routeIdentifier(w http.ResponseWriter, r *http.Request) (identifier, string, bool) {
|
||||
delimiter := requestDelimiter(r)
|
||||
id, err := parseIdentifier(mux.Vars(r)["id"], delimiter)
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
|
||||
return nil, delimiter, false
|
||||
}
|
||||
return id, delimiter, true
|
||||
}
|
||||
|
||||
// checkBodyIdentifier enforces the spec rule that a route and a body naming
|
||||
// different objects is a bad request rather than a silent preference.
|
||||
func checkBodyIdentifier(w http.ResponseWriter, r *http.Request, id identifier, body []string) bool {
|
||||
if id.matchesBody(body) {
|
||||
return true
|
||||
}
|
||||
writeError(w, r, http.StatusBadRequest, codeInvalidInput,
|
||||
"the identifier in the request body does not match the one in the route")
|
||||
return false
|
||||
}
|
||||
|
||||
func pageSize(r *http.Request) int {
|
||||
if raw := r.URL.Query().Get("limit"); raw != "" {
|
||||
if parsed, err := strconv.Atoi(raw); err == nil && parsed > 0 {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return defaultPageSize
|
||||
}
|
||||
|
||||
func boolQuery(r *http.Request, name string) bool {
|
||||
value, err := strconv.ParseBool(r.URL.Query().Get(name))
|
||||
return err == nil && value
|
||||
}
|
||||
|
||||
// wants resolves a tri-state request flag: the query parameter the REST spec
|
||||
// adds, else the body field, else the implementation's own choice.
|
||||
func wants(r *http.Request, name string, body *bool) bool {
|
||||
if boolQuery(r, name) {
|
||||
return true
|
||||
}
|
||||
return body != nil && *body
|
||||
}
|
||||
|
||||
// tableLocation is where a table's dataset lives when the caller does not name
|
||||
// a location. It mirrors the Iceberg catalog's layout so both catalogs put a
|
||||
// table of the same name in the same place.
|
||||
func tableLocation(bucket string, ns []string, name string) string {
|
||||
return fmt.Sprintf("s3://%s/%s/%s", bucket, strings.Join(ns, "."), name)
|
||||
}
|
||||
|
||||
// confineLocation rejects a client-supplied table location that does not resolve
|
||||
// inside the caller's own bucket, the way the Iceberg gateway confines the same
|
||||
// field. The location feeds TableDataDirFromMetadataLocation, which joins it
|
||||
// under /buckets and collapses any "../" segments; an unconfined location lets a
|
||||
// marker write escape into another tenant's namespace.
|
||||
func confineLocation(bucket, location string) error {
|
||||
rest, ok := strings.CutPrefix(location, "s3://")
|
||||
if !ok {
|
||||
return fmt.Errorf("location must be an s3:// URI")
|
||||
}
|
||||
name, keys, _ := strings.Cut(rest, "/")
|
||||
if name != bucket {
|
||||
return fmt.Errorf("location must be within bucket %s", bucket)
|
||||
}
|
||||
for _, segment := range strings.Split(keys, "/") {
|
||||
switch {
|
||||
case segment == "." || segment == "..":
|
||||
return fmt.Errorf("location must not contain a path traversal segment")
|
||||
case strings.ContainsAny(segment, "\\\x00"):
|
||||
return fmt.Errorf("location contains an invalid character")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// storageOptions builds the object_store settings a Lance client needs to reach
|
||||
// the dataset. The key names are the aws_-prefixed forms Lance clients pass
|
||||
// through to object_store.
|
||||
func (s *Server) storageOptions(r *http.Request, bucket, location string, vend bool) (map[string]string, error) {
|
||||
options := map[string]string{}
|
||||
if s.s3Endpoint != "" {
|
||||
options["aws_endpoint"] = s.s3Endpoint
|
||||
if strings.HasPrefix(s.s3Endpoint, "http://") {
|
||||
// object_store refuses a plaintext endpoint unless told to allow it,
|
||||
// and the resulting failure reads like a credential problem.
|
||||
options["allow_http"] = "true"
|
||||
}
|
||||
}
|
||||
if s.s3Region != "" {
|
||||
options["aws_region"] = s.s3Region
|
||||
}
|
||||
|
||||
if !vend || s.credentialVendor == nil {
|
||||
return options, nil
|
||||
}
|
||||
|
||||
principal := s3_constants.GetIdentityNameFromContext(r)
|
||||
credentials, err := s.credentialVendor.VendTableCredentials(r.Context(), principal, bucket, locationPrefix(location))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if credentials == nil {
|
||||
return options, nil
|
||||
}
|
||||
options["aws_access_key_id"] = credentials.AccessKeyID
|
||||
options["aws_secret_access_key"] = credentials.SecretAccessKey
|
||||
if credentials.SessionToken != "" {
|
||||
options["aws_session_token"] = credentials.SessionToken
|
||||
}
|
||||
if !credentials.Expiration.IsZero() {
|
||||
options["expires_at_millis"] = strconv.FormatInt(credentials.Expiration.UnixMilli(), 10)
|
||||
}
|
||||
return options, nil
|
||||
}
|
||||
|
||||
// locationPrefix strips the s3://bucket/ part of a location, leaving the key
|
||||
// prefix a credential is scoped to.
|
||||
func locationPrefix(location string) string {
|
||||
trimmed := strings.TrimPrefix(location, "s3://")
|
||||
if _, prefix, found := strings.Cut(trimmed, "/"); found {
|
||||
return prefix
|
||||
}
|
||||
return ""
|
||||
}
|
||||
149
weed/s3api/lance/types.go
Normal file
149
weed/s3api/lance/types.go
Normal file
@@ -0,0 +1,149 @@
|
||||
// Package lance serves the Lance Namespace REST spec over the same table
|
||||
// buckets the Iceberg REST catalog uses. A Lance table is a catalog entry with
|
||||
// format LANCE: the namespace records where the dataset lives and vends
|
||||
// credentials for it, and the Lance client owns everything under that location.
|
||||
package lance
|
||||
|
||||
// Modes shared by create and register. The spec matches them case-insensitively
|
||||
// and accepts both PascalCase and snake_case.
|
||||
const (
|
||||
modeCreate = "create"
|
||||
modeExistOk = "existok"
|
||||
modeOverwrite = "overwrite"
|
||||
modeFail = "fail"
|
||||
modeSkip = "skip"
|
||||
|
||||
behaviorRestrict = "restrict"
|
||||
behaviorCascade = "cascade"
|
||||
)
|
||||
|
||||
type CreateNamespaceRequest struct {
|
||||
ID []string `json:"id,omitempty"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
Properties map[string]string `json:"properties,omitempty"`
|
||||
}
|
||||
|
||||
type CreateNamespaceResponse struct {
|
||||
Properties map[string]string `json:"properties"`
|
||||
}
|
||||
|
||||
type ListNamespacesResponse struct {
|
||||
Namespaces []string `json:"namespaces"`
|
||||
PageToken string `json:"page_token,omitempty"`
|
||||
}
|
||||
|
||||
type DescribeNamespaceRequest struct {
|
||||
ID []string `json:"id,omitempty"`
|
||||
}
|
||||
|
||||
type DescribeNamespaceResponse struct {
|
||||
Properties map[string]string `json:"properties"`
|
||||
}
|
||||
|
||||
type DropNamespaceRequest struct {
|
||||
ID []string `json:"id,omitempty"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
Behavior string `json:"behavior,omitempty"`
|
||||
}
|
||||
|
||||
type DropNamespaceResponse struct {
|
||||
Properties map[string]string `json:"properties,omitempty"`
|
||||
}
|
||||
|
||||
type NamespaceExistsRequest struct {
|
||||
ID []string `json:"id,omitempty"`
|
||||
}
|
||||
|
||||
type ListTablesResponse struct {
|
||||
Tables []string `json:"tables"`
|
||||
PageToken string `json:"page_token,omitempty"`
|
||||
}
|
||||
|
||||
type DeclareTableRequest struct {
|
||||
ID []string `json:"id,omitempty"`
|
||||
Location string `json:"location,omitempty"`
|
||||
VendCredentials *bool `json:"vend_credentials,omitempty"`
|
||||
Properties map[string]string `json:"properties,omitempty"`
|
||||
}
|
||||
|
||||
type DeclareTableResponse struct {
|
||||
Location string `json:"location"`
|
||||
StorageOptions map[string]string `json:"storage_options,omitempty"`
|
||||
// Properties are null rather than {}: the catalog does not keep a table's
|
||||
// properties, and {} would claim it kept them and found none.
|
||||
Properties map[string]string `json:"properties"`
|
||||
// ManagedVersioning stays false: the dataset owns its version history,
|
||||
// because this store can order commits without the catalog in the path.
|
||||
ManagedVersioning bool `json:"managed_versioning"`
|
||||
}
|
||||
|
||||
type DescribeTableRequest struct {
|
||||
ID []string `json:"id,omitempty"`
|
||||
Version *int64 `json:"version,omitempty"`
|
||||
Tag string `json:"tag,omitempty"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
// The REST spec carries these as query parameters, but clients also put
|
||||
// them in the body, so honour both.
|
||||
WithTableURI *bool `json:"with_table_uri,omitempty"`
|
||||
LoadDetailedMetadata *bool `json:"load_detailed_metadata,omitempty"`
|
||||
CheckDeclared *bool `json:"check_declared,omitempty"`
|
||||
VendCredentials *bool `json:"vend_credentials,omitempty"`
|
||||
}
|
||||
|
||||
type DescribeTableResponse struct {
|
||||
Table string `json:"table,omitempty"`
|
||||
Namespace []string `json:"namespace,omitempty"`
|
||||
Version *int64 `json:"version,omitempty"`
|
||||
Location string `json:"location"`
|
||||
TableURI string `json:"table_uri,omitempty"`
|
||||
StorageOptions map[string]string `json:"storage_options,omitempty"`
|
||||
// Null, not {}: this catalog does not keep a table's properties.
|
||||
Properties map[string]string `json:"properties"`
|
||||
// ManagedVersioning stays false: the dataset owns its version history,
|
||||
// because this store can order commits without the catalog in the path.
|
||||
ManagedVersioning bool `json:"managed_versioning"`
|
||||
IsOnlyDeclared *bool `json:"is_only_declared,omitempty"`
|
||||
}
|
||||
|
||||
type TableExistsRequest struct {
|
||||
ID []string `json:"id,omitempty"`
|
||||
}
|
||||
|
||||
type RegisterTableRequest struct {
|
||||
ID []string `json:"id,omitempty"`
|
||||
Location string `json:"location"`
|
||||
Mode string `json:"mode,omitempty"`
|
||||
Properties map[string]string `json:"properties,omitempty"`
|
||||
}
|
||||
|
||||
type RegisterTableResponse struct {
|
||||
Location string `json:"location"`
|
||||
Properties map[string]string `json:"properties"`
|
||||
}
|
||||
|
||||
type DeregisterTableRequest struct {
|
||||
ID []string `json:"id,omitempty"`
|
||||
}
|
||||
|
||||
type DeregisterTableResponse struct {
|
||||
ID []string `json:"id"`
|
||||
Location string `json:"location"`
|
||||
Properties map[string]string `json:"properties"`
|
||||
}
|
||||
|
||||
type DropTableRequest struct {
|
||||
ID []string `json:"id,omitempty"`
|
||||
}
|
||||
|
||||
type DropTableResponse struct {
|
||||
ID []string `json:"id"`
|
||||
Location string `json:"location"`
|
||||
Properties map[string]string `json:"properties"`
|
||||
}
|
||||
|
||||
type RenameTableRequest struct {
|
||||
ID []string `json:"id,omitempty"`
|
||||
NewID []string `json:"new_id"`
|
||||
}
|
||||
|
||||
type RenameTableResponse struct{}
|
||||
Reference in New Issue
Block a user