snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

100
weed/s3api/lance/errors.go Normal file
View File

@@ -0,0 +1,100 @@
package lance
import (
"encoding/json"
"errors"
"net/http"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// Lance Namespace error codes. The spec numbers them, so a client switches on
// the code rather than parsing the message.
const (
codeUnsupported = 0
codeNamespaceNotFound = 1
codeNamespaceAlreadyExists = 2
codeNamespaceNotEmpty = 3
codeTableNotFound = 4
codeTableAlreadyExists = 5
codeTableVersionNotFound = 11
codeInvalidInput = 13
codeConcurrentModification = 14
codePermissionDenied = 15
codeUnauthenticated = 16
codeInternal = 18
)
type errorResponse struct {
Error string `json:"error,omitempty"`
Code int `json:"code"`
Detail string `json:"detail,omitempty"`
Instance string `json:"instance,omitempty"`
}
func writeJSON(w http.ResponseWriter, status int, body interface{}) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
if body == nil {
return
}
if err := json.NewEncoder(w).Encode(body); err != nil {
glog.Warningf("lance: failed to encode response: %v", err)
}
}
func writeError(w http.ResponseWriter, r *http.Request, status, code int, message string) {
instance := ""
if r != nil {
instance = r.URL.Path
}
writeJSON(w, status, errorResponse{Error: message, Code: code, Instance: instance})
}
// writeStorageError translates an S3 Tables storage error into the Lance error
// model. A table bucket is the first namespace level here, so a missing bucket
// is a missing namespace rather than a missing catalog.
func writeStorageError(w http.ResponseWriter, r *http.Request, err error) {
var storageErr *s3tables.S3TablesError
if !errors.As(err, &storageErr) {
writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
return
}
status, code := http.StatusInternalServerError, codeInternal
switch storageErr.Type {
case s3tables.ErrCodeNoSuchBucket, s3tables.ErrCodeNoSuchNamespace:
status, code = http.StatusNotFound, codeNamespaceNotFound
case s3tables.ErrCodeNoSuchTable, s3tables.ErrCodeNoSuchView:
status, code = http.StatusNotFound, codeTableNotFound
case s3tables.ErrCodeBucketAlreadyExists, s3tables.ErrCodeNamespaceAlreadyExists:
status, code = http.StatusConflict, codeNamespaceAlreadyExists
case s3tables.ErrCodeTableAlreadyExists, s3tables.ErrCodeViewAlreadyExists:
status, code = http.StatusConflict, codeTableAlreadyExists
case s3tables.ErrCodeBucketNotEmpty, s3tables.ErrCodeNamespaceNotEmpty:
status, code = http.StatusConflict, codeNamespaceNotEmpty
case s3tables.ErrCodeConflict:
status, code = http.StatusConflict, codeConcurrentModification
case s3tables.ErrCodeAccessDenied:
status, code = http.StatusForbidden, codePermissionDenied
case s3tables.ErrCodeInvalidRequest, s3tables.ErrCodeInvalidIcebergLayout:
status, code = http.StatusBadRequest, codeInvalidInput
}
writeError(w, r, status, code, storageErr.Message)
}
// isNotFound reports whether a storage error means the object is absent, so
// exists-style handlers can answer without a second lookup.
func isNotFound(err error) bool {
var storageErr *s3tables.S3TablesError
if !errors.As(err, &storageErr) {
return false
}
switch storageErr.Type {
case s3tables.ErrCodeNoSuchBucket, s3tables.ErrCodeNoSuchNamespace,
s3tables.ErrCodeNoSuchTable, s3tables.ErrCodeNoSuchView:
return true
}
return false
}

View File

@@ -0,0 +1,341 @@
package lance
import (
"net/http"
"strings"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// handleCreateNamespace creates a table bucket for a one-part identifier and a
// namespace inside one for anything deeper. The root cannot be created.
func (s *Server) handleCreateNamespace(w http.ResponseWriter, r *http.Request) {
id, _, ok := routeIdentifier(w, r)
if !ok {
return
}
var req CreateNamespaceRequest
if err := decodeBody(r, &req); err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
if len(id) == 0 && len(req.ID) > 0 {
id = req.ID
}
if !checkBodyIdentifier(w, r, id, req.ID) {
return
}
if len(id) == 0 {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, "the root namespace always exists and cannot be created")
return
}
mode := normalizeMode(req.Mode, modeCreate)
switch mode {
case modeCreate, modeExistOk, modeOverwrite:
default:
writeError(w, r, http.StatusBadRequest, codeInvalidInput, "mode must be Create, ExistOk or Overwrite")
return
}
bucket, ns := id.namespace()
exists, err := s.namespaceExists(r, bucket, ns)
if err != nil {
writeStorageError(w, r, err)
return
}
if exists {
switch mode {
case modeExistOk:
writeJSON(w, http.StatusOK, CreateNamespaceResponse{Properties: req.Properties})
return
case modeCreate:
writeError(w, r, http.StatusConflict, codeNamespaceAlreadyExists, "namespace already exists")
return
}
// Overwrite replaces the namespace with an empty one, so the drop has to
// succeed first. A namespace holding tables refuses, which is the point.
if err := s.dropNamespace(r, bucket, ns); err != nil {
writeStorageError(w, r, err)
return
}
}
if err := s.createNamespace(r, bucket, ns, req.Properties); err != nil {
writeStorageError(w, r, err)
return
}
writeJSON(w, http.StatusOK, CreateNamespaceResponse{Properties: normalizeProperties(req.Properties)})
}
// handleListNamespaces lists the children of a namespace: table buckets at the
// root, and the next path component below that.
func (s *Server) handleListNamespaces(w http.ResponseWriter, r *http.Request) {
id, _, ok := routeIdentifier(w, r)
if !ok {
return
}
bucket, ns := id.namespace()
if bucket == "" {
var resp s3tables.ListTableBucketsResponse
req := &s3tables.ListTableBucketsRequest{
ContinuationToken: r.URL.Query().Get("page_token"),
MaxBuckets: pageSize(r),
}
if err := s.execute(r, "ListTableBuckets", req, &resp); err != nil {
writeStorageError(w, r, err)
return
}
names := make([]string, 0, len(resp.TableBuckets))
for _, b := range resp.TableBuckets {
names = append(names, b.Name)
}
writeJSON(w, http.StatusOK, ListNamespacesResponse{Namespaces: names, PageToken: resp.ContinuationToken})
return
}
var resp s3tables.ListNamespacesResponse
req := &s3tables.ListNamespacesRequest{
TableBucketARN: bucketARN(bucket),
ContinuationToken: r.URL.Query().Get("page_token"),
MaxNamespaces: pageSize(r),
}
if len(ns) > 0 {
req.Prefix = strings.Join(ns, ".") + "."
}
if err := s.execute(r, "ListNamespaces", req, &resp); err != nil {
writeStorageError(w, r, err)
return
}
// Storage namespaces are full paths; the spec wants the child name relative
// to the parent, so take the next component and drop repeats.
children := make([]string, 0, len(resp.Namespaces))
seen := make(map[string]struct{}, len(resp.Namespaces))
for _, summary := range resp.Namespaces {
if len(summary.Namespace) <= len(ns) {
continue
}
child := summary.Namespace[len(ns)]
if _, done := seen[child]; done {
continue
}
seen[child] = struct{}{}
children = append(children, child)
}
writeJSON(w, http.StatusOK, ListNamespacesResponse{Namespaces: children, PageToken: resp.ContinuationToken})
}
// handleDescribeNamespace returns a namespace's properties.
func (s *Server) handleDescribeNamespace(w http.ResponseWriter, r *http.Request) {
id, _, ok := routeIdentifier(w, r)
if !ok {
return
}
var req DescribeNamespaceRequest
if err := decodeBody(r, &req); err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
if len(id) == 0 && len(req.ID) > 0 {
id = req.ID
}
if !checkBodyIdentifier(w, r, id, req.ID) {
return
}
bucket, ns := id.namespace()
if bucket == "" {
writeJSON(w, http.StatusOK, DescribeNamespaceResponse{Properties: map[string]string{}})
return
}
if len(ns) == 0 {
var resp s3tables.GetTableBucketResponse
if err := s.execute(r, "GetTableBucket", &s3tables.GetTableBucketRequest{TableBucketARN: bucketARN(bucket)}, &resp); err != nil {
writeStorageError(w, r, err)
return
}
writeJSON(w, http.StatusOK, DescribeNamespaceResponse{Properties: map[string]string{}})
return
}
var resp s3tables.GetNamespaceResponse
req2 := &s3tables.GetNamespaceRequest{TableBucketARN: bucketARN(bucket), Namespace: ns}
if err := s.execute(r, "GetNamespace", req2, &resp); err != nil {
writeStorageError(w, r, err)
return
}
writeJSON(w, http.StatusOK, DescribeNamespaceResponse{Properties: normalizeProperties(resp.Properties)})
}
// handleDropNamespace removes a namespace or table bucket.
func (s *Server) handleDropNamespace(w http.ResponseWriter, r *http.Request) {
id, _, ok := routeIdentifier(w, r)
if !ok {
return
}
var req DropNamespaceRequest
if err := decodeBody(r, &req); err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
if len(id) == 0 && len(req.ID) > 0 {
id = req.ID
}
if !checkBodyIdentifier(w, r, id, req.ID) {
return
}
if len(id) == 0 {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, "the root namespace cannot be dropped")
return
}
if normalizeMode(req.Behavior, behaviorRestrict) == behaviorCascade {
writeError(w, r, http.StatusNotImplemented, codeUnsupported,
"cascade drop is not supported; drop the tables in the namespace first")
return
}
bucket, ns := id.namespace()
err := s.dropNamespace(r, bucket, ns)
if err == nil {
writeJSON(w, http.StatusOK, DropNamespaceResponse{})
return
}
if isNotFound(err) {
// Skip reports success on a missing namespace; Fail reports 400 rather
// than 404, which is what the spec asks for on this operation alone.
if normalizeMode(req.Mode, modeFail) == modeSkip {
w.WriteHeader(http.StatusNoContent)
return
}
writeError(w, r, http.StatusBadRequest, codeNamespaceNotFound, "namespace does not exist")
return
}
writeStorageError(w, r, err)
}
// handleNamespaceExists answers with the status code and no body.
func (s *Server) handleNamespaceExists(w http.ResponseWriter, r *http.Request) {
id, _, ok := routeIdentifier(w, r)
if !ok {
return
}
var req NamespaceExistsRequest
if err := decodeBody(r, &req); err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
if len(id) == 0 && len(req.ID) > 0 {
id = req.ID
}
if !checkBodyIdentifier(w, r, id, req.ID) {
return
}
bucket, ns := id.namespace()
if bucket == "" {
w.WriteHeader(http.StatusOK)
return
}
exists, err := s.namespaceExists(r, bucket, ns)
if err != nil {
writeStorageError(w, r, err)
return
}
if !exists {
writeError(w, r, http.StatusNotFound, codeNamespaceNotFound, "namespace does not exist")
return
}
w.WriteHeader(http.StatusOK)
}
func (s *Server) namespaceExists(r *http.Request, bucket string, ns []string) (bool, error) {
var err error
if len(ns) == 0 {
var resp s3tables.GetTableBucketResponse
err = s.execute(r, "GetTableBucket", &s3tables.GetTableBucketRequest{TableBucketARN: bucketARN(bucket)}, &resp)
} else {
var resp s3tables.GetNamespaceResponse
err = s.execute(r, "GetNamespace", &s3tables.GetNamespaceRequest{TableBucketARN: bucketARN(bucket), Namespace: ns}, &resp)
}
if err == nil {
return true, nil
}
if isNotFound(err) {
return false, nil
}
return false, err
}
func (s *Server) createNamespace(r *http.Request, bucket string, ns []string, properties map[string]string) error {
if len(ns) == 0 {
// A bucket made through this surface holds Lance tables. Saying so is
// what stops it from being described to a client as an Iceberg catalog.
var resp s3tables.CreateTableBucketResponse
return s.execute(r, "CreateTableBucket", &s3tables.CreateTableBucketRequest{
Name: bucket,
Format: s3tables.FormatLance,
}, &resp)
}
// A table bucket is a tenant resource with its own policy and lifecycle, so
// it is created deliberately, never as a side effect of naming a namespace
// inside it.
// The immediate parent has to exist too. Storage keeps a namespace's parts
// flattened, so creating "a.b" without "a" leaves an intermediate that
// listing derives from the name and describe then denies exists. The spec
// asks for NamespaceNotFound here, which also keeps the two consistent.
if len(ns) > 1 {
parent := ns[:len(ns)-1]
if exists, err := s.namespaceExists(r, bucket, parent); err != nil {
return err
} else if !exists {
return &s3tables.S3TablesError{
Type: s3tables.ErrCodeNoSuchNamespace,
Message: "parent namespace " + strings.Join(parent, ".") + " does not exist",
}
}
}
if exists, err := s.namespaceExists(r, bucket, nil); err != nil {
return err
} else if !exists {
return &s3tables.S3TablesError{
Type: s3tables.ErrCodeNoSuchBucket,
Message: "table bucket " + bucket + " does not exist",
}
}
var resp s3tables.CreateNamespaceResponse
req := &s3tables.CreateNamespaceRequest{
TableBucketARN: bucketARN(bucket),
Namespace: ns,
Properties: properties,
}
return s.execute(r, "CreateNamespace", req, &resp)
}
func (s *Server) dropNamespace(r *http.Request, bucket string, ns []string) error {
if len(ns) == 0 {
return s.execute(r, "DeleteTableBucket", &s3tables.DeleteTableBucketRequest{TableBucketARN: bucketARN(bucket)}, nil)
}
req := &s3tables.DeleteNamespaceRequest{TableBucketARN: bucketARN(bucket), Namespace: ns}
return s.execute(r, "DeleteNamespace", req, nil)
}
// normalizeMode folds a spec mode or behavior value, which is case-insensitive
// and spelled either PascalCase or snake_case, onto its lowercase form.
func normalizeMode(value, fallback string) string {
value = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(value), "_", ""))
if value == "" {
return fallback
}
return value
}
func normalizeProperties(properties map[string]string) map[string]string {
if properties == nil {
return map[string]string{}
}
return properties
}

View File

@@ -0,0 +1,573 @@
package lance
import (
"net/http"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// handleListTables lists the Lance tables under a namespace. The spec asks for
// full string identifiers, not bare names, so a recursive listing stays
// unambiguous.
func (s *Server) handleListTables(w http.ResponseWriter, r *http.Request) {
id, delimiter, ok := routeIdentifier(w, r)
if !ok {
return
}
bucket, ns := id.namespace()
if bucket == "" {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, "listing tables needs a namespace, not the root")
return
}
req := &s3tables.ListTablesRequest{
TableBucketARN: bucketARN(bucket),
Namespace: ns,
ContinuationToken: r.URL.Query().Get("page_token"),
MaxTables: pageSize(r),
}
var resp s3tables.ListTablesResponse
if err := s.execute(r, "ListTables", req, &resp); err != nil {
writeStorageError(w, r, err)
return
}
includeDeclared := true
if raw := r.URL.Query().Get("include_declared"); raw != "" {
includeDeclared = boolQuery(r, "include_declared")
}
writeJSON(w, http.StatusOK, ListTablesResponse{
Tables: s.lanceTableIDs(r, bucket, resp.Tables, delimiter, includeDeclared),
PageToken: resp.ContinuationToken,
})
}
// handleListAllTables lists every Lance table the caller can see, across every
// table bucket.
func (s *Server) handleListAllTables(w http.ResponseWriter, r *http.Request) {
delimiter := requestDelimiter(r)
var buckets s3tables.ListTableBucketsResponse
if err := s.execute(r, "ListTableBuckets", &s3tables.ListTableBucketsRequest{MaxBuckets: pageSize(r)}, &buckets); err != nil {
writeStorageError(w, r, err)
return
}
// The spec marks `tables` required, so an empty catalog answers with an
// empty list rather than null.
all := []string{}
for _, bucket := range buckets.TableBuckets {
var tables s3tables.ListTablesResponse
req := &s3tables.ListTablesRequest{TableBucketARN: bucketARN(bucket.Name), MaxTables: pageSize(r)}
if err := s.execute(r, "ListTables", req, &tables); err != nil {
// One unreadable bucket must not hide the rest; a caller with access
// to some buckets still gets those.
glog.V(2).Infof("lance: skipping bucket %s in ListAllTables: %v", bucket.Name, err)
continue
}
all = append(all, s.lanceTableIDs(r, bucket.Name, tables.Tables, delimiter, true)...)
}
writeJSON(w, http.StatusOK, ListTablesResponse{Tables: all})
}
// lanceTableIDs keeps the Lance tables out of a listing that also carries
// Iceberg tables, and drops deregistered ones because they are meant to be
// invisible until re-registered.
func (s *Server) lanceTableIDs(r *http.Request, bucket string, summaries []s3tables.TableSummary, delimiter string, includeDeclared bool) []string {
ids := make([]string, 0, len(summaries))
for _, summary := range summaries {
if summary.Format != s3tables.FormatLance {
continue
}
location := summary.MetadataLocation
if location == "" {
location = tableLocation(bucket, summary.Namespace, summary.Name)
}
deregistered, hasData, err := s.datasetState(r, location)
if err != nil {
glog.V(2).Infof("lance: cannot read dataset state for %s: %v", location, err)
continue
}
if deregistered {
continue
}
if !hasData && !includeDeclared {
continue
}
parts := append([]string{bucket}, summary.Namespace...)
ids = append(ids, identifier(append(parts, summary.Name)).String(delimiter))
}
return ids
}
// handleDeclareTable records a table that does not exist on storage yet. This
// is what a Lance client calls on CREATE TABLE, before it writes any data.
func (s *Server) handleDeclareTable(w http.ResponseWriter, r *http.Request) {
id, _, ok := routeIdentifier(w, r)
if !ok {
return
}
var req DeclareTableRequest
if err := decodeBody(r, &req); err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
if len(id) == 0 && len(req.ID) > 0 {
id = req.ID
}
if !checkBodyIdentifier(w, r, id, req.ID) {
return
}
bucket, ns, name, err := id.table()
if err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
location := strings.TrimSuffix(req.Location, "/")
if location == "" {
location = tableLocation(bucket, ns, name)
} else if err := confineLocation(bucket, location); err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
if err := s.createTable(r, bucket, ns, name, location); err != nil {
writeStorageError(w, r, err)
return
}
if err := s.writeMarker(r, location, reservedMarker); err != nil {
// The catalog entry is the authority; the marker only mirrors it for
// clients that read the storage prefix directly.
glog.V(1).Infof("lance: could not write %s for %s: %v", reservedMarker, location, err)
}
// Declaring a name that was deregistered brings it back, the same way
// registering it does.
if err := s.removeMarker(r, location, deregisteredMarker); err != nil {
glog.V(2).Infof("lance: could not clear %s for %s: %v", deregisteredMarker, location, err)
}
options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials))
if err != nil {
writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
return
}
// Properties are not persisted for a table, so neither response carries
// them: null says "this catalog does not keep them", where echoing the
// request back or answering {} would say they were stored and are empty.
writeJSON(w, http.StatusOK, DeclareTableResponse{
Location: location,
StorageOptions: options,
})
}
// handleDescribeTable resolves a table to a location, and to credentials when
// the caller asks for them.
func (s *Server) handleDescribeTable(w http.ResponseWriter, r *http.Request) {
id, _, ok := routeIdentifier(w, r)
if !ok {
return
}
var req DescribeTableRequest
if err := decodeBody(r, &req); err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
if len(id) == 0 && len(req.ID) > 0 {
id = req.ID
}
if !checkBodyIdentifier(w, r, id, req.ID) {
return
}
bucket, ns, name, err := id.table()
if err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
if req.Version != nil || req.Tag != "" || req.Branch != "" {
writeError(w, r, http.StatusNotImplemented, codeUnsupported,
"this namespace does not resolve versions, tags or branches; the dataset owns them")
return
}
table, err := s.loadLanceTable(r, bucket, ns, name)
if err != nil {
writeStorageError(w, r, err)
return
}
location := table.location
deregistered, hasData, err := s.datasetState(r, location)
if err != nil {
writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
return
}
if deregistered {
writeError(w, r, http.StatusNotFound, codeTableNotFound, "table is deregistered")
return
}
options, err := s.storageOptions(r, bucket, location, wants(r, "vend_credentials", req.VendCredentials))
if err != nil {
writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
return
}
resp := DescribeTableResponse{
Location: location,
}
if len(options) > 0 {
resp.StorageOptions = options
}
if wants(r, "with_table_uri", req.WithTableURI) {
resp.TableURI = location
}
if wants(r, "check_declared", req.CheckDeclared) {
onlyDeclared := !hasData
resp.IsOnlyDeclared = &onlyDeclared
}
if wants(r, "load_detailed_metadata", req.LoadDetailedMetadata) {
resp.Table = name
resp.Namespace = ns
}
writeJSON(w, http.StatusOK, resp)
}
// handleTableExists answers with the status code and no body.
func (s *Server) handleTableExists(w http.ResponseWriter, r *http.Request) {
id, _, ok := routeIdentifier(w, r)
if !ok {
return
}
var req TableExistsRequest
if err := decodeBody(r, &req); err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
if len(id) == 0 && len(req.ID) > 0 {
id = req.ID
}
if !checkBodyIdentifier(w, r, id, req.ID) {
return
}
bucket, ns, name, err := id.table()
if err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
table, err := s.loadLanceTable(r, bucket, ns, name)
if err != nil {
if isNotFound(err) {
writeError(w, r, http.StatusNotFound, codeTableNotFound, "table does not exist")
return
}
writeStorageError(w, r, err)
return
}
deregistered, _, err := s.datasetState(r, table.location)
if err != nil {
writeError(w, r, http.StatusInternalServerError, codeInternal, err.Error())
return
}
if deregistered {
writeError(w, r, http.StatusNotFound, codeTableNotFound, "table is deregistered")
return
}
w.WriteHeader(http.StatusOK)
}
// handleRegisterTable points a table name at an existing dataset.
func (s *Server) handleRegisterTable(w http.ResponseWriter, r *http.Request) {
id, _, ok := routeIdentifier(w, r)
if !ok {
return
}
var req RegisterTableRequest
if err := decodeBody(r, &req); err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
if len(id) == 0 && len(req.ID) > 0 {
id = req.ID
}
if !checkBodyIdentifier(w, r, id, req.ID) {
return
}
bucket, ns, name, err := id.table()
if err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
location := strings.TrimSuffix(req.Location, "/")
if location == "" {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, "location is required")
return
}
if err := confineLocation(bucket, location); err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
mode := normalizeMode(req.Mode, modeCreate)
if mode != modeCreate && mode != modeOverwrite {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, "mode must be Create or Overwrite")
return
}
existing, err := s.loadLanceTable(r, bucket, ns, name)
switch {
case err != nil && !isNotFound(err):
writeStorageError(w, r, err)
return
case err == nil:
// A deregistered table is absent as far as the spec is concerned, so
// registering over it is a re-registration rather than a conflict.
deregistered, _, stateErr := s.datasetState(r, existing.location)
if stateErr != nil {
writeError(w, r, http.StatusInternalServerError, codeInternal, stateErr.Error())
return
}
if !deregistered && mode == modeCreate {
writeError(w, r, http.StatusConflict, codeTableAlreadyExists, "table already exists")
return
}
if existing.location != location {
// Repointing the name at another dataset is an update. Dropping and
// recreating the entry would take the old dataset's files with it,
// because the entry is the directory holding them.
if err := s.repointTable(r, bucket, ns, name, location, existing.versionToken); err != nil {
writeStorageError(w, r, err)
return
}
}
default:
if err := s.createTable(r, bucket, ns, name, location); err != nil {
writeStorageError(w, r, err)
return
}
}
// Registering a deregistered dataset brings it back.
if err := s.removeMarker(r, location, deregisteredMarker); err != nil {
glog.V(2).Infof("lance: could not clear %s for %s: %v", deregisteredMarker, location, err)
}
writeJSON(w, http.StatusOK, RegisterTableResponse{
Location: location,
Properties: normalizeProperties(req.Properties),
})
}
// handleDeregisterTable forgets a table without touching its data.
func (s *Server) handleDeregisterTable(w http.ResponseWriter, r *http.Request) {
id, _, ok := routeIdentifier(w, r)
if !ok {
return
}
var req DeregisterTableRequest
if err := decodeBody(r, &req); err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
if len(id) == 0 && len(req.ID) > 0 {
id = req.ID
}
if !checkBodyIdentifier(w, r, id, req.ID) {
return
}
bucket, ns, name, err := id.table()
if err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
table, err := s.loadLanceTable(r, bucket, ns, name)
if err != nil {
writeStorageError(w, r, err)
return
}
// Deregistering is a state, not a deletion. Dropping the catalog entry would
// take the dataset with it, because the entry is the dataset directory and
// DeleteTable purges what it holds. The marker is what hides the table, and
// it is also what a directory-catalog client reads.
if err := s.writeMarker(r, table.location, deregisteredMarker); err != nil {
writeError(w, r, http.StatusInternalServerError, codeInternal,
"could not mark the table deregistered: "+err.Error())
return
}
writeJSON(w, http.StatusOK, DeregisterTableResponse{
ID: append(append([]string{bucket}, ns...), name),
Location: table.location,
Properties: map[string]string{},
})
}
// handleDropTable removes the table and its data.
func (s *Server) handleDropTable(w http.ResponseWriter, r *http.Request) {
id, _, ok := routeIdentifier(w, r)
if !ok {
return
}
var req DropTableRequest
if err := decodeBody(r, &req); err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
if len(id) == 0 && len(req.ID) > 0 {
id = req.ID
}
if !checkBodyIdentifier(w, r, id, req.ID) {
return
}
bucket, ns, name, err := id.table()
if err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
table, err := s.loadLanceTable(r, bucket, ns, name)
if err != nil {
writeStorageError(w, r, err)
return
}
if err := s.dropTableEntry(r, bucket, ns, name); err != nil {
writeStorageError(w, r, err)
return
}
writeJSON(w, http.StatusOK, DropTableResponse{
ID: append(append([]string{bucket}, ns...), name),
Location: table.location,
Properties: map[string]string{},
})
}
// handleRenameTable moves a table's catalog entry. The dataset stays put, which
// is what the storage layer already does for a renamed Iceberg table.
func (s *Server) handleRenameTable(w http.ResponseWriter, r *http.Request) {
id, _, ok := routeIdentifier(w, r)
if !ok {
return
}
var req RenameTableRequest
if err := decodeBody(r, &req); err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
if len(id) == 0 && len(req.ID) > 0 {
id = req.ID
}
if !checkBodyIdentifier(w, r, id, req.ID) {
return
}
bucket, ns, name, err := id.table()
if err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return
}
destBucket, destNS, destName, err := identifier(req.NewID).table()
if err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, "new_id: "+err.Error())
return
}
if destBucket != bucket {
writeError(w, r, http.StatusBadRequest, codeInvalidInput,
"a table cannot move between table buckets")
return
}
if _, err := s.loadLanceTable(r, bucket, ns, name); err != nil {
writeStorageError(w, r, err)
return
}
renameReq := &s3tables.RenameTableRequest{
TableBucketARN: bucketARN(bucket),
SourceNamespace: ns,
SourceName: name,
DestNamespace: destNS,
DestName: destName,
}
var renameResp s3tables.RenameTableResponse
if err := s.execute(r, "RenameTable", renameReq, &renameResp); err != nil {
writeStorageError(w, r, err)
return
}
writeJSON(w, http.StatusOK, RenameTableResponse{})
}
// lanceTable is the catalog's record of one Lance table.
type lanceTable struct {
location string
versionToken string
}
// loadLanceTable reads a table and refuses one that is not a Lance table, so a
// Lance client never resolves an Iceberg table's location and writes over it.
func (s *Server) loadLanceTable(r *http.Request, bucket string, ns []string, name string) (*lanceTable, error) {
req := &s3tables.GetTableRequest{
TableBucketARN: bucketARN(bucket),
Namespace: ns,
Name: name,
}
var resp s3tables.GetTableResponse
if err := s.execute(r, "GetTable", req, &resp); err != nil {
return nil, err
}
if resp.Format != s3tables.FormatLance {
return nil, &s3tables.S3TablesError{
Type: s3tables.ErrCodeNoSuchTable,
Message: "table " + name + " is not a lance table",
}
}
location := resp.MetadataLocation
if location == "" {
location = tableLocation(bucket, resp.Namespace, resp.Name)
}
return &lanceTable{location: location, versionToken: resp.VersionToken}, nil
}
func (s *Server) createTable(r *http.Request, bucket string, ns []string, name, location string) error {
req := &s3tables.CreateTableRequest{
TableBucketARN: bucketARN(bucket),
Namespace: ns,
Name: name,
Format: s3tables.FormatLance,
// A Lance table has no metadata file, so the location the catalog stores
// is the dataset root itself.
MetadataLocation: location,
}
var resp s3tables.CreateTableResponse
return s.execute(r, "CreateTable", req, &resp)
}
// repointTable moves an existing entry to another dataset location without
// touching either dataset's files.
func (s *Server) repointTable(r *http.Request, bucket string, ns []string, name, location, versionToken string) error {
req := &s3tables.UpdateTableRequest{
TableBucketARN: bucketARN(bucket),
Namespace: ns,
Name: name,
VersionToken: versionToken,
MetadataLocation: location,
}
var resp s3tables.UpdateTableResponse
return s.execute(r, "UpdateTable", req, &resp)
}
// dropTableEntry removes the catalog entry and the dataset under it. Only
// DropTable wants this; deregistering and repointing must leave the files.
func (s *Server) dropTableEntry(r *http.Request, bucket string, ns []string, name string) error {
req := &s3tables.DeleteTableRequest{
TableBucketARN: bucketARN(bucket),
Namespace: ns,
Name: name,
}
return s.execute(r, "DeleteTable", req, nil)
}

View File

@@ -0,0 +1,417 @@
package lance
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables/s3tablestest"
)
// openAuthenticator stands in for a gateway with no IAM configured, which is the
// mode the namespace falls open in.
type openAuthenticator struct{}
func (openAuthenticator) AuthenticateRequest(*http.Request) (string, interface{}, s3err.ErrorCode) {
return s3tables.DefaultAccountID, nil, s3err.ErrNone
}
func (openAuthenticator) DefaultAllow() bool { return true }
type testHarness struct {
router *mux.Router
filer *s3tablestest.MemFiler
admin *s3tables.Manager
server *Server
}
func newTestHarness(t *testing.T) *testHarness {
t.Helper()
filer := s3tablestest.Start(t)
server := NewServer(s3tables.NewManagerClient(filer.Client), openAuthenticator{})
server.SetS3Endpoint("http://127.0.0.1:8333")
server.SetS3Region(s3tables.DefaultRegion)
router := mux.NewRouter().SkipClean(true)
server.RegisterRoutes(router)
// Table buckets are created by an operator, not by a namespace client, so
// the harness seeds them the way the shell and admin console would.
admin := s3tables.NewManager()
admin.SetTrusted(true)
return &testHarness{router: router, filer: filer, admin: admin, server: server}
}
// createBucket seeds a table bucket the Lance namespace can then be pointed at.
// An empty format strips the declaration afterwards, which is the only way to
// get the shape a bucket made before formats existed has: one that still
// accepts either format.
func (h *testHarness) createBucket(t *testing.T, name, format string) {
t.Helper()
var resp s3tables.CreateTableBucketResponse
declared := format
if declared == "" {
declared = s3tables.FormatIceberg
}
err := h.admin.Execute(t.Context(), s3tables.NewManagerClient(h.filer.Client), "CreateTableBucket",
&s3tables.CreateTableBucketRequest{Name: name, Format: declared}, &resp, "")
if err != nil {
t.Fatalf("create table bucket %s: %v", name, err)
}
if format == "" {
h.undeclareBucket(t, name)
}
}
// undeclareBucket removes a bucket's format from its metadata, ageing it back to
// what the filer holds for a bucket created before the field existed.
func (h *testHarness) undeclareBucket(t *testing.T, name string) {
t.Helper()
entry := h.filer.Get(s3tables.TablesPath, name)
if entry == nil {
t.Fatalf("table bucket %s is not in the filer", name)
}
var metadata map[string]any
if err := json.Unmarshal(entry.Extended[s3tables.ExtendedKeyMetadata], &metadata); err != nil {
t.Fatalf("read bucket metadata: %v", err)
}
delete(metadata, "format")
updated, err := json.Marshal(metadata)
if err != nil {
t.Fatalf("write bucket metadata: %v", err)
}
extended := map[string][]byte{}
for key, value := range entry.Extended {
extended[key] = value
}
extended[s3tables.ExtendedKeyMetadata] = updated
h.filer.Put(s3tables.TablesPath, name, extended)
}
func (h *testHarness) bucketARN(t *testing.T, name string) string {
t.Helper()
arn, err := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3tables.DefaultAccountID, name)
if err != nil {
t.Fatalf("build arn: %v", err)
}
return arn
}
func (h *testHarness) do(t *testing.T, method, target, body string) *httptest.ResponseRecorder {
t.Helper()
var reader *strings.Reader
if body == "" {
reader = strings.NewReader("")
} else {
reader = strings.NewReader(body)
}
req := httptest.NewRequest(method, target, reader)
recorder := httptest.NewRecorder()
h.router.ServeHTTP(recorder, req)
return recorder
}
func (h *testHarness) mustDo(t *testing.T, method, target, body string, want int) *httptest.ResponseRecorder {
t.Helper()
recorder := h.do(t, method, target, body)
if recorder.Code != want {
t.Fatalf("%s %s = %d (%s), want %d", method, target, recorder.Code, recorder.Body.String(), want)
}
return recorder
}
func decode[T any](t *testing.T, recorder *httptest.ResponseRecorder) T {
t.Helper()
var out T
if err := json.Unmarshal(recorder.Body.Bytes(), &out); err != nil {
t.Fatalf("decode %s: %v", recorder.Body.String(), err)
}
return out
}
// The lifecycle a Lance client drives: create the namespace, declare the table
// before any data exists, resolve it to a location, then deregister and bring it
// back by registering the same location.
func TestTableLifecycle(t *testing.T) {
h := newTestHarness(t)
h.createBucket(t, "analytics", s3tables.FormatLance)
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
declared := decode[DeclareTableResponse](t,
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/declare", `{}`, http.StatusOK))
if declared.Location != "s3://analytics/sales/orders" {
t.Fatalf("declared location = %q", declared.Location)
}
// The endpoint is plaintext, so a client that does not get allow_http fails
// with what looks like a credential error.
if declared.StorageOptions["aws_endpoint"] != "http://127.0.0.1:8333" ||
declared.StorageOptions["allow_http"] != "true" {
t.Fatalf("storage options = %v", declared.StorageOptions)
}
described := decode[DescribeTableResponse](t,
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/describe?check_declared=true", `{}`, http.StatusOK))
if described.Location != declared.Location {
t.Fatalf("described location = %q, want %q", described.Location, declared.Location)
}
if described.IsOnlyDeclared == nil || !*described.IsOnlyDeclared {
t.Fatalf("is_only_declared = %v, want true for a table with no data", described.IsOnlyDeclared)
}
listed := decode[ListTablesResponse](t,
h.mustDo(t, http.MethodGet, "/v1/namespace/analytics$sales/table/list", "", http.StatusOK))
if len(listed.Tables) != 1 || listed.Tables[0] != "analytics$sales$orders" {
t.Fatalf("listed tables = %v, want the full identifier", listed.Tables)
}
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/exists", `{}`, http.StatusOK)
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/deregister", `{}`, http.StatusOK)
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/exists", `{}`, http.StatusNotFound)
afterDrop := decode[ListTablesResponse](t,
h.mustDo(t, http.MethodGet, "/v1/namespace/analytics$sales/table/list", "", http.StatusOK))
if len(afterDrop.Tables) != 0 {
t.Fatalf("deregistered table still listed: %v", afterDrop.Tables)
}
// Deregistering preserves the data. The catalog entry is the dataset
// directory, so dropping it would take the dataset with it.
if h.filer.Get(s3tables.GetNamespacePath("analytics", "sales"), "orders") == nil {
t.Fatal("deregister deleted the dataset directory")
}
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/register",
`{"location":"s3://analytics/sales/orders"}`, http.StatusOK)
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/exists", `{}`, http.StatusOK)
// Dropping is the operation that does remove the data.
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/drop", `{}`, http.StatusOK)
if h.filer.Get(s3tables.GetNamespacePath("analytics", "sales"), "orders") != nil {
t.Fatal("drop left the dataset directory behind")
}
}
// Repointing a registered name at another dataset must not take the dataset it
// used to name with it.
func TestRegisterOverwriteKeepsTheOldDataset(t *testing.T) {
h := newTestHarness(t)
h.createBucket(t, "analytics", s3tables.FormatLance)
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/declare", `{}`, http.StatusOK)
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$archive/declare", `{}`, http.StatusOK)
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/register",
`{"location":"s3://analytics/sales/archive","mode":"Overwrite"}`, http.StatusOK)
described := decode[DescribeTableResponse](t,
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/describe", `{}`, http.StatusOK))
if described.Location != "s3://analytics/sales/archive" {
t.Fatalf("location after repointing = %q", described.Location)
}
if h.filer.Get(s3tables.GetNamespacePath("analytics", "sales"), "orders") == nil {
t.Fatal("repointing deleted the dataset the name used to hold")
}
}
// A required list field answers empty rather than null, which a generated
// client may decode differently.
func TestListAllTablesIsNeverNull(t *testing.T) {
h := newTestHarness(t)
h.createBucket(t, "analytics", s3tables.FormatLance)
body := h.mustDo(t, http.MethodGet, "/v1/table", "", http.StatusOK).Body.String()
if strings.Contains(body, `"tables":null`) {
t.Fatalf("ListAllTables returned null for a required field: %s", body)
}
}
func TestNamespaceListing(t *testing.T) {
h := newTestHarness(t)
h.createBucket(t, "analytics", s3tables.FormatLance)
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$finance/create", `{}`, http.StatusOK)
roots := decode[ListNamespacesResponse](t,
h.mustDo(t, http.MethodGet, "/v1/namespace/$/list", "", http.StatusOK))
if len(roots.Namespaces) != 1 || roots.Namespaces[0] != "analytics" {
t.Fatalf("root listing = %v, want the table buckets", roots.Namespaces)
}
children := decode[ListNamespacesResponse](t,
h.mustDo(t, http.MethodGet, "/v1/namespace/analytics/list", "", http.StatusOK))
if len(children.Namespaces) != 2 {
t.Fatalf("bucket listing = %v, want two namespaces", children.Namespaces)
}
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/exists", `{}`, http.StatusOK)
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$nope/exists", `{}`, http.StatusNotFound)
// Fail mode reports a missing namespace as 400 on this operation, which is
// what the spec asks for; Skip reports success.
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$nope/drop", `{}`, http.StatusBadRequest)
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$nope/drop", `{"mode":"Skip"}`, http.StatusNoContent)
}
// Storage flattens a namespace's parts, so creating a$b without a would leave
// an intermediate that listing derives from the name and describe denies
// exists. The spec asks for NamespaceNotFound, which keeps them consistent.
func TestCreateNamespaceRequiresItsParent(t *testing.T) {
h := newTestHarness(t)
h.createBucket(t, "analytics", s3tables.FormatLance)
recorder := h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$missing$child/create", `{}`, http.StatusNotFound)
if got := decode[errorResponse](t, recorder); got.Code != codeNamespaceNotFound {
t.Fatalf("error code = %d, want %d", got.Code, codeNamespaceNotFound)
}
// With the parent in place the child is fine, and both are then listed and
// describable.
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$parent/create", `{}`, http.StatusOK)
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$parent$child/create", `{}`, http.StatusOK)
listed := decode[ListNamespacesResponse](t,
h.mustDo(t, http.MethodGet, "/v1/namespace/analytics/list", "", http.StatusOK))
for _, name := range listed.Namespaces {
path := "/v1/namespace/analytics$" + name
h.mustDo(t, http.MethodPost, path+"/exists", `{}`, http.StatusOK)
}
}
func TestCreateNamespaceModes(t *testing.T) {
h := newTestHarness(t)
h.createBucket(t, "analytics", s3tables.FormatLance)
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusConflict)
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{"mode":"ExistOk"}`, http.StatusOK)
// snake_case is the other spelling the spec accepts.
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{"mode":"exist_ok"}`, http.StatusOK)
h.mustDo(t, http.MethodPost, "/v1/namespace/$/create", `{}`, http.StatusBadRequest)
}
// A Lance client must never resolve an Iceberg table's location, or it would
// write a dataset over a table another engine owns.
func TestIcebergTablesAreInvisible(t *testing.T) {
h := newTestHarness(t)
// Undeclared, because a bucket that declares one format cannot hold the
// other - and mixing is exactly what this test needs to prove is hidden.
h.createBucket(t, "analytics", "")
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$vectors/declare", `{}`, http.StatusOK)
manager := h.admin
var created s3tables.CreateTableResponse
err := manager.Execute(t.Context(), s3tables.NewManagerClient(h.filer.Client), "CreateTable", &s3tables.CreateTableRequest{
TableBucketARN: h.bucketARN(t, "analytics"),
Namespace: []string{"sales"},
Name: "ledger",
Format: s3tables.FormatIceberg,
MetadataLocation: "s3://analytics/sales/ledger/metadata/v1.metadata.json",
}, &created, "")
if err != nil {
t.Fatalf("create iceberg table: %v", err)
}
listed := decode[ListTablesResponse](t,
h.mustDo(t, http.MethodGet, "/v1/namespace/analytics$sales/table/list", "", http.StatusOK))
if len(listed.Tables) != 1 || listed.Tables[0] != "analytics$sales$vectors" {
t.Fatalf("listing = %v, want only the lance table", listed.Tables)
}
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$ledger/describe", `{}`, http.StatusNotFound)
h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$ledger/exists", `{}`, http.StatusNotFound)
// Declaring over the Iceberg table must not quietly succeed and hand the
// Lance client a directory another format owns.
recorder := h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$ledger/declare", `{}`, http.StatusConflict)
if got := decode[errorResponse](t, recorder); got.Code != codeTableAlreadyExists {
t.Fatalf("error code = %d, want %d", got.Code, codeTableAlreadyExists)
}
}
// The route and the body naming different objects is a bad request, not a silent
// preference for one of them.
func TestRouteAndBodyMustAgree(t *testing.T) {
h := newTestHarness(t)
h.createBucket(t, "analytics", s3tables.FormatLance)
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
recorder := h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/declare",
`{"id":["analytics","sales","other"]}`, http.StatusBadRequest)
if got := decode[errorResponse](t, recorder); got.Code != codeInvalidInput {
t.Fatalf("error code = %d, want %d", got.Code, codeInvalidInput)
}
}
// A client-supplied location must resolve inside its own bucket. Without this,
// a "../"-laden or cross-bucket location escapes when the marker path is joined
// under /buckets, letting the request reach another tenant's namespace, or
// outside /buckets entirely, and auto-create the parent directories on the way.
func TestLocationOutsideBucketIsRejected(t *testing.T) {
cases := []struct {
name string
location string
}{
{"another bucket", "s3://victim/secret"},
{"traversal into another bucket", "s3://analytics/../victim/secret"},
{"traversal above the buckets root", "s3://analytics/../../etc/cron.d"},
{"not an s3 uri", "../../etc/cron.d"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
dir := s3tables.TableDataDirFromMetadataLocation(tc.location)
body := `{"location":"` + tc.location + `"}`
// declare writes .lance-reserved at the location: rejection must leave
// nothing behind at the escaped directory.
t.Run("declare", func(t *testing.T) {
h := newTestHarness(t)
h.createBucket(t, "analytics", s3tables.FormatLance)
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
recorder := h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/declare", body, http.StatusBadRequest)
if got := decode[errorResponse](t, recorder); got.Code != codeInvalidInput {
t.Fatalf("error code = %d, want %d", got.Code, codeInvalidInput)
}
if dir != "" && h.filer.Get(dir, reservedMarker) != nil {
t.Fatalf("declare wrote %s into %s despite rejection", reservedMarker, dir)
}
})
// register removes .lance-deregistered at the location: rejection must
// leave a victim's marker in place rather than un-hiding their table.
t.Run("register", func(t *testing.T) {
h := newTestHarness(t)
h.createBucket(t, "analytics", s3tables.FormatLance)
h.mustDo(t, http.MethodPost, "/v1/namespace/analytics$sales/create", `{}`, http.StatusOK)
if dir != "" {
h.filer.PutFile(dir, deregisteredMarker, time.Unix(1, 0))
}
recorder := h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/register", body, http.StatusBadRequest)
if got := decode[errorResponse](t, recorder); got.Code != codeInvalidInput {
t.Fatalf("error code = %d, want %d", got.Code, codeInvalidInput)
}
if dir != "" && h.filer.Get(dir, deregisteredMarker) == nil {
t.Fatalf("register removed %s from %s despite rejection", deregisteredMarker, dir)
}
})
})
}
}
// The data plane needs Lance format support that does not exist in Go, so it
// answers with the spec's own Unsupported code rather than a bare 404.
func TestDataPlaneIsUnsupported(t *testing.T) {
h := newTestHarness(t)
recorder := h.mustDo(t, http.MethodPost, "/v1/table/analytics$sales$orders/query", `{}`, http.StatusNotImplemented)
if got := decode[errorResponse](t, recorder); got.Code != codeUnsupported {
t.Fatalf("error code = %d, want %d", got.Code, codeUnsupported)
}
}

View File

@@ -0,0 +1,84 @@
package lance
import (
"fmt"
"net/http"
"strings"
)
// defaultDelimiter joins the parts of a Lance string identifier when the caller
// does not pass ?delimiter=. An id equal to the delimiter is the root namespace,
// so /v1/namespace/$/list lists the root's children.
const defaultDelimiter = "$"
// identifier is a Lance object identifier: zero parts is the root namespace, one
// part names a table bucket, and the rest are namespace parts with a table name
// last. The mapping onto storage is bucket / namespace / table, which is the
// three-level shape Lance clients already use.
type identifier []string
func requestDelimiter(r *http.Request) string {
if d := r.URL.Query().Get("delimiter"); d != "" {
return d
}
return defaultDelimiter
}
// parseIdentifier decodes the {id} route variable. Empty parts are rejected
// rather than dropped so that "a$$b" cannot silently resolve to "a$b".
func parseIdentifier(encoded, delimiter string) (identifier, error) {
if encoded == "" || encoded == delimiter {
return nil, nil
}
parts := strings.Split(encoded, delimiter)
for _, part := range parts {
if part == "" {
return nil, fmt.Errorf("identifier %q has an empty part", encoded)
}
}
return parts, nil
}
func (id identifier) String(delimiter string) string {
if len(id) == 0 {
return delimiter
}
return strings.Join(id, delimiter)
}
// namespace splits a namespace identifier into the table bucket and the parts of
// the namespace inside it. The root and a bare bucket both return an empty
// namespace, so callers check the bucket to tell them apart.
func (id identifier) namespace() (bucket string, ns []string) {
if len(id) == 0 {
return "", nil
}
return id[0], id[1:]
}
// table splits a table identifier. A table needs a bucket, at least one
// namespace part and a name, because storage has no unnamespaced tables.
func (id identifier) table() (bucket string, ns []string, name string, err error) {
if len(id) < 3 {
return "", nil, "", fmt.Errorf("table identifier needs a bucket, a namespace and a name")
}
return id[0], id[1 : len(id)-1], id[len(id)-1], nil
}
// matchesBody reports whether an identifier carried in the request body agrees
// with the one in the route. The spec requires 400 when both are present and
// differ, and requires the route to win when the body omits it.
func (id identifier) matchesBody(body []string) bool {
if len(body) == 0 {
return true
}
if len(body) != len(id) {
return false
}
for i := range body {
if body[i] != id[i] {
return false
}
}
return true
}

View File

@@ -0,0 +1,104 @@
package lance
import "testing"
func TestParseIdentifier(t *testing.T) {
cases := []struct {
name string
encoded string
delimiter string
want []string
wantErr bool
}{
{name: "root is the delimiter", encoded: "$", delimiter: "$"},
{name: "empty is the root", encoded: "", delimiter: "$"},
{name: "bucket", encoded: "analytics", delimiter: "$", want: []string{"analytics"}},
{name: "namespace", encoded: "analytics$sales", delimiter: "$", want: []string{"analytics", "sales"}},
{name: "table", encoded: "analytics$sales$orders", delimiter: "$", want: []string{"analytics", "sales", "orders"}},
{name: "custom delimiter", encoded: "a.b.c", delimiter: ".", want: []string{"a", "b", "c"}},
{name: "empty part is rejected", encoded: "a$$b", delimiter: "$", wantErr: true},
{name: "trailing delimiter is rejected", encoded: "a$", delimiter: "$", wantErr: true},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got, err := parseIdentifier(c.encoded, c.delimiter)
if c.wantErr {
if err == nil {
t.Fatalf("parseIdentifier(%q) error = nil, want an error", c.encoded)
}
return
}
if err != nil {
t.Fatalf("parseIdentifier(%q) error = %v", c.encoded, err)
}
if len(got) != len(c.want) {
t.Fatalf("parseIdentifier(%q) = %v, want %v", c.encoded, got, c.want)
}
for i := range got {
if got[i] != c.want[i] {
t.Fatalf("parseIdentifier(%q) = %v, want %v", c.encoded, got, c.want)
}
}
})
}
}
func TestIdentifierNamespace(t *testing.T) {
bucket, ns := identifier(nil).namespace()
if bucket != "" || len(ns) != 0 {
t.Fatalf("root namespace() = %q %v, want empty", bucket, ns)
}
bucket, ns = identifier{"analytics"}.namespace()
if bucket != "analytics" || len(ns) != 0 {
t.Fatalf("bucket namespace() = %q %v", bucket, ns)
}
bucket, ns = identifier{"analytics", "sales", "eu"}.namespace()
if bucket != "analytics" || len(ns) != 2 || ns[0] != "sales" || ns[1] != "eu" {
t.Fatalf("nested namespace() = %q %v", bucket, ns)
}
}
// A table always needs a bucket, a namespace and a name: storage has no
// unnamespaced tables, so a two-part identifier is a client error rather than a
// table at the top of a bucket.
func TestIdentifierTable(t *testing.T) {
if _, _, _, err := (identifier{"analytics", "orders"}).table(); err == nil {
t.Fatal("table() on a two-part identifier error = nil, want an error")
}
bucket, ns, name, err := identifier{"analytics", "sales", "eu", "orders"}.table()
if err != nil {
t.Fatalf("table() error = %v", err)
}
if bucket != "analytics" || name != "orders" || len(ns) != 2 || ns[0] != "sales" || ns[1] != "eu" {
t.Fatalf("table() = %q %v %q", bucket, ns, name)
}
}
func TestIdentifierMatchesBody(t *testing.T) {
id := identifier{"a", "b", "c"}
if !id.matchesBody(nil) {
t.Fatal("an absent body identifier must defer to the route")
}
if !id.matchesBody([]string{"a", "b", "c"}) {
t.Fatal("an equal body identifier must match")
}
if id.matchesBody([]string{"a", "b"}) {
t.Fatal("a shorter body identifier must not match")
}
if id.matchesBody([]string{"a", "b", "d"}) {
t.Fatal("a different body identifier must not match")
}
}
func TestIdentifierString(t *testing.T) {
if got := identifier(nil).String("$"); got != "$" {
t.Fatalf("root String() = %q, want %q", got, "$")
}
if got := (identifier{"a", "b"}).String("$"); got != "a$b" {
t.Fatalf("String() = %q, want %q", got, "a$b")
}
}

View File

@@ -0,0 +1,93 @@
package lance
import (
"net/http"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// Marker files the Lance Directory Catalog reads. Writing them beside the
// dataset keeps a client that bypasses this namespace, and lists the storage
// prefix directly, seeing the same table states the catalog reports.
const (
reservedMarker = ".lance-reserved"
deregisteredMarker = ".lance-deregistered"
versionsDir = "_versions"
)
// datasetDir maps a table location onto the filer directory holding it.
func datasetDir(location string) string {
return s3tables.TableDataDirFromMetadataLocation(location)
}
func (s *Server) entryExists(r *http.Request, dir, name string) (bool, error) {
found := false
err := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
_, lookupErr := filer_pb.LookupEntry(r.Context(), client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if lookupErr == filer_pb.ErrNotFound {
return nil
}
if lookupErr != nil {
return lookupErr
}
found = true
return nil
})
return found, err
}
// writeMarker drops a zero-length marker into the dataset directory. The
// directory has to exist already: a table declared through this namespace has
// one, and a table registered at a location that does not yet exist has no
// storage to mark.
func (s *Server) writeMarker(r *http.Request, location, name string) error {
dir := datasetDir(location)
if dir == "" {
return nil
}
now := time.Now().Unix()
return s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return filer_pb.CreateEntry(r.Context(), client, &filer_pb.CreateEntryRequest{
Directory: dir,
Entry: &filer_pb.Entry{
Name: name,
Attributes: &filer_pb.FuseAttributes{
Mtime: now,
Crtime: now,
FileMode: uint32(0644),
},
},
})
})
}
func (s *Server) removeMarker(r *http.Request, location, name string) error {
dir := datasetDir(location)
if dir == "" {
return nil
}
return s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return filer_pb.DoRemove(r.Context(), client, dir, name, true, false, true, false, nil)
})
}
// datasetState reports the two things the spec asks about a table's storage:
// whether it has been deregistered, and whether it holds data yet. A dataset
// holds data once it has a version manifest, which is how a directory catalog
// decides the same question.
func (s *Server) datasetState(r *http.Request, location string) (deregistered, hasData bool, err error) {
dir := datasetDir(location)
if dir == "" {
return false, false, nil
}
if deregistered, err = s.entryExists(r, dir, deregisteredMarker); err != nil {
return false, false, err
}
hasData, err = s.entryExists(r, dir, versionsDir)
return deregistered, hasData, err
}

188
weed/s3api/lance/server.go Normal file
View File

@@ -0,0 +1,188 @@
package lance
import (
"context"
"net/http"
"time"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// FilerClient provides access to the filer for storage operations.
type FilerClient interface {
WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error
}
type S3Authenticator interface {
AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode)
DefaultAllow() bool
}
// VendedCredentials are short-lived S3 credentials scoped to one table.
type VendedCredentials struct {
AccessKeyID string
SecretAccessKey string
SessionToken string
Expiration time.Time
}
// CredentialVendor mints credentials limited to a single table's prefix for a
// caller the catalog has already authenticated and authorized. A nil result with
// no error means the deployment has vending switched off.
type CredentialVendor interface {
VendTableCredentials(ctx context.Context, principal, bucket, prefix string) (*VendedCredentials, error)
}
// Server implements the Lance Namespace REST spec.
type Server struct {
filerClient FilerClient
tablesManager *s3tables.Manager
authenticator S3Authenticator
credentialVendor CredentialVendor
s3Endpoint string
s3Region string
}
// NewServer creates a Lance namespace server over the given filer.
func NewServer(filerClient FilerClient, authenticator S3Authenticator) *Server {
manager := s3tables.NewManager()
// Mirror the S3 port: fall open by default only when the gateway itself is
// open, so an authenticated caller still passes the normal permission check.
if authenticator != nil {
manager.SetDefaultAllow(authenticator.DefaultAllow())
}
return &Server{
filerClient: filerClient,
tablesManager: manager,
authenticator: authenticator,
}
}
// SetCredentialVendor enables storage_options credential vending for clients
// that ask for it with vend_credentials.
func (s *Server) SetCredentialVendor(vendor CredentialVendor) {
s.credentialVendor = vendor
}
// SetS3Endpoint configures the S3 endpoint advertised in storage_options so a
// client can reach the dataset without separately discovering the S3 address.
func (s *Server) SetS3Endpoint(endpoint string) {
s.s3Endpoint = endpoint
}
// SetS3Region configures the region advertised in storage_options.
func (s *Server) SetS3Region(region string) {
s.s3Region = region
}
// RegisterRoutes registers the Lance Namespace REST routes.
//
// The spec puts the identifier in the path rather than the body so a reverse
// proxy can route and authorize without deserializing the request.
func (s *Server) RegisterRoutes(router *mux.Router) {
router.Use(loggingMiddleware)
router.HandleFunc("/v1/namespace/{id}/create", s.Auth(s.handleCreateNamespace)).Methods(http.MethodPost)
router.HandleFunc("/v1/namespace/{id}/list", s.Auth(s.handleListNamespaces)).Methods(http.MethodGet)
router.HandleFunc("/v1/namespace/{id}/describe", s.Auth(s.handleDescribeNamespace)).Methods(http.MethodPost)
router.HandleFunc("/v1/namespace/{id}/drop", s.Auth(s.handleDropNamespace)).Methods(http.MethodPost)
router.HandleFunc("/v1/namespace/{id}/exists", s.Auth(s.handleNamespaceExists)).Methods(http.MethodPost)
router.HandleFunc("/v1/namespace/{id}/table/list", s.Auth(s.handleListTables)).Methods(http.MethodGet)
router.HandleFunc("/v1/table", s.Auth(s.handleListAllTables)).Methods(http.MethodGet)
router.HandleFunc("/v1/table/{id}/declare", s.Auth(s.handleDeclareTable)).Methods(http.MethodPost)
router.HandleFunc("/v1/table/{id}/describe", s.Auth(s.handleDescribeTable)).Methods(http.MethodPost)
router.HandleFunc("/v1/table/{id}/exists", s.Auth(s.handleTableExists)).Methods(http.MethodPost)
router.HandleFunc("/v1/table/{id}/register", s.Auth(s.handleRegisterTable)).Methods(http.MethodPost)
router.HandleFunc("/v1/table/{id}/deregister", s.Auth(s.handleDeregisterTable)).Methods(http.MethodPost)
router.HandleFunc("/v1/table/{id}/drop", s.Auth(s.handleDropTable)).Methods(http.MethodPost)
router.HandleFunc("/v1/table/{id}/rename", s.Auth(s.handleRenameTable)).Methods(http.MethodPost)
// The data plane needs Lance format support that does not exist in Go. Say
// so with the spec's own code instead of returning a bare 404.
for _, action := range []string{"create", "insert", "merge_insert", "update", "delete",
"query", "count_rows", "explain_plan", "analyze_plan", "restore",
"add_columns", "alter_columns", "drop_columns", "backfill_column",
"create_index", "create_scalar_index", "stats", "schema_metadata/update"} {
router.HandleFunc("/v1/table/{id}/"+action, s.Auth(s.handleUnsupported)).Methods(http.MethodPost)
}
// Version ops exist in the spec for stores that cannot order commits
// themselves. Ours can: a Lance commit is a put-if-not-exists, and this S3
// evaluates that precondition at the object's owner filer under a per-path
// lock, so the dataset keeps its own version history and the catalog stays
// out of the commit path.
for _, action := range []string{
"version/create", "version/list", "version/describe", "version/delete",
"index/list", "tags/list", "tags/version", "tags/create", "tags/delete", "tags/update",
"branches/list", "branches/create", "branches/delete"} {
router.HandleFunc("/v1/table/{id}/"+action, s.Auth(s.handleUnsupported)).Methods(http.MethodPost)
}
router.PathPrefix("/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
glog.V(2).Infof("lance: no route for %s %s", r.Method, r.RequestURI)
writeError(w, r, http.StatusNotFound, codeUnsupported, "no such operation")
})
glog.V(2).Infof("Registered Lance Namespace routes")
}
func (s *Server) handleUnsupported(w http.ResponseWriter, r *http.Request) {
writeError(w, r, http.StatusNotImplemented, codeUnsupported,
"this namespace records table metadata only; run data operations through a Lance client against the table location")
}
func loggingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
glog.V(2).Infof("lance request: %s %s from %s", r.Method, r.RequestURI, r.RemoteAddr)
next.ServeHTTP(w, r)
})
}
// Auth authenticates the caller and puts the identity in the request context.
// The Lance spec maps identity onto the same headers the S3 authenticator
// already understands, so SigV4 and bearer tokens both keep working.
func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if s.authenticator == nil {
writeError(w, r, http.StatusUnauthorized, codeUnauthenticated, "authentication required")
return
}
identityName, identity, errCode := s.authenticator.AuthenticateRequest(r)
if errCode != s3err.ErrNone {
if !s.authenticator.DefaultAllow() {
apiErr := s3err.GetAPIError(errCode)
code := codeInternal
switch apiErr.HTTPStatusCode {
case http.StatusForbidden:
code = codePermissionDenied
case http.StatusUnauthorized:
code = codeUnauthenticated
case http.StatusBadRequest:
code = codeInvalidInput
}
writeError(w, r, apiErr.HTTPStatusCode, code, apiErr.Description)
return
}
glog.V(2).Infof("lance: authentication failed (%v) but the gateway is open, proceeding", errCode)
}
if identityName != "" || identity != nil {
ctx := r.Context()
if identityName != "" {
ctx = s3_constants.SetIdentityNameInContext(ctx, identityName)
}
if identity != nil {
ctx = s3_constants.SetIdentityInContext(ctx, identity)
}
r = r.WithContext(ctx)
}
handler(w, r)
}
}

183
weed/s3api/lance/storage.go Normal file
View File

@@ -0,0 +1,183 @@
package lance
import (
"encoding/json"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
const defaultPageSize = 1000
// execute runs one S3 Tables operation as the request's authenticated caller.
func (s *Server) execute(r *http.Request, operation string, req, resp interface{}) error {
identityName := s3_constants.GetIdentityNameFromContext(r)
return s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return s.tablesManager.Execute(r.Context(), s3tables.NewManagerClient(client), operation, req, resp, identityName)
})
}
func bucketARN(bucket string) string {
arn, _ := s3tables.BuildBucketARN(s3tables.DefaultRegion, s3_constants.AccountAdminId, bucket)
return arn
}
// maxRequestBody bounds what one call can make the catalog hold. Every request
// this surface takes is a small JSON envelope; the largest carries a set of
// properties, not data.
const maxRequestBody = 4 << 20
// decodeBody reads an optional JSON request body. Every Lance operation carries
// one, but the fields that matter are also in the route, so an empty body is
// not an error.
func decodeBody(r *http.Request, into interface{}) error {
body, err := io.ReadAll(io.LimitReader(r.Body, maxRequestBody+1))
if err != nil {
return fmt.Errorf("read request body: %w", err)
}
if len(body) > maxRequestBody {
return fmt.Errorf("request body is larger than %d bytes", maxRequestBody)
}
if len(strings.TrimSpace(string(body))) == 0 {
return nil
}
if err := json.Unmarshal(body, into); err != nil {
return fmt.Errorf("invalid request body: %w", err)
}
return nil
}
// routeIdentifier parses the {id} route variable, writing the error response
// itself when the identifier is malformed.
func routeIdentifier(w http.ResponseWriter, r *http.Request) (identifier, string, bool) {
delimiter := requestDelimiter(r)
id, err := parseIdentifier(mux.Vars(r)["id"], delimiter)
if err != nil {
writeError(w, r, http.StatusBadRequest, codeInvalidInput, err.Error())
return nil, delimiter, false
}
return id, delimiter, true
}
// checkBodyIdentifier enforces the spec rule that a route and a body naming
// different objects is a bad request rather than a silent preference.
func checkBodyIdentifier(w http.ResponseWriter, r *http.Request, id identifier, body []string) bool {
if id.matchesBody(body) {
return true
}
writeError(w, r, http.StatusBadRequest, codeInvalidInput,
"the identifier in the request body does not match the one in the route")
return false
}
func pageSize(r *http.Request) int {
if raw := r.URL.Query().Get("limit"); raw != "" {
if parsed, err := strconv.Atoi(raw); err == nil && parsed > 0 {
return parsed
}
}
return defaultPageSize
}
func boolQuery(r *http.Request, name string) bool {
value, err := strconv.ParseBool(r.URL.Query().Get(name))
return err == nil && value
}
// wants resolves a tri-state request flag: the query parameter the REST spec
// adds, else the body field, else the implementation's own choice.
func wants(r *http.Request, name string, body *bool) bool {
if boolQuery(r, name) {
return true
}
return body != nil && *body
}
// tableLocation is where a table's dataset lives when the caller does not name
// a location. It mirrors the Iceberg catalog's layout so both catalogs put a
// table of the same name in the same place.
func tableLocation(bucket string, ns []string, name string) string {
return fmt.Sprintf("s3://%s/%s/%s", bucket, strings.Join(ns, "."), name)
}
// confineLocation rejects a client-supplied table location that does not resolve
// inside the caller's own bucket, the way the Iceberg gateway confines the same
// field. The location feeds TableDataDirFromMetadataLocation, which joins it
// under /buckets and collapses any "../" segments; an unconfined location lets a
// marker write escape into another tenant's namespace.
func confineLocation(bucket, location string) error {
rest, ok := strings.CutPrefix(location, "s3://")
if !ok {
return fmt.Errorf("location must be an s3:// URI")
}
name, keys, _ := strings.Cut(rest, "/")
if name != bucket {
return fmt.Errorf("location must be within bucket %s", bucket)
}
for _, segment := range strings.Split(keys, "/") {
switch {
case segment == "." || segment == "..":
return fmt.Errorf("location must not contain a path traversal segment")
case strings.ContainsAny(segment, "\\\x00"):
return fmt.Errorf("location contains an invalid character")
}
}
return nil
}
// storageOptions builds the object_store settings a Lance client needs to reach
// the dataset. The key names are the aws_-prefixed forms Lance clients pass
// through to object_store.
func (s *Server) storageOptions(r *http.Request, bucket, location string, vend bool) (map[string]string, error) {
options := map[string]string{}
if s.s3Endpoint != "" {
options["aws_endpoint"] = s.s3Endpoint
if strings.HasPrefix(s.s3Endpoint, "http://") {
// object_store refuses a plaintext endpoint unless told to allow it,
// and the resulting failure reads like a credential problem.
options["allow_http"] = "true"
}
}
if s.s3Region != "" {
options["aws_region"] = s.s3Region
}
if !vend || s.credentialVendor == nil {
return options, nil
}
principal := s3_constants.GetIdentityNameFromContext(r)
credentials, err := s.credentialVendor.VendTableCredentials(r.Context(), principal, bucket, locationPrefix(location))
if err != nil {
return nil, err
}
if credentials == nil {
return options, nil
}
options["aws_access_key_id"] = credentials.AccessKeyID
options["aws_secret_access_key"] = credentials.SecretAccessKey
if credentials.SessionToken != "" {
options["aws_session_token"] = credentials.SessionToken
}
if !credentials.Expiration.IsZero() {
options["expires_at_millis"] = strconv.FormatInt(credentials.Expiration.UnixMilli(), 10)
}
return options, nil
}
// locationPrefix strips the s3://bucket/ part of a location, leaving the key
// prefix a credential is scoped to.
func locationPrefix(location string) string {
trimmed := strings.TrimPrefix(location, "s3://")
if _, prefix, found := strings.Cut(trimmed, "/"); found {
return prefix
}
return ""
}

149
weed/s3api/lance/types.go Normal file
View File

@@ -0,0 +1,149 @@
// Package lance serves the Lance Namespace REST spec over the same table
// buckets the Iceberg REST catalog uses. A Lance table is a catalog entry with
// format LANCE: the namespace records where the dataset lives and vends
// credentials for it, and the Lance client owns everything under that location.
package lance
// Modes shared by create and register. The spec matches them case-insensitively
// and accepts both PascalCase and snake_case.
const (
modeCreate = "create"
modeExistOk = "existok"
modeOverwrite = "overwrite"
modeFail = "fail"
modeSkip = "skip"
behaviorRestrict = "restrict"
behaviorCascade = "cascade"
)
type CreateNamespaceRequest struct {
ID []string `json:"id,omitempty"`
Mode string `json:"mode,omitempty"`
Properties map[string]string `json:"properties,omitempty"`
}
type CreateNamespaceResponse struct {
Properties map[string]string `json:"properties"`
}
type ListNamespacesResponse struct {
Namespaces []string `json:"namespaces"`
PageToken string `json:"page_token,omitempty"`
}
type DescribeNamespaceRequest struct {
ID []string `json:"id,omitempty"`
}
type DescribeNamespaceResponse struct {
Properties map[string]string `json:"properties"`
}
type DropNamespaceRequest struct {
ID []string `json:"id,omitempty"`
Mode string `json:"mode,omitempty"`
Behavior string `json:"behavior,omitempty"`
}
type DropNamespaceResponse struct {
Properties map[string]string `json:"properties,omitempty"`
}
type NamespaceExistsRequest struct {
ID []string `json:"id,omitempty"`
}
type ListTablesResponse struct {
Tables []string `json:"tables"`
PageToken string `json:"page_token,omitempty"`
}
type DeclareTableRequest struct {
ID []string `json:"id,omitempty"`
Location string `json:"location,omitempty"`
VendCredentials *bool `json:"vend_credentials,omitempty"`
Properties map[string]string `json:"properties,omitempty"`
}
type DeclareTableResponse struct {
Location string `json:"location"`
StorageOptions map[string]string `json:"storage_options,omitempty"`
// Properties are null rather than {}: the catalog does not keep a table's
// properties, and {} would claim it kept them and found none.
Properties map[string]string `json:"properties"`
// ManagedVersioning stays false: the dataset owns its version history,
// because this store can order commits without the catalog in the path.
ManagedVersioning bool `json:"managed_versioning"`
}
type DescribeTableRequest struct {
ID []string `json:"id,omitempty"`
Version *int64 `json:"version,omitempty"`
Tag string `json:"tag,omitempty"`
Branch string `json:"branch,omitempty"`
// The REST spec carries these as query parameters, but clients also put
// them in the body, so honour both.
WithTableURI *bool `json:"with_table_uri,omitempty"`
LoadDetailedMetadata *bool `json:"load_detailed_metadata,omitempty"`
CheckDeclared *bool `json:"check_declared,omitempty"`
VendCredentials *bool `json:"vend_credentials,omitempty"`
}
type DescribeTableResponse struct {
Table string `json:"table,omitempty"`
Namespace []string `json:"namespace,omitempty"`
Version *int64 `json:"version,omitempty"`
Location string `json:"location"`
TableURI string `json:"table_uri,omitempty"`
StorageOptions map[string]string `json:"storage_options,omitempty"`
// Null, not {}: this catalog does not keep a table's properties.
Properties map[string]string `json:"properties"`
// ManagedVersioning stays false: the dataset owns its version history,
// because this store can order commits without the catalog in the path.
ManagedVersioning bool `json:"managed_versioning"`
IsOnlyDeclared *bool `json:"is_only_declared,omitempty"`
}
type TableExistsRequest struct {
ID []string `json:"id,omitempty"`
}
type RegisterTableRequest struct {
ID []string `json:"id,omitempty"`
Location string `json:"location"`
Mode string `json:"mode,omitempty"`
Properties map[string]string `json:"properties,omitempty"`
}
type RegisterTableResponse struct {
Location string `json:"location"`
Properties map[string]string `json:"properties"`
}
type DeregisterTableRequest struct {
ID []string `json:"id,omitempty"`
}
type DeregisterTableResponse struct {
ID []string `json:"id"`
Location string `json:"location"`
Properties map[string]string `json:"properties"`
}
type DropTableRequest struct {
ID []string `json:"id,omitempty"`
}
type DropTableResponse struct {
ID []string `json:"id"`
Location string `json:"location"`
Properties map[string]string `json:"properties"`
}
type RenameTableRequest struct {
ID []string `json:"id,omitempty"`
NewID []string `json:"new_id"`
}
type RenameTableResponse struct{}