snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/s3tables/catalog/issue_9103_test.go

@@ -0,0 +1,283 @@
+// Reproduction tests for https://github.com/seaweedfs/seaweedfs/issues/9103
+//
+// The issue reports two distinct failure modes when using DuckDB against the
+// SeaweedFS Iceberg REST catalog:
+//
+//  1. `ATTACH 's3://test/' AS cat (TYPE 'ICEBERG', ...); SELECT * FROM cat.ovirt.disk;`
+//     fails with "Table with name 'ovirt.disk' does not exist because schema
+//     'ovirt' does not exist." The namespace "ovirt" does exist in the bucket.
+//
+//  2. `SELECT * FROM iceberg_scan('s3://test/ovirt/disk');` fails with HTTP 403
+//     AccessDenied when DuckDB tries to glob `s3://test/ovirt/disk/metadata/v*`
+//     because the LoadTable response does not vend S3 file-io credentials back
+//     to the client.
+//
+// These tests reproduce the underlying catalog-level misbehavior at the REST
+// protocol layer so they run quickly and deterministically. A DuckDB-based
+// end-to-end reproduction is also included (gated on Docker availability).
+package catalog
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+)
+
+// TestIssue9103_ConfigDoesNotVendWarehousePrefix reproduces failure mode #2.
+//
+// Per the Iceberg REST spec, a client attaching with warehouse=s3://<bucket>/
+// calls GET /v1/config?warehouse=s3://<bucket>/ and expects the server to
+// return an `overrides.prefix` that identifies the catalog namespace for
+// subsequent requests. Without it the client falls back to unprefixed paths
+// like /v1/namespaces, which on SeaweedFS resolve to the hard-coded default
+// bucket ("warehouse") and therefore do not list the user's namespaces.
+//
+// Currently the handler ignores the warehouse query parameter and returns
+// empty defaults/overrides. That is exactly what makes DuckDB's ATTACH flow
+// report `schema "ovirt" does not exist`.
+func TestIssue9103_ConfigDoesNotVendWarehousePrefix(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env := sharedEnv
+	bucketName := "warehouse-9103cfg-" + randomSuffix()
+	createTableBucket(t, env, bucketName, "")
+
+	warehouse := fmt.Sprintf("s3://%s/", bucketName)
+	u := fmt.Sprintf("%s/v1/config?warehouse=%s", env.IcebergURL(), url.QueryEscape(warehouse))
+
+	status, body, err := doIcebergJSONRequest(env, http.MethodGet, fmt.Sprintf("/v1/config?warehouse=%s", url.QueryEscape(warehouse)), nil)
+	if err != nil {
+		t.Fatalf("GET %s failed: %v", u, err)
+	}
+	if status != http.StatusOK {
+		t.Fatalf("GET %s status = %d, want 200", u, status)
+	}
+
+	overrides, _ := body["overrides"].(map[string]any)
+	gotPrefix, _ := overrides["prefix"].(string)
+	if gotPrefix != bucketName {
+		t.Fatalf("GET /v1/config?warehouse=%s: overrides.prefix = %q, want %q (catalog must echo the warehouse's table bucket so clients like DuckDB know which /v1/{prefix}/namespaces to use)",
+			warehouse, gotPrefix, bucketName)
+	}
+}
+
+// TestIssue9103_BareNamespacesListMissesNamespaceInAttachedBucket
+// demonstrates the downstream effect of the /v1/config bug.
+//
+// The client created the namespace "ovirt" inside bucket "test" (via
+// /v1/{bucket}/namespaces), but when it later issues GET /v1/namespaces (no
+// prefix, because /v1/config didn't give it one), that request resolves to
+// the default bucket and returns no namespaces. DuckDB surfaces this as
+// `schema "ovirt" does not exist`.
+func TestIssue9103_BareNamespacesListMissesNamespaceInAttachedBucket(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env := sharedEnv
+	bucketName := "warehouse-9103ns-" + randomSuffix()
+	createTableBucket(t, env, bucketName, "")
+
+	namespace := "ovirt"
+	status, _, err := doIcebergJSONRequest(env, http.MethodPost,
+		icebergPath(bucketName, "/v1/namespaces"),
+		map[string]any{"namespace": []string{namespace}})
+	if err != nil {
+		t.Fatalf("create namespace: %v", err)
+	}
+	if status != http.StatusOK && status != http.StatusConflict {
+		t.Fatalf("create namespace status = %d, want 200 or 409", status)
+	}
+
+	// Sanity: prefixed listing finds the namespace.
+	status, prefixedBody, err := doIcebergJSONRequest(env, http.MethodGet,
+		icebergPath(bucketName, "/v1/namespaces"), nil)
+	if err != nil {
+		t.Fatalf("list namespaces with prefix: %v", err)
+	}
+	if status != http.StatusOK {
+		t.Fatalf("prefixed list status = %d, want 200", status)
+	}
+	if !containsNamespace(prefixedBody, namespace) {
+		t.Fatalf("prefixed list missing namespace %q: %v", namespace, prefixedBody)
+	}
+
+	// Simulate what a DuckDB-style client does when /v1/config did not vend a
+	// prefix: it falls back to the unprefixed listing. It should still be
+	// able to discover the namespace (e.g. via a warehouse query parameter),
+	// but today it cannot, which is the user-visible bug.
+	bareWarehouse := "s3://" + bucketName + "/"
+	status, bareBody, err := doIcebergJSONRequest(env, http.MethodGet,
+		fmt.Sprintf("/v1/namespaces?warehouse=%s", url.QueryEscape(bareWarehouse)), nil)
+	if err != nil {
+		t.Fatalf("bare list namespaces: %v", err)
+	}
+	switch {
+	case status != http.StatusOK:
+		t.Fatalf("GET /v1/namespaces?warehouse=%s status = %d, want 200 (server ignores the warehouse query parameter and falls back to the default %q bucket, causing DuckDB to see no schemas)",
+			bareWarehouse, status, "warehouse")
+	case !containsNamespace(bareBody, namespace):
+		t.Fatalf("GET /v1/namespaces?warehouse=%s did not return %q: %v (server must honor the warehouse query parameter here or vend overrides.prefix from /v1/config so the client can route to the right bucket)",
+			bareWarehouse, namespace, bareBody)
+	}
+}
+
+// TestIssue9103_DuckDBAttachCannotResolveNamespace is the end-to-end
+// reproduction of the issue using DuckDB in Docker, mirroring the exact
+// sequence of commands from the bug report. It runs under its own weed
+// mini with IAM configured so the OAuth2 client_credentials flow that
+// DuckDB's iceberg extension requires actually works (the shared env has
+// no credentials registered). Gated on Docker availability.
+func TestIssue9103_DuckDBAttachCannotResolveNamespace(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	if !testutil.HasDocker() {
+		t.Skip("Docker not available, skipping DuckDB integration reproduction")
+	}
+
+	env := newOAuthTestEnv(t)
+	defer env.cleanup(t)
+	env.start(t)
+
+	bucketName := "test9103-" + randomSuffix()
+	createTableBucketViaShell(t, env, bucketName)
+
+	namespace := "ovirt"
+	tableName := "disk"
+
+	token := requestOAuthToken(t, env, env.accessKey, env.secretKey)
+	createNamespaceWithToken(t, env, token, bucketName, namespace)
+	createTableWithToken(t, env, token, bucketName, namespace, tableName)
+
+	sql := fmt.Sprintf(`
+INSTALL iceberg;
+LOAD iceberg;
+
+CREATE SECRET test_berg (
+    TYPE ICEBERG,
+    ENDPOINT 'http://host.docker.internal:%d',
+    SCOPE 's3://%s/',
+    CLIENT_ID '%s',
+    CLIENT_SECRET '%s'
+);
+
+CREATE SECRET s3_berg (
+    TYPE S3,
+    KEY_ID '%s',
+    SECRET '%s',
+    ENDPOINT 'host.docker.internal:%d',
+    URL_STYLE 'path',
+    USE_SSL false,
+    SCOPE 's3://%s/'
+);
+
+ATTACH 's3://%s/' AS test_catalog (TYPE 'ICEBERG', secret test_berg);
+SELECT * FROM test_catalog.%s.%s LIMIT 0;
+`, env.icebergPort, bucketName,
+		env.accessKey, env.secretKey,
+		env.accessKey, env.secretKey, env.s3Port, bucketName,
+		bucketName, namespace, tableName)
+
+	sqlFile := filepath.Join(env.dataDir, "issue_9103.sql")
+	if err := os.WriteFile(sqlFile, []byte(sql), 0644); err != nil {
+		t.Fatalf("write SQL: %v", err)
+	}
+
+	cmd := exec.Command("docker", "run", "--rm",
+		"-v", fmt.Sprintf("%s:/test", env.dataDir),
+		"--add-host", "host.docker.internal:host-gateway",
+		"-e", "AWS_REGION=us-east-1",
+		"--entrypoint", "duckdb",
+		"duckdb/duckdb:latest",
+		"-init", "/test/issue_9103.sql",
+		"-c", "SELECT 1",
+	)
+
+	out, runErr := cmd.CombinedOutput()
+	outStr := string(out)
+	t.Logf("DuckDB output:\n%s", outStr)
+
+	if strings.Contains(outStr, "iceberg extension is not available") ||
+		strings.Contains(outStr, "Failed to load") ||
+		strings.Contains(outStr, "could not fetch extension") ||
+		strings.Contains(outStr, "failed to download") {
+		t.Skip("Iceberg extension not available in DuckDB Docker image")
+	}
+
+	if runErr != nil {
+		if strings.Contains(outStr, "does not exist because schema") &&
+			strings.Contains(outStr, namespace) {
+			t.Fatalf("reproduced issue #9103: DuckDB cannot see namespace %q after ATTACH 's3://%s/'; see output above",
+				namespace, bucketName)
+		}
+		t.Fatalf("DuckDB run failed for an unexpected reason: %v", runErr)
+	}
+}
+
+// createTableWithToken creates an Iceberg table using an OAuth bearer token.
+func createTableWithToken(t *testing.T, env *oauthTestEnv, token, bucketName, namespace, tableName string) {
+	t.Helper()
+
+	payload := map[string]any{
+		"name": tableName,
+		"schema": map[string]any{
+			"type":      "struct",
+			"schema-id": 0,
+			"fields": []map[string]any{
+				{"id": 1, "name": "id", "required": true, "type": "long"},
+			},
+		},
+	}
+	body, err := json.Marshal(payload)
+	if err != nil {
+		t.Fatalf("marshal create-table payload: %v", err)
+	}
+
+	url := fmt.Sprintf("%s/v1/%s/namespaces/%s/tables", env.icebergURL(), bucketName, namespace)
+	req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
+	if err != nil {
+		t.Fatalf("create request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "Bearer "+token)
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("create table: %v", err)
+	}
+	defer resp.Body.Close()
+
+	respBody, _ := io.ReadAll(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("create table failed: status=%d body=%s", resp.StatusCode, respBody)
+	}
+	t.Logf("Created table %s.%s in bucket %s", namespace, tableName, bucketName)
+}
+
+// containsNamespace checks whether an Iceberg REST ListNamespaces response
+// contains the given single-level namespace.
+func containsNamespace(body map[string]any, name string) bool {
+	arr, _ := body["namespaces"].([]any)
+	for _, item := range arr {
+		parts, _ := item.([]any)
+		if len(parts) == 1 {
+			if s, ok := parts[0].(string); ok && s == name {
+				return true
+			}
+		}
+	}
+	return false
+}