snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,248 @@
package catalog_trino
import (
"fmt"
"os/exec"
"strconv"
"strings"
"testing"
"time"
)
func TestTrinoBlogOperations(t *testing.T) {
env := setupTrinoTest(t)
defer env.Cleanup(t)
schemaName := "blog_ns_" + randomString(6)
customersTable := "customers_" + randomString(6)
trinoCustomersTable := "trino_customers_" + randomString(6)
warehouseBucket := "iceberg-tables"
customersLocation := fmt.Sprintf("s3://%s/%s/%s_%s", warehouseBucket, schemaName, customersTable, randomString(6))
trinoCustomersLocation := fmt.Sprintf("s3://%s/%s/%s_%s", warehouseBucket, schemaName, trinoCustomersTable, randomString(6))
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS iceberg.%s", schemaName))
defer runTrinoSQLAllowNamespaceNotEmpty(t, env.trinoContainer, fmt.Sprintf("DROP SCHEMA IF EXISTS iceberg.%s CASCADE", schemaName))
defer runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS iceberg.%s.%s", schemaName, trinoCustomersTable))
defer runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS iceberg.%s.%s", schemaName, customersTable))
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS iceberg.%s.%s", schemaName, trinoCustomersTable))
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS iceberg.%s.%s", schemaName, customersTable))
createCustomersSQL := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS iceberg.%s.%s (
customer_sk INT,
customer_id VARCHAR,
salutation VARCHAR,
first_name VARCHAR,
last_name VARCHAR,
preferred_cust_flag VARCHAR,
birth_day INT,
birth_month INT,
birth_year INT,
birth_country VARCHAR,
login VARCHAR
) WITH (
format = 'PARQUET',
sorted_by = ARRAY['customer_id'],
location = '%s'
)`, schemaName, customersTable, customersLocation)
runTrinoSQLAllowExists(t, env.trinoContainer, createCustomersSQL)
insertCustomersSQL := fmt.Sprintf(`INSERT INTO iceberg.%s.%s VALUES
(1, 'AAAAA', 'Mrs', 'Amanda', 'Olson', 'Y', 8, 4, 1984, 'US', 'aolson'),
(2, 'AAAAB', 'Mr', 'Leonard', 'Eads', 'N', 22, 6, 2001, 'US', 'leads'),
(3, 'BAAAA', 'Mr', 'David', 'White', 'Y', 16, 2, 1999, 'US', 'dwhite'),
(4, 'BBAAA', 'Mr', 'Melvin', 'Lee', 'N', 30, 3, 1973, 'US', 'mlee'),
(5, 'AACAA', 'Mr', 'Donald', 'Holt', 'N', 2, 6, 1982, 'CA', 'dholt'),
(6, 'ABAAA', 'Mrs', 'Jacqueline', 'Harvey', 'N', 5, 12, 1988, 'US', 'jharvey'),
(7, 'BBAAA', 'Ms', 'Debbie', 'Ward', 'N', 6, 1, 2006, 'MX', 'dward'),
(8, 'ACAAA', 'Mr', 'Tim', 'Strong', 'N', 15, 7, 1976, 'US', 'tstrong')
`, schemaName, customersTable)
runTrinoSQL(t, env.trinoContainer, insertCustomersSQL)
countOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s", schemaName, customersTable))
rowCount := mustParseCSVInt64(t, countOutput)
if rowCount != 8 {
t.Fatalf("expected 8 rows in customers table, got %d", rowCount)
}
output := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT first_name FROM iceberg.%s.%s WHERE customer_sk = 1", schemaName, customersTable))
if !strings.Contains(output, "Amanda") {
t.Fatalf("expected sample query to include Amanda, got: %s", output)
}
ctasSQL := fmt.Sprintf(`CREATE TABLE iceberg.%s.%s
WITH (
format = 'PARQUET',
location = '%s'
)
AS SELECT * FROM iceberg.%s.%s`, schemaName, trinoCustomersTable, trinoCustomersLocation, schemaName, customersTable)
ctasInsertSQL := fmt.Sprintf("INSERT INTO iceberg.%s.%s SELECT * FROM iceberg.%s.%s", schemaName, trinoCustomersTable, schemaName, customersTable)
ctasDeleteSQL := fmt.Sprintf("DELETE FROM iceberg.%s.%s", schemaName, trinoCustomersTable)
runTrinoCTAS(t, env.trinoContainer, ctasSQL, ctasDeleteSQL, ctasInsertSQL)
countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s", schemaName, trinoCustomersTable))
rowCount = mustParseCSVInt64(t, countOutput)
if rowCount != 8 {
t.Fatalf("expected 8 rows in CTAS table, got %d", rowCount)
}
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("ALTER TABLE iceberg.%s.%s ADD COLUMN updated_at TIMESTAMP", schemaName, trinoCustomersTable))
output = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DESCRIBE iceberg.%s.%s", schemaName, trinoCustomersTable))
if !strings.Contains(output, "updated_at") {
t.Fatalf("expected updated_at column in describe output, got: %s", output)
}
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("UPDATE iceberg.%s.%s SET updated_at = current_timestamp", schemaName, trinoCustomersTable))
// Sleep to ensure timestamps are in the past for time travel queries
time.Sleep(1 * time.Second)
snapshotOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(`SELECT snapshot_id FROM iceberg.%s."%s$snapshots" ORDER BY committed_at DESC LIMIT 1`, schemaName, trinoCustomersTable))
snapshotID := mustParseCSVInt64(t, snapshotOutput)
if snapshotID == 0 {
t.Fatalf("expected snapshot ID from snapshots table, got 0")
}
filesOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(`SELECT file_path FROM iceberg.%s."%s$files" LIMIT 1`, schemaName, trinoCustomersTable))
if !hasCSVDataRow(filesOutput) {
t.Fatalf("expected files metadata rows, got: %s", filesOutput)
}
historyOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(`SELECT made_current_at FROM iceberg.%s."%s$history" LIMIT 1`, schemaName, trinoCustomersTable))
if !hasCSVDataRow(historyOutput) {
t.Fatalf("expected history metadata rows, got: %s", historyOutput)
}
countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s FOR VERSION AS OF %d", schemaName, trinoCustomersTable, snapshotID))
versionCount := mustParseCSVInt64(t, countOutput)
if versionCount != 8 {
t.Fatalf("expected 8 rows for version time travel, got %d", versionCount)
}
// Use current_timestamp - interval '1 second' to ensure it's in the past (Iceberg requirement)
countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s FOR TIMESTAMP AS OF (current_timestamp - interval '1' second)", schemaName, trinoCustomersTable))
timestampCount := mustParseCSVInt64(t, countOutput)
if timestampCount != 8 {
t.Fatalf("expected 8 rows for timestamp time travel, got %d", timestampCount)
}
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DELETE FROM iceberg.%s.%s WHERE customer_sk = 8", schemaName, trinoCustomersTable))
countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s", schemaName, trinoCustomersTable))
rowCount = mustParseCSVInt64(t, countOutput)
if rowCount != 7 {
t.Fatalf("expected 7 rows after delete, got %d", rowCount)
}
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("ALTER TABLE iceberg.%s.%s EXECUTE rollback_to_snapshot(%d)", schemaName, trinoCustomersTable, snapshotID))
countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM iceberg.%s.%s", schemaName, trinoCustomersTable))
rowCount = mustParseCSVInt64(t, countOutput)
if rowCount != 8 {
t.Fatalf("expected 8 rows after rollback, got %d", rowCount)
}
}
func runTrinoSQLAllowExists(t *testing.T, containerName, sql string) string {
t.Helper()
cmd := exec.Command("docker", "exec", containerName,
"trino", "--catalog", "iceberg",
"--output-format", "CSV",
"--execute", sql,
)
output, err := cmd.CombinedOutput()
if err != nil {
outputStr := string(output)
if strings.Contains(outputStr, "already exists") {
return sanitizeTrinoOutput(outputStr)
}
t.Fatalf("Trino command failed: %v\nSQL: %s\nOutput:\n%s", err, sql, outputStr)
}
return sanitizeTrinoOutput(string(output))
}
func runTrinoSQLAllowNamespaceNotEmpty(t *testing.T, containerName, sql string) string {
t.Helper()
var output []byte
var err error
for attempt := 0; attempt < 3; attempt++ {
cmd := exec.Command("docker", "exec", containerName,
"trino", "--catalog", "iceberg",
"--output-format", "CSV",
"--execute", sql,
)
output, err = cmd.CombinedOutput()
if err == nil {
return sanitizeTrinoOutput(string(output))
}
outputStr := string(output)
if !strings.Contains(outputStr, "Namespace is not empty") {
t.Fatalf("Trino command failed: %v\nSQL: %s\nOutput:\n%s", err, sql, outputStr)
}
time.Sleep(500 * time.Millisecond)
}
t.Logf("Ignoring cleanup error for SQL %s: %s", sql, sanitizeTrinoOutput(string(output)))
return sanitizeTrinoOutput(string(output))
}
func runTrinoCTAS(t *testing.T, containerName, createSQL, deleteSQL, insertSQL string) {
t.Helper()
cmd := exec.Command("docker", "exec", containerName,
"trino", "--catalog", "iceberg",
"--output-format", "CSV",
"--execute", createSQL,
)
output, err := cmd.CombinedOutput()
if err != nil {
outputStr := string(output)
if strings.Contains(outputStr, "already exists") {
if deleteSQL != "" {
runTrinoSQL(t, containerName, deleteSQL)
}
runTrinoSQL(t, containerName, insertSQL)
return
}
t.Fatalf("Trino command failed: %v\nSQL: %s\nOutput:\n%s", err, createSQL, outputStr)
}
}
func hasCSVDataRow(output string) bool {
lines := strings.Split(strings.TrimSpace(output), "\n")
if len(lines) == 0 {
return false
}
for _, line := range lines {
if strings.TrimSpace(line) != "" {
return true
}
}
return false
}
func mustParseCSVInt64(t *testing.T, output string) int64 {
t.Helper()
value := mustFirstCSVValue(t, output)
parsed, err := strconv.ParseInt(value, 10, 64)
if err != nil {
t.Fatalf("failed to parse int from output %q: %v", output, err)
}
return parsed
}
func mustFirstCSVValue(t *testing.T, output string) string {
t.Helper()
lines := strings.Split(strings.TrimSpace(output), "\n")
if len(lines) == 0 {
t.Fatalf("expected CSV output with data row, got: %q", output)
}
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
parts := strings.Split(line, ",")
return strings.Trim(parts[0], "\"")
}
t.Fatalf("no CSV data rows found in output: %q", output)
return ""
}

View File

@@ -0,0 +1,616 @@
package catalog_trino
import (
"context"
"crypto/rand"
"fmt"
"io"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/seaweedfs/seaweedfs/test/testutil"
)
type TestEnvironment struct {
seaweedDir string
weedBinary string
dataDir string
bindIP string
s3Port int
s3GrpcPort int
icebergPort int
masterPort int
masterGrpcPort int
filerPort int
filerGrpcPort int
volumePort int
volumeGrpcPort int
weedProcess *exec.Cmd
weedCancel context.CancelFunc
trinoContainer string
dockerAvailable bool
accessKey string
secretKey string
}
func TestTrinoIcebergCatalog(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
env := NewTestEnvironment(t)
defer env.Cleanup(t)
if !env.dockerAvailable {
t.Skip("Docker not available, skipping Trino integration test")
}
fmt.Printf(">>> Starting SeaweedFS...\n")
env.StartSeaweedFS(t)
fmt.Printf(">>> SeaweedFS started.\n")
tableBucket := "iceberg-tables"
catalogBucket := tableBucket
fmt.Printf(">>> Creating table bucket: %s\n", tableBucket)
createTableBucket(t, env, tableBucket)
fmt.Printf(">>> All buckets created.\n")
// Test Iceberg REST API directly
testIcebergRestAPI(t, env)
configDir := env.writeTrinoConfig(t, catalogBucket)
env.startTrinoContainer(t, configDir)
waitForTrino(t, env.trinoContainer, 60*time.Second)
schemaName := "trino_" + randomString(6)
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS iceberg.%s", schemaName))
output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
if !strings.Contains(output, schemaName) {
t.Fatalf("Expected schema %s in output:\n%s", schemaName, output)
}
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SHOW TABLES FROM iceberg.%s", schemaName))
}
// TestTrinoMultiLevelNamespace tests that multi-level namespaces (dot-separated)
// produce correct S3 paths so Trino can read back data it writes.
// Regression test for https://github.com/seaweedfs/seaweedfs/issues/8959
func TestTrinoMultiLevelNamespace(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
env := NewTestEnvironment(t)
defer env.Cleanup(t)
if !env.dockerAvailable {
t.Skip("Docker not available, skipping Trino integration test")
}
t.Logf(">>> Starting SeaweedFS...")
env.StartSeaweedFS(t)
tableBucket := "iceberg-tables"
createTableBucket(t, env, tableBucket)
configDir := env.writeTrinoConfig(t, tableBucket, withNestedNamespace())
env.startTrinoContainer(t, configDir)
waitForTrino(t, env.trinoContainer, 60*time.Second)
// Use a two-level namespace: "analytics.daily"
nsLevel1 := "analytics_" + randomString(4)
nsLevel2 := "daily_" + randomString(4)
flatNs := fmt.Sprintf("%s.%s", nsLevel1, nsLevel2)
// Trino uses double-quoted schema names for multi-level namespaces
multiNs := fmt.Sprintf(`"%s"`, flatNs)
tableName := "events_" + randomString(4)
// Create multi-level namespace (schema)
t.Logf(">>> Creating multi-level schema: %s", flatNs)
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS iceberg.%s", multiNs))
// Verify the schema shows up
output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
if !strings.Contains(output, flatNs) {
t.Fatalf("Expected schema %s in output:\n%s", flatNs, output)
}
// Create table with explicit location to avoid non-empty location conflict.
// The location uses the dot-separated namespace — if #8959 regresses
// (unit separator instead of dot), data would be written to the wrong path.
tableLocation := fmt.Sprintf("s3://%s/%s/%s_%s", tableBucket, flatNs, tableName, randomString(6))
t.Logf(">>> Creating table at location: %s", tableLocation)
createSQL := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS iceberg.%s.%s (
id INTEGER,
event VARCHAR,
ts TIMESTAMP(6)
) WITH (
format = 'PARQUET',
location = '%s'
)`, multiNs, tableName, tableLocation)
runTrinoSQLAllowExists(t, env.trinoContainer, createSQL)
// Insert data
t.Logf(">>> Inserting data into multi-level namespace table")
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(`
INSERT INTO iceberg.%s.%s VALUES
(1, 'click', TIMESTAMP '2025-01-01 00:00:00'),
(2, 'view', TIMESTAMP '2025-01-01 01:00:00'),
(3, 'click', TIMESTAMP '2025-01-02 00:00:00')
`, multiNs, tableName))
// Query data back — if the namespace path separator were wrong (\x1F
// instead of "."), the metadata location would point to a non-existent
// S3 path and this query would fail.
t.Logf(">>> Querying data from multi-level namespace table")
countOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
"SELECT count(*) FROM iceberg.%s.%s", multiNs, tableName))
rowCount := mustParseCSVInt64(t, countOutput)
if rowCount != 3 {
t.Fatalf("expected row count 3, got %d", rowCount)
}
// Verify the S3 file path contains the dot-separated namespace, not \x1F.
filesOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
`SELECT file_path FROM iceberg.%s."%s$files" LIMIT 1`, multiNs, tableName))
filePath := strings.TrimSpace(filesOutput)
if filePath == "" {
t.Fatalf("expected at least one data file, got empty output")
}
if !strings.Contains(filePath, flatNs+"/") {
t.Errorf("expected file path to contain dot-separated namespace %q, got: %s", flatNs, filePath)
}
if strings.Contains(filePath, "\x1F") {
t.Errorf("file path contains unit separator (\\x1F), expected dot separator: %s", filePath)
}
t.Logf(">>> Trino multi-level namespace test passed")
}
func NewTestEnvironment(t *testing.T) *TestEnvironment {
t.Helper()
wd, err := os.Getwd()
if err != nil {
t.Fatalf("Failed to get working directory: %v", err)
}
seaweedDir := wd
for i := 0; i < 6; i++ {
if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
break
}
seaweedDir = filepath.Dir(seaweedDir)
}
weedBinary := filepath.Join(seaweedDir, "weed", "weed")
info, err := os.Stat(weedBinary)
if err != nil || info.IsDir() {
// Try looking for weed/weed/weed
weedBinary = filepath.Join(seaweedDir, "weed", "weed", "weed")
info, err = os.Stat(weedBinary)
if err != nil || info.IsDir() {
weedBinary = "weed"
if _, err := exec.LookPath(weedBinary); err != nil {
t.Skip("weed binary not found, skipping integration test")
}
}
}
dataDir, err := os.MkdirTemp("", "seaweed-trino-test-*")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
bindIP := testutil.FindBindIP()
// 9 ports: master(2), volume(2), filer(2), s3(2), iceberg(1)
ports := testutil.MustAllocatePorts(t, 9)
env := &TestEnvironment{
seaweedDir: seaweedDir,
weedBinary: weedBinary,
dataDir: dataDir,
bindIP: bindIP,
masterPort: ports[0],
masterGrpcPort: ports[1],
volumePort: ports[2],
volumeGrpcPort: ports[3],
filerPort: ports[4],
filerGrpcPort: ports[5],
s3Port: ports[6],
s3GrpcPort: ports[7],
icebergPort: ports[8],
}
env.dockerAvailable = hasDocker()
env.accessKey = "AKIAIOSFODNN7EXAMPLE"
env.secretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
return env
}
func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
t.Helper()
// Create IAM config file
iamConfigPath, err := testutil.WriteIAMConfig(env.dataDir, env.accessKey, env.secretKey)
if err != nil {
t.Fatalf("Failed to create IAM config: %v", err)
}
securityToml := filepath.Join(env.dataDir, "security.toml")
if err := os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644); err != nil {
t.Fatalf("Failed to create security.toml: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
env.weedCancel = cancel
cmd := exec.CommandContext(ctx, env.weedBinary, "mini",
"-master.port", fmt.Sprintf("%d", env.masterPort),
"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
"-volume.port", fmt.Sprintf("%d", env.volumePort),
"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
"-filer.port", fmt.Sprintf("%d", env.filerPort),
"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
"-s3.port", fmt.Sprintf("%d", env.s3Port),
"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
"-s3.port.iceberg", fmt.Sprintf("%d", env.icebergPort),
"-s3.config", iamConfigPath,
"-ip", env.bindIP,
"-ip.bind", "0.0.0.0",
"-dir", env.dataDir,
)
cmd.Dir = env.dataDir
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
// Set AWS credentials in environment (for compatibility)
cmd.Env = append(os.Environ(),
"AWS_ACCESS_KEY_ID="+env.accessKey,
"AWS_SECRET_ACCESS_KEY="+env.secretKey,
"ICEBERG_WAREHOUSE=s3://iceberg-tables",
"S3TABLES_DEFAULT_BUCKET=iceberg-tables",
)
if err := cmd.Start(); err != nil {
t.Fatalf("Failed to start SeaweedFS: %v", err)
}
env.weedProcess = cmd
// Try to check if Iceberg API is ready
// First try checking the /v1/config endpoint (requires auth, so will return 401 if server is up)
icebergURL := fmt.Sprintf("http://%s:%d/v1/config", env.bindIP, env.icebergPort)
if !env.waitForService(icebergURL, 30*time.Second) {
// Try to get more info about why it failed
client := &http.Client{Timeout: 2 * time.Second}
resp, err := client.Get(icebergURL)
if err != nil {
t.Logf("WARNING: Could not connect to Iceberg service at %s: %v", icebergURL, err)
} else {
t.Logf("WARNING: Iceberg service returned status %d at %s", resp.StatusCode, icebergURL)
resp.Body.Close()
}
t.Fatalf("Iceberg REST API did not become ready")
}
}
func (env *TestEnvironment) Cleanup(t *testing.T) {
t.Helper()
if env.trinoContainer != "" {
_ = exec.Command("docker", "rm", "-f", env.trinoContainer).Run()
}
if env.weedCancel != nil {
env.weedCancel()
}
if env.weedProcess != nil {
time.Sleep(2 * time.Second)
_ = env.weedProcess.Wait()
}
if env.dataDir != "" {
_ = os.RemoveAll(env.dataDir)
}
}
func (env *TestEnvironment) waitForService(url string, timeout time.Duration) bool {
client := &http.Client{Timeout: 2 * time.Second}
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
resp, err := client.Get(url)
if err != nil {
// Service not responding yet
time.Sleep(500 * time.Millisecond)
continue
}
statusCode := resp.StatusCode
resp.Body.Close()
// Accept 2xx status codes (successful responses)
if statusCode >= 200 && statusCode < 300 {
return true
}
// Also accept 401/403 (auth errors mean service is up, just needs credentials)
if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden {
return true
}
// For other status codes, keep trying
time.Sleep(500 * time.Millisecond)
}
return false
}
func testIcebergRestAPI(t *testing.T, env *TestEnvironment) {
t.Helper()
fmt.Printf(">>> Testing Iceberg REST API directly...\n")
// First, verify the service is listening
addr := net.JoinHostPort(env.bindIP, fmt.Sprintf("%d", env.icebergPort))
conn, err := net.Dial("tcp", addr)
if err != nil {
t.Fatalf("Cannot connect to Iceberg service at %s: %v", addr, err)
}
conn.Close()
t.Logf("Successfully connected to Iceberg service at %s", addr)
// Test /v1/config endpoint
url := fmt.Sprintf("http://%s:%d/v1/config", env.bindIP, env.icebergPort)
t.Logf("Testing Iceberg REST API at %s", url)
resp, err := http.Get(url)
if err != nil {
t.Fatalf("Failed to connect to Iceberg REST API at %s: %v", url, err)
}
defer resp.Body.Close()
t.Logf("Iceberg REST API response status: %d", resp.StatusCode)
body, _ := io.ReadAll(resp.Body)
t.Logf("Iceberg REST API response body: %s", string(body))
if resp.StatusCode != http.StatusOK {
t.Fatalf("Expected 200 OK from /v1/config, got %d", resp.StatusCode)
}
}
func (env *TestEnvironment) writeTrinoConfig(t *testing.T, warehouseBucket string, opts ...func(*trinoConfigOptions)) string {
t.Helper()
o := trinoConfigOptions{}
for _, fn := range opts {
fn(&o)
}
dirName := "trino"
if o.nestedNamespace {
dirName = "trino-nested"
}
configDir := filepath.Join(env.dataDir, dirName)
if err := os.MkdirAll(configDir, 0755); err != nil {
t.Fatalf("Failed to create Trino config dir: %v", err)
}
nestedLine := ""
if o.nestedNamespace {
nestedLine = "\niceberg.rest-catalog.nested-namespace-enabled=true"
}
uniqueTableLocation := "true"
if o.deterministicTableLocation {
// Skip UUID suffixing in TrinoRestCatalog.createLocationForTable so each
// table lands at the deterministic <namespace-location>/<table> path.
uniqueTableLocation = "false"
}
config := fmt.Sprintf(`connector.name=iceberg
iceberg.catalog.type=rest
iceberg.rest-catalog.uri=http://host.docker.internal:%d
iceberg.rest-catalog.warehouse=s3://%s%s
iceberg.file-format=PARQUET
iceberg.unique-table-location=%s
# S3 storage config
fs.native-s3.enabled=true
s3.endpoint=http://host.docker.internal:%d
s3.path-style-access=true
s3.signer-type=AwsS3V4Signer
s3.aws-access-key=%s
s3.aws-secret-key=%s
s3.region=us-west-2
# REST catalog authentication
iceberg.rest-catalog.security=SIGV4
`, env.icebergPort, warehouseBucket, nestedLine, uniqueTableLocation, env.s3Port, env.accessKey, env.secretKey)
if err := os.WriteFile(filepath.Join(configDir, "iceberg.properties"), []byte(config), 0644); err != nil {
t.Fatalf("Failed to write Trino config: %v", err)
}
return configDir
}
type trinoConfigOptions struct {
nestedNamespace bool
deterministicTableLocation bool
}
func withNestedNamespace() func(*trinoConfigOptions) {
return func(o *trinoConfigOptions) { o.nestedNamespace = true }
}
// withDeterministicTableLocation flips Trino's iceberg.unique-table-location
// flag to false so each table's <location> is the deterministic
// <namespace-location>/<tableName> path instead of a UUID-suffixed one.
func withDeterministicTableLocation() func(*trinoConfigOptions) {
return func(o *trinoConfigOptions) { o.deterministicTableLocation = true }
}
func (env *TestEnvironment) startTrinoContainer(t *testing.T, configDir string) {
t.Helper()
containerName := "seaweed-trino-" + randomString(8)
env.trinoContainer = containerName
cmd := exec.Command("docker", "run", "-d",
"--name", containerName,
"--add-host", "host.docker.internal:host-gateway",
"-v", fmt.Sprintf("%s:/etc/trino/catalog", configDir),
"-v", fmt.Sprintf("%s:/test", env.dataDir),
"-e", "AWS_ACCESS_KEY_ID="+env.accessKey,
"-e", "AWS_SECRET_ACCESS_KEY="+env.secretKey,
"-e", "AWS_REGION=us-west-2",
"trinodb/trino:479",
)
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("Failed to start Trino container: %v\n%s", err, string(output))
}
}
func waitForTrino(t *testing.T, containerName string, timeout time.Duration) {
t.Helper()
deadline := time.Now().Add(timeout)
var lastOutput []byte
retryCount := 0
for time.Now().Before(deadline) {
// Try system catalog query as a readiness check
cmd := exec.Command("docker", "exec", containerName,
"trino", "--catalog", "system", "--schema", "runtime",
"--execute", "SELECT 1",
)
if output, err := cmd.CombinedOutput(); err == nil {
return
} else {
lastOutput = output
outputStr := string(output)
if strings.Contains(outputStr, "No such container") ||
strings.Contains(outputStr, "is not running") {
break
}
retryCount++
}
time.Sleep(1 * time.Second)
}
// If we can't connect to system catalog, try to at least connect to Trino server
cmd := exec.Command("docker", "exec", containerName, "trino", "--version")
if err := cmd.Run(); err == nil {
// Trino process is running, even if catalog isn't ready yet
// Give it a bit more time
time.Sleep(5 * time.Second)
return
}
t.Fatalf("Timed out waiting for Trino to be ready\nLast output:\n%s", string(lastOutput))
}
func runTrinoSQL(t *testing.T, containerName, sql string) string {
t.Helper()
cmd := exec.Command("docker", "exec", containerName,
"trino", "--catalog", "iceberg",
"--output-format", "CSV",
"--execute", sql,
)
output, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("Trino command failed: %v\nSQL: %s\nOutput:\n%s", err, sql, string(output))
}
return sanitizeTrinoOutput(string(output))
}
func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
t.Helper()
// Use weed shell to create the table bucket
// Create with "000000000000" account ID (matches AccountAdmin.Id from auth_credentials.go)
// This ensures bucket owner matches authenticated identity's Account.Id
cmd := exec.Command(env.weedBinary, "shell",
fmt.Sprintf("-master=%s:%d.%d", env.bindIP, env.masterPort, env.masterGrpcPort),
)
cmd.Stdin = strings.NewReader(fmt.Sprintf("s3tables.bucket -create -name %s -account 000000000000\nexit\n", bucketName))
fmt.Printf(">>> EXECUTING: %v\n", cmd.Args)
output, err := cmd.CombinedOutput()
if err != nil {
fmt.Printf(">>> ERROR Output: %s\n", string(output))
t.Fatalf("Failed to create table bucket %s via weed shell: %v\nOutput: %s", bucketName, err, string(output))
}
fmt.Printf(">>> SUCCESS: Created table bucket %s\n", bucketName)
t.Logf("Created table bucket: %s", bucketName)
}
func sanitizeTrinoOutput(output string) string {
lines := strings.Split(strings.TrimSpace(output), "\n")
filtered := make([]string, 0, len(lines))
for _, line := range lines {
if strings.Contains(line, "org.jline.utils.Log") {
continue
}
if strings.Contains(line, "Unable to create a system terminal") {
continue
}
if strings.HasPrefix(line, "WARNING:") {
continue
}
if strings.TrimSpace(line) == "" {
continue
}
filtered = append(filtered, line)
}
if len(filtered) == 0 {
return ""
}
return strings.Join(filtered, "\n") + "\n"
}
func createObjectBucket(t *testing.T, env *TestEnvironment, bucketName string) {
t.Helper()
// Create an AWS S3 client with the test credentials pointing to our local server
cfg := aws.Config{
Region: "us-east-1",
Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")),
BaseEndpoint: aws.String(fmt.Sprintf("http://%s:%d", env.bindIP, env.s3Port)),
}
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = true
})
// Create the bucket using standard S3 API
_, err := client.CreateBucket(context.Background(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
})
if err != nil {
t.Fatalf("Failed to create object bucket %s: %v", bucketName, err)
}
}
func hasDocker() bool {
cmd := exec.Command("docker", "version")
return cmd.Run() == nil
}
func randomString(length int) string {
const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
b := make([]byte, length)
if _, err := rand.Read(b); err != nil {
panic("failed to generate random string: " + err.Error())
}
for i := range b {
b[i] = charset[int(b[i])%len(charset)]
}
return string(b)
}

View File

@@ -0,0 +1,208 @@
package catalog_trino
import (
"fmt"
"strings"
"testing"
"time"
)
// setupTrinoTest is a helper function that sets up the common test environment for all Trino CRUD tests
func setupTrinoTest(t *testing.T, configOpts ...func(*trinoConfigOptions)) *TestEnvironment {
t.Helper()
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
env := NewTestEnvironment(t)
if !env.dockerAvailable {
t.Skip("Docker not available, skipping Trino integration test")
}
t.Logf(">>> Starting SeaweedFS...")
env.StartSeaweedFS(t)
tableBucket := "iceberg-tables"
catalogBucket := tableBucket
createTableBucket(t, env, tableBucket)
configDir := env.writeTrinoConfig(t, catalogBucket, configOpts...)
env.startTrinoContainer(t, configDir)
waitForTrino(t, env.trinoContainer, 60*time.Second)
return env
}
// TestNamespaceCRUD tests namespace (schema) CRUD operations via Trino SQL
// Namespaces are the key container for tables in Iceberg, and this test
// verifies the full CRUD lifecycle: Create, Read (List), Update, and Delete
func TestNamespaceCRUD(t *testing.T) {
env := setupTrinoTest(t)
defer env.Cleanup(t)
// CREATE: Create a namespace
namespace1 := "crud_test_ns1_" + randomString(6)
t.Logf(">>> CREATE: Creating namespace %s", namespace1)
runTrinoSQL(t, env.trinoContainer, "CREATE SCHEMA IF NOT EXISTS iceberg."+namespace1)
t.Logf(">>> Namespace %s created", namespace1)
namespace2 := "crud_test_ns2_" + randomString(6)
t.Logf(">>> CREATE: Creating second namespace %s", namespace2)
runTrinoSQL(t, env.trinoContainer, "CREATE SCHEMA IF NOT EXISTS iceberg."+namespace2)
t.Logf(">>> Namespace %s created", namespace2)
// READ: List all namespaces
t.Logf(">>> READ: Listing all namespaces")
output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
if !strings.Contains(output, namespace1) {
t.Fatalf("Expected namespace %s in listing output:\n%s", namespace1, output)
}
if !strings.Contains(output, namespace2) {
t.Fatalf("Expected namespace %s in listing output:\n%s", namespace2, output)
}
t.Logf(">>> Both namespaces found in listing")
// UPDATE: Namespaces typically don't have "update" semantics in SQL,
// but we can verify properties via metadata queries
t.Logf(">>> UPDATE: Simulating namespace properties (via SQL properties check)")
// Iceberg REST API supports namespace properties, but Trino SQL doesn't expose them directly
// This test verifies the namespace still exists and is accessible
output = runTrinoSQL(t, env.trinoContainer, "SHOW TABLES FROM iceberg."+namespace1)
// DELETE: Drop namespaces
t.Logf(">>> DELETE: Dropping namespace %s", namespace1)
runTrinoSQL(t, env.trinoContainer, "DROP SCHEMA iceberg."+namespace1)
t.Logf(">>> Namespace %s dropped", namespace1)
t.Logf(">>> DELETE: Dropping namespace %s", namespace2)
runTrinoSQL(t, env.trinoContainer, "DROP SCHEMA iceberg."+namespace2)
t.Logf(">>> Namespace %s dropped", namespace2)
// Verify deletion: Namespaces should no longer exist
t.Logf(">>> VERIFY: Checking that namespaces are deleted")
output = runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
if strings.Contains(output, namespace1) {
t.Errorf("Namespace %s still appears in listing after deletion", namespace1)
} else {
t.Logf(">>> Namespace %s correctly deleted", namespace1)
}
if strings.Contains(output, namespace2) {
t.Errorf("Namespace %s still appears in listing after deletion", namespace2)
} else {
t.Logf(">>> Namespace %s correctly deleted", namespace2)
}
t.Logf(">>> TestNamespaceCRUD PASSED")
}
// TestNamespaceListingPagination tests that namespace listing works correctly
// This verifies the LIST operation with multiple namespaces
func TestNamespaceListingPagination(t *testing.T) {
env := setupTrinoTest(t)
defer env.Cleanup(t)
// Create multiple namespaces
numNamespaces := 5
namespaces := make([]string, numNamespaces)
t.Logf(">>> Creating %d namespaces for listing test", numNamespaces)
for i := 0; i < numNamespaces; i++ {
namespaces[i] = "list_test_ns" + fmt.Sprintf("%d", i+1) + "_" + randomString(4)
runTrinoSQL(t, env.trinoContainer, "CREATE SCHEMA IF NOT EXISTS iceberg."+namespaces[i])
t.Logf(">>> Created namespace %d: %s", i+1, namespaces[i])
}
// List all namespaces
t.Logf(">>> Listing all namespaces")
output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
// Verify all namespaces are in the listing
t.Logf(">>> Verifying all namespaces are in listing")
for i, ns := range namespaces {
if !strings.Contains(output, ns) {
t.Fatalf("Expected namespace %d (%s) in listing output", i+1, ns)
}
t.Logf(">>> Namespace %d (%s) found in listing", i+1, ns)
}
// Clean up
t.Logf(">>> Cleaning up namespaces")
for _, ns := range namespaces {
runTrinoSQL(t, env.trinoContainer, "DROP SCHEMA iceberg."+ns)
}
t.Logf(">>> Cleanup complete")
t.Logf(">>> TestNamespaceListingPagination PASSED")
}
// TestNamespaceErrorHandling tests error handling for namespace operations
// Tests both idempotent operations and actual error cases
func TestNamespaceErrorHandling(t *testing.T) {
env := setupTrinoTest(t)
defer env.Cleanup(t)
// Test 1: Create and drop same namespace
ns := "error_test_ns_" + randomString(6)
t.Logf(">>> Test 1: Creating and dropping namespace %s", ns)
runTrinoSQL(t, env.trinoContainer, "CREATE SCHEMA IF NOT EXISTS iceberg."+ns)
runTrinoSQL(t, env.trinoContainer, "DROP SCHEMA iceberg."+ns)
t.Logf(">>> Namespace created and dropped successfully")
// Test 2: Try to drop non-existent namespace with IF EXISTS (should succeed gracefully)
nonExistent := "nonexistent_" + randomString(6)
t.Logf(">>> Test 2: Attempting to drop non-existent namespace %s with IF EXISTS", nonExistent)
runTrinoSQL(t, env.trinoContainer, "DROP SCHEMA IF EXISTS iceberg."+nonExistent)
t.Logf(">>> Drop non-existent namespace handled gracefully (IF EXISTS clause)")
// Test 3: Creating duplicate namespace (with IF NOT EXISTS - should succeed gracefully)
ns2 := "dup_test_ns_" + randomString(6)
t.Logf(">>> Test 3: Creating namespace %s twice (using IF NOT EXISTS)", ns2)
runTrinoSQL(t, env.trinoContainer, "CREATE SCHEMA IF NOT EXISTS iceberg."+ns2)
runTrinoSQL(t, env.trinoContainer, "CREATE SCHEMA IF NOT EXISTS iceberg."+ns2)
t.Logf(">>> Duplicate creation handled gracefully")
// Test 4: Verify schema properties persist after creation
t.Logf(">>> Test 4: Verifying namespace still exists after duplicate creation attempt")
output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
if !strings.Contains(output, ns2) {
t.Errorf("Expected namespace %s to exist in listing after duplicate creation attempt", ns2)
} else {
t.Logf(">>> Namespace %s correctly persists", ns2)
}
// Cleanup
runTrinoSQL(t, env.trinoContainer, "DROP SCHEMA iceberg."+ns2)
t.Logf(">>> TestNamespaceErrorHandling PASSED")
}
// TestSchemaIntegrationWithCatalog tests that schemas are properly integrated with the catalog
// This verifies that schema operations through Trino are visible through Iceberg REST API concepts
func TestSchemaIntegrationWithCatalog(t *testing.T) {
env := setupTrinoTest(t)
defer env.Cleanup(t)
// Create a schema through Trino
schemaName := "catalog_integration_" + randomString(6)
t.Logf(">>> Creating schema through Trino SQL: %s", schemaName)
runTrinoSQL(t, env.trinoContainer, "CREATE SCHEMA IF NOT EXISTS iceberg."+schemaName)
// Verify it's accessible through catalog (list schemas)
t.Logf(">>> Verifying schema is visible through catalog (SHOW SCHEMAS)")
output := runTrinoSQL(t, env.trinoContainer, "SHOW SCHEMAS FROM iceberg")
if !strings.Contains(output, schemaName) {
t.Fatalf("Created schema %s not visible in catalog listing", schemaName)
}
t.Logf(">>> Schema successfully verified in catalog")
// Verify empty schema
t.Logf(">>> Verifying schema is empty (no tables)")
output = runTrinoSQL(t, env.trinoContainer, "SHOW TABLES FROM iceberg."+schemaName)
// Clean up
t.Logf(">>> Cleaning up schema")
runTrinoSQL(t, env.trinoContainer, "DROP SCHEMA iceberg."+schemaName)
t.Logf(">>> TestSchemaIntegrationWithCatalog PASSED")
}

View File

@@ -0,0 +1,121 @@
package catalog_trino
import (
"context"
"fmt"
"strings"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
// TestTrinoDeterministicLocationCTAS exercises the Trino REST catalog with
// `iceberg.unique-table-location=false`. With #9246 advertising a default
// namespace `location` property, Trino's defaultTableLocation is non-null,
// so even with the unique-table-location flag flipped off the CREATE flow
// takes the deferred-transaction branch and the listFiles emptiness check
// passes.
//
// The benefit users opt into by flipping the flag is a single dir per
// table on disk (catalog entry and Iceberg <location> share a path, no
// sibling `<table>-<uuid>/` directory). This test exercises a fresh CTAS
// in that mode and asserts the on-disk layout has no UUID-suffixed sibling.
func TestTrinoDeterministicLocationCTAS(t *testing.T) {
env := setupTrinoTest(t, withDeterministicTableLocation())
defer env.Cleanup(t)
schemaName := "detloc_" + randomString(6)
srcName := "src_" + randomString(6)
ctasName := "ctas_" + randomString(6)
srcQualified := fmt.Sprintf("iceberg.%s.%s", schemaName, srcName)
ctasQualified := fmt.Sprintf("iceberg.%s.%s", schemaName, ctasName)
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS iceberg.%s", schemaName))
defer runTrinoSQLAllowNamespaceNotEmpty(t, env.trinoContainer, fmt.Sprintf("DROP SCHEMA IF EXISTS iceberg.%s", schemaName))
defer runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS %s", srcQualified))
defer runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS %s", ctasQualified))
t.Logf(">>> CREATE source %s (no explicit location, unique-table-location=false)", srcQualified)
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
"CREATE TABLE %s (id INTEGER, name VARCHAR)", srcQualified))
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
"INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", srcQualified))
ctasSQL := fmt.Sprintf("CREATE TABLE %s AS SELECT * FROM %s", ctasQualified, srcQualified)
t.Logf(">>> CTAS (no location override): %s", ctasSQL)
runTrinoSQL(t, env.trinoContainer, ctasSQL)
countOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM %s", ctasQualified))
if got := mustParseCSVInt64(t, countOutput); got != 3 {
t.Fatalf("CTAS target: expected 3 rows, got %d", got)
}
out := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT name FROM %s ORDER BY id", ctasQualified))
for _, want := range []string{"a", "b", "c"} {
if !strings.Contains(out, want) {
t.Fatalf("CTAS target missing row %q; got:\n%s", want, out)
}
}
// Layout assertion: with unique-table-location=false, both tables must
// land at the deterministic <bucket>/<schema>/<tableName> path without
// any UUID-suffixed sibling directory under the namespace.
keys := listBucketKeysUnder(t, env, "iceberg-tables", schemaName+"/")
expectKeyAtLocation(t, keys, schemaName, srcName)
expectKeyAtLocation(t, keys, schemaName, ctasName)
rejectUUIDSuffixedSibling(t, keys, schemaName, srcName)
rejectUUIDSuffixedSibling(t, keys, schemaName, ctasName)
}
// listBucketKeysUnder lists every object key under the given bucket prefix
// using the test S3 endpoint.
func listBucketKeysUnder(t *testing.T, env *TestEnvironment, bucket, prefix string) []string {
t.Helper()
cfg := aws.Config{
Region: "us-east-1",
Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")),
BaseEndpoint: aws.String(fmt.Sprintf("http://%s:%d", env.bindIP, env.s3Port)),
}
client := s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true })
out, err := client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
Prefix: aws.String(prefix),
})
if err != nil {
t.Fatalf("ListObjectsV2 prefix=%s: %v", prefix, err)
}
keys := make([]string, 0, len(out.Contents))
for _, obj := range out.Contents {
keys = append(keys, aws.ToString(obj.Key))
}
return keys
}
// expectKeyAtLocation asserts at least one object key sits under
// <schema>/<table>/ (any depth), proving the table has a deterministic path.
func expectKeyAtLocation(t *testing.T, keys []string, schema, table string) {
t.Helper()
want := schema + "/" + table + "/"
for _, k := range keys {
if strings.HasPrefix(k, want) {
return
}
}
t.Fatalf("expected at least one object under %q; got keys: %v", want, keys)
}
// rejectUUIDSuffixedSibling asserts no object key sits under
// <schema>/<table>-<hex>/, since a UUID-suffixed sibling would mean Trino
// did add a UUID despite unique-table-location=false.
func rejectUUIDSuffixedSibling(t *testing.T, keys []string, schema, table string) {
t.Helper()
prefix := schema + "/" + table + "-"
for _, k := range keys {
if strings.HasPrefix(k, prefix) {
t.Fatalf("found UUID-suffixed sibling key %q for table %q (unique-table-location=false should produce no -<uuid> suffix)", k, table)
}
}
}

View File

@@ -0,0 +1,76 @@
package catalog_trino
import (
"fmt"
"strings"
"testing"
)
// TestTrinoIssue9074CTASFreshTable is a regression test for the follow-up
// report on https://github.com/seaweedfs/seaweedfs/issues/9074.
//
// The original fix purged stale data on DROP so a re-CREATE at the same
// path could succeed. The reporter then observed that even a fresh table
// name fails — they ran a CTAS without an explicit `location =` clause and
// hit the same "Cannot create a table on a non-empty location" error.
//
// Root cause: when (a) the user does not pass `location = ...`, and (b) the
// namespace has no Iceberg `location` property, Trino's REST catalog goes
// through TrinoRestCatalog.newCreateTableTransaction's eager branch
// (`tableBuilder.create().newTransaction()`) which immediately invokes the
// REST POST. Our handleCreateTable persists `<location>/metadata/v1.metadata.json`
// as part of that call. Trino's IcebergMetadata.beginCreateTable then runs
// `fileSystem.listFiles(location).hasNext()` and trips on the metadata file
// we just wrote.
//
// Fix: handleGetNamespace / handleCreateNamespace now advertise a default
// `location` property of `s3://<bucket>/<flattened-namespace>` when the
// namespace doesn't already carry one. Trino's `defaultTableLocation` then
// returns `<namespace-location>/<table>-<UUID>` (UUID added by the standard
// `iceberg.unique-table-location=true` connector setting), and the REST
// catalog takes the deferred `createTransaction` branch that does NOT call
// REST POST until commit-time. The empty-location check sees a genuinely
// empty UUID-suffixed path and passes; on commit, our metadata write lands
// alongside the data files Trino has already produced.
//
// This test exercises the exact pattern from the report: a fresh schema, a
// fresh table name, no `location =` clause, and a CTAS on top.
func TestTrinoIssue9074CTASFreshTable(t *testing.T) {
env := setupTrinoTest(t)
defer env.Cleanup(t)
schemaName := "issue9074_" + randomString(6)
srcName := "src_" + randomString(6)
ctasName := "ctas_" + randomString(6)
srcQualified := fmt.Sprintf("iceberg.%s.%s", schemaName, srcName)
ctasQualified := fmt.Sprintf("iceberg.%s.%s", schemaName, ctasName)
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS iceberg.%s", schemaName))
defer runTrinoSQLAllowNamespaceNotEmpty(t, env.trinoContainer, fmt.Sprintf("DROP SCHEMA IF EXISTS iceberg.%s", schemaName))
defer runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS %s", srcQualified))
defer runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS %s", ctasQualified))
t.Logf(">>> CREATE source %s (no explicit location)", srcQualified)
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
"CREATE TABLE %s (id INTEGER, name VARCHAR)", srcQualified))
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
"INSERT INTO %s VALUES (1, 'a'), (2, 'b'), (3, 'c')", srcQualified))
ctasSQL := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s AS (SELECT * FROM %s)", ctasQualified, srcQualified)
t.Logf(">>> CTAS (no location override): %s", ctasSQL)
runTrinoSQL(t, env.trinoContainer, ctasSQL)
countOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM %s", ctasQualified))
got := mustParseCSVInt64(t, countOutput)
if got != 3 {
t.Fatalf("CTAS target: expected 3 rows, got %d (resulting table should not be empty)", got)
}
out := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT name FROM %s ORDER BY id", ctasQualified))
for _, want := range []string{"a", "b", "c"} {
if !strings.Contains(out, want) {
t.Fatalf("CTAS target missing row %q; got:\n%s", want, out)
}
}
}

View File

@@ -0,0 +1,120 @@
package catalog_trino
import (
"fmt"
"strings"
"testing"
)
// TestTrinoCreateDropRecreateTable is a regression test for
// https://github.com/seaweedfs/seaweedfs/issues/9074
//
// Trino CTAS was failing with "Cannot create a table on a non-empty
// location" because Trino's pre-write check saw leftover data files from
// an earlier table at the same S3 path — DROP TABLE removed the catalog
// entry but left the underlying files behind. The Iceberg REST server now
// treats the S3 Tables catalog as the authority on table existence: when a
// DROP succeeds, it purges the storage under the table's location so that
// a subsequent CREATE at the same path finds it empty.
//
// This test pins an explicit location (so the recreated table lands at the
// exact same path, matching what the bug reporter saw), then walks through:
// 1. CREATE TABLE with explicit location
// 2. INSERT rows (produces data files under that location)
// 3. DROP TABLE (catalog entry removed, data files should also be purged)
// 4. CREATE TABLE again at the same explicit location — previously failed
// with "Cannot create a table on a non-empty location"
// 5. CREATE TABLE AS SELECT on top of the recreated table, which is the
// exact operation reported in the issue.
func TestTrinoCreateDropRecreateTable(t *testing.T) {
env := setupTrinoTest(t)
defer env.Cleanup(t)
tableBucket := "iceberg-tables"
schemaName := "recreate_" + randomString(6)
tableName := "events_" + randomString(6)
ctasName := "events_ctas_" + randomString(6)
tableLocation := fmt.Sprintf("s3://%s/%s/%s", tableBucket, schemaName, tableName)
ctasLocation := fmt.Sprintf("s3://%s/%s/%s", tableBucket, schemaName, ctasName)
qualified := fmt.Sprintf("iceberg.%s.%s", schemaName, tableName)
ctasQualified := fmt.Sprintf("iceberg.%s.%s", schemaName, ctasName)
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS iceberg.%s", schemaName))
defer runTrinoSQLAllowNamespaceNotEmpty(t, env.trinoContainer, fmt.Sprintf("DROP SCHEMA IF EXISTS iceberg.%s", schemaName))
// Registered after the DROP SCHEMA defer so LIFO ordering drops the
// tables before the schema. Without this, a failure between the
// CREATE TABLE below and the explicit DROP later in the test would
// leak the table and block the deferred schema drop.
defer runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS %s", qualified))
defer runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE IF EXISTS %s", ctasQualified))
createSQL := fmt.Sprintf(`CREATE TABLE %s (
id INTEGER,
label VARCHAR
) WITH (
format = 'PARQUET',
location = '%s'
)`, qualified, tableLocation)
t.Logf(">>> CREATE #1: %s (location=%s)", qualified, tableLocation)
runTrinoSQL(t, env.trinoContainer, createSQL)
t.Logf(">>> INSERT into %s", qualified)
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
"INSERT INTO %s VALUES (1, 'alpha'), (2, 'beta')", qualified))
countOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM %s", qualified))
if got := mustParseCSVInt64(t, countOutput); got != 2 {
t.Fatalf("after first insert: expected 2 rows, got %d", got)
}
t.Logf(">>> DROP %s", qualified)
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE %s", qualified))
// The recreate must succeed at the exact same location; DROP is expected
// to have purged the data files so Trino's pre-write check sees an empty
// path.
t.Logf(">>> CREATE #2 at the same location: %s", qualified)
runTrinoSQL(t, env.trinoContainer, createSQL)
t.Logf(">>> INSERT into recreated %s", qualified)
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf(
"INSERT INTO %s VALUES (10, 'gamma')", qualified))
countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM %s", qualified))
if got := mustParseCSVInt64(t, countOutput); got != 1 {
t.Fatalf("after recreate+insert: expected 1 row (not %d) — recreated table should not see dropped data", got)
}
labelOutput := runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT label FROM %s WHERE id = 10", qualified))
if !strings.Contains(labelOutput, "gamma") {
t.Fatalf("recreated table missing expected row; got:\n%s", labelOutput)
}
// CTAS on top of the recreated table is the exact operation from #9074.
ctasSQL := fmt.Sprintf(`CREATE TABLE %s
WITH (
format = 'PARQUET',
location = '%s'
)
AS SELECT * FROM %s`, ctasQualified, ctasLocation, qualified)
t.Logf(">>> CTAS %s FROM %s", ctasQualified, qualified)
runTrinoSQL(t, env.trinoContainer, ctasSQL)
countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM %s", ctasQualified))
if got := mustParseCSVInt64(t, countOutput); got != 1 {
t.Fatalf("CTAS target: expected 1 row, got %d", got)
}
// Drop the CTAS target and recreate it at the same path to make sure
// the drop-cleanup path is exercised for CTAS too.
t.Logf(">>> DROP then CTAS-recreate %s at the same location", ctasQualified)
runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("DROP TABLE %s", ctasQualified))
runTrinoSQL(t, env.trinoContainer, ctasSQL)
countOutput = runTrinoSQL(t, env.trinoContainer, fmt.Sprintf("SELECT count(*) FROM %s", ctasQualified))
if got := mustParseCSVInt64(t, countOutput); got != 1 {
t.Fatalf("CTAS after drop-recreate: expected 1 row, got %d", got)
}
}