snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/s3tables/catalog_spark/setup_test.go

@@ -0,0 +1,361 @@
+package catalog_spark
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+	"github.com/testcontainers/testcontainers-go"
+)
+
+var (
+	miniProcessMu   sync.Mutex
+	lastMiniProcess *exec.Cmd
+)
+
+func stopPreviousMini() {
+	miniProcessMu.Lock()
+	defer miniProcessMu.Unlock()
+
+	if lastMiniProcess != nil && lastMiniProcess.Process != nil {
+		_ = lastMiniProcess.Process.Kill()
+		_ = lastMiniProcess.Wait()
+	}
+	lastMiniProcess = nil
+}
+
+func registerMiniProcess(cmd *exec.Cmd) {
+	miniProcessMu.Lock()
+	lastMiniProcess = cmd
+	miniProcessMu.Unlock()
+}
+
+func clearMiniProcess(cmd *exec.Cmd) {
+	miniProcessMu.Lock()
+	if lastMiniProcess == cmd {
+		lastMiniProcess = nil
+	}
+	miniProcessMu.Unlock()
+}
+
+type TestEnvironment struct {
+	t                *testing.T
+	dockerAvailable  bool
+	seaweedfsDataDir string
+	sparkConfigDir   string
+	masterPort       int
+	filerPort        int
+	s3Port           int
+	icebergRestPort  int
+	accessKey        string
+	secretKey        string
+	sparkContainer   testcontainers.Container
+	masterProcess    *exec.Cmd
+}
+
+func NewTestEnvironment(t *testing.T) *TestEnvironment {
+	env := &TestEnvironment{
+		t:         t,
+		accessKey: "test",
+		secretKey: "test",
+	}
+
+	// Check if Docker is available
+	cmd := exec.Command("docker", "version")
+	env.dockerAvailable = cmd.Run() == nil
+
+	return env
+}
+
+func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
+	t.Helper()
+
+	stopPreviousMini()
+
+	var err error
+	env.seaweedfsDataDir, err = os.MkdirTemp("", "seaweed-spark-test-")
+	if err != nil {
+		t.Fatalf("failed to create temp directory: %v", err)
+	}
+
+	ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg"})
+	env.masterPort = ports[0]
+	env.filerPort = ports[1]
+	env.s3Port = ports[2]
+	env.icebergRestPort = ports[3]
+
+	bindIP := testutil.FindBindIP()
+
+	iamConfigPath, err := testutil.WriteIAMConfig(env.seaweedfsDataDir, env.accessKey, env.secretKey)
+	if err != nil {
+		t.Fatalf("failed to create IAM config: %v", err)
+	}
+
+	// Start SeaweedFS using weed mini (all-in-one including Iceberg REST)
+	env.masterProcess = exec.Command(
+		"weed", "mini",
+		"-ip", bindIP,
+		"-ip.bind", "0.0.0.0",
+		"-master.port", fmt.Sprintf("%d", env.masterPort),
+		"-filer.port", fmt.Sprintf("%d", env.filerPort),
+		"-s3.port", fmt.Sprintf("%d", env.s3Port),
+		"-s3.port.iceberg", fmt.Sprintf("%d", env.icebergRestPort),
+		"-s3.config", iamConfigPath,
+		"-dir", env.seaweedfsDataDir,
+	)
+	env.masterProcess.Env = append(os.Environ(),
+		"AWS_ACCESS_KEY_ID="+env.accessKey,
+		"AWS_SECRET_ACCESS_KEY="+env.secretKey,
+		"ICEBERG_WAREHOUSE=s3://iceberg-tables",
+		"S3TABLES_DEFAULT_BUCKET=iceberg-tables",
+	)
+	env.masterProcess.Stdout = os.Stdout
+	env.masterProcess.Stderr = os.Stderr
+	if err := env.masterProcess.Start(); err != nil {
+		t.Fatalf("failed to start weed mini: %v", err)
+	}
+	registerMiniProcess(env.masterProcess)
+
+	// Wait for all services to be ready
+	if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
+	}
+	if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
+	}
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
+	}
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/v1/config", env.icebergRestPort), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - iceberg rest endpoint http://127.0.0.1:%d/v1/config not responding", env.icebergRestPort)
+	}
+}
+
+func (env *TestEnvironment) writeSparkConfig(t *testing.T, catalogBucket string) string {
+	t.Helper()
+
+	configDir, err := os.MkdirTemp("", "spark-config-")
+	if err != nil {
+		t.Fatalf("failed to create config directory: %v", err)
+	}
+
+	// Store for cleanup
+	env.sparkConfigDir = configDir
+
+	s3Endpoint := fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
+	catalogEndpoint := fmt.Sprintf("http://host.docker.internal:%d", env.icebergRestPort)
+
+	sparkConfig := fmt.Sprintf(`
+[spark]
+master = "local"
+app.name = "SeaweedFS Iceberg Test"
+
+[storage]
+s3.endpoint = "%s"
+s3.access-key = "test"
+s3.secret-key = "test"
+s3.path-style-access = "true"
+s3.bucket = "%s"
+
+[iceberg]
+catalog.type = "rest"
+catalog.uri = "%s"
+catalog.s3.endpoint = "%s"
+catalog.s3.access-key = "test"
+catalog.s3.secret-key = "test"
+catalog.s3.path-style-access = "true"
+`, s3Endpoint, catalogBucket, catalogEndpoint, s3Endpoint)
+
+	configPath := filepath.Join(configDir, "spark-config.ini")
+	if err := os.WriteFile(configPath, []byte(sparkConfig), 0644); err != nil {
+		t.Fatalf("failed to write spark config: %v", err)
+	}
+
+	return configDir
+}
+
+func (env *TestEnvironment) startSparkContainer(t *testing.T, configDir string) {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+
+	req := testcontainers.ContainerRequest{
+		Image:        "apache/spark:3.5.1",
+		ExposedPorts: []string{"4040/tcp"},
+		Mounts: testcontainers.Mounts(
+			testcontainers.BindMount(configDir, "/config"),
+		),
+		Env: map[string]string{
+			"SPARK_LOCAL_IP": "localhost",
+		},
+		ExtraHosts: []string{"host.docker.internal:host-gateway"},
+		Cmd:        []string{"/bin/sh", "-c", "sleep 3600"},
+	}
+
+	container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: req,
+		Started:          true,
+	})
+	if err != nil {
+		t.Fatalf("failed to start spark container: %v", err)
+	}
+
+	env.sparkContainer = container
+}
+
+func (env *TestEnvironment) Cleanup(t *testing.T) {
+	t.Helper()
+
+	// Kill weed mini process
+	if env.masterProcess != nil && env.masterProcess.Process != nil {
+		env.masterProcess.Process.Kill()
+		env.masterProcess.Wait()
+	}
+	clearMiniProcess(env.masterProcess)
+
+	// Stop Spark container
+	if env.sparkContainer != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		env.sparkContainer.Terminate(ctx)
+	}
+
+	// Remove temporary directories after processes are stopped
+	if env.seaweedfsDataDir != "" {
+		os.RemoveAll(env.seaweedfsDataDir)
+	}
+	if env.sparkConfigDir != "" {
+		os.RemoveAll(env.sparkConfigDir)
+	}
+}
+
+func randomString(n int) string {
+	const letters = "abcdefghijklmnopqrstuvwxyz0123456789"
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+	return string(b)
+}
+
+func runSparkPySQL(t *testing.T, container testcontainers.Container, sql string, icebergPort int, s3Port int) string {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
+	defer cancel()
+
+	pythonScript := fmt.Sprintf(`
+import glob
+import os
+import sys
+
+spark_home = os.environ.get("SPARK_HOME", "/opt/spark")
+python_path = os.path.join(spark_home, "python")
+py4j_glob = glob.glob(os.path.join(python_path, "lib", "py4j-*.zip"))
+ivy_dir = "/tmp/ivy"
+os.makedirs(ivy_dir, exist_ok=True)
+os.environ["AWS_REGION"] = "us-west-2"
+os.environ["AWS_DEFAULT_REGION"] = "us-west-2"
+os.environ["AWS_ACCESS_KEY_ID"] = "test"
+os.environ["AWS_SECRET_ACCESS_KEY"] = "test"
+if python_path not in sys.path:
+    sys.path.insert(0, python_path)
+if py4j_glob and py4j_glob[0] not in sys.path:
+    sys.path.insert(0, py4j_glob[0])
+
+from pyspark.sql import SparkSession
+
+spark = (SparkSession.builder
+    .appName("SeaweedFS Iceberg Test")
+    .config("spark.jars.ivy", ivy_dir)
+    .config("spark.jars.packages", "org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.5.2,org.apache.iceberg:iceberg-aws-bundle:1.5.2")
+    .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions")
+    .config("spark.sql.catalog.iceberg", "org.apache.iceberg.spark.SparkCatalog")
+    .config("spark.sql.catalog.iceberg.type", "rest")
+    .config("spark.sql.catalog.iceberg.metrics-reporter-impl", "org.apache.iceberg.metrics.LoggingMetricsReporter")
+    .config("spark.sql.catalog.iceberg.uri", "http://host.docker.internal:%d")
+    .config("spark.sql.catalog.iceberg.rest.auth.type", "sigv4")
+    .config("spark.sql.catalog.iceberg.rest.auth.sigv4.delegate-auth-type", "none")
+    .config("spark.sql.catalog.iceberg.rest.sigv4-enabled", "true")
+    .config("spark.sql.catalog.iceberg.rest.signing-name", "s3")
+    .config("spark.sql.catalog.iceberg.rest.access-key-id", "test")
+    .config("spark.sql.catalog.iceberg.rest.secret-access-key", "test")
+    .config("spark.sql.catalog.iceberg.s3.endpoint", "http://host.docker.internal:%d")
+    .config("spark.sql.catalog.iceberg.s3.region", "us-west-2")
+    .config("spark.sql.catalog.iceberg.s3.access-key", "test")
+    .config("spark.sql.catalog.iceberg.s3.secret-key", "test")
+    .config("spark.sql.catalog.iceberg.s3.path-style-access", "true")
+    .config("spark.sql.catalog.iceberg.io-impl", "org.apache.iceberg.aws.s3.S3FileIO")
+    .getOrCreate())
+
+%s
+`, icebergPort, s3Port, sql)
+
+	code, out, err := container.Exec(ctx, []string{"python3", "-c", pythonScript})
+	var output string
+	if out != nil {
+		outputBytes, readErr := io.ReadAll(out)
+		if readErr != nil {
+			t.Logf("failed to read output: %v", readErr)
+		} else {
+			output = string(outputBytes)
+		}
+	}
+	if code != 0 {
+		t.Logf("Spark Python execution failed with code %d: %v, output: %s", code, err, output)
+		return output
+	}
+
+	return output
+}
+
+func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	masterGrpcPort := env.masterPort + 10000
+	cmd := exec.CommandContext(ctx, "weed", "shell",
+		fmt.Sprintf("-master=localhost:%d.%d", env.masterPort, masterGrpcPort),
+	)
+	cmd.Stdin = strings.NewReader(fmt.Sprintf("s3tables.bucket -create -name %s -account 000000000000\nexit\n", bucketName))
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("failed to create table bucket %s via weed shell: %v\nOutput: %s", bucketName, err, string(output))
+	}
+}
+
+func createObjectBucket(t *testing.T, env *TestEnvironment, bucketName string) {
+	t.Helper()
+
+	cfg := aws.Config{
+		Region:       "us-east-1",
+		Credentials:  aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")),
+		BaseEndpoint: aws.String(fmt.Sprintf("http://localhost:%d", env.s3Port)),
+	}
+
+	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+	})
+
+	_, err := client.CreateBucket(context.Background(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		t.Fatalf("failed to create object bucket %s: %v", bucketName, err)
+	}
+}

test/s3tables/catalog_spark/spark_deterministic_location_test.go

@@ -0,0 +1,135 @@
+package catalog_spark
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+// TestSparkDeterministicLocationCTAS exercises Spark's iceberg-spark
+// connector against the SeaweedFS REST catalog with no explicit `LOCATION`
+// clause and no UUID-suffixing. iceberg-spark does not implement Trino's
+// unique-table-location feature, so every CREATE TABLE here produces a
+// deterministic <namespace-location>/<tableName> path. The test asserts
+// the fresh-CTAS pattern works AND the resulting on-disk layout has no
+// UUID-suffixed sibling — i.e. one directory per table.
+//
+// The fix in #9246 (advertise a default `location` namespace property)
+// is what allows iceberg-spark to resolve the table location at all when
+// the user did not pass one. Without it, a Spark-side CREATE TABLE on a
+// namespace with no `location` property fails to compute a path.
+func TestSparkDeterministicLocationCTAS(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env, _, tableBucket := setupSparkTestEnv(t)
+
+	namespace := "detloc_" + randomString(6)
+	srcTable := "src_" + randomString(6)
+	ctasTable := "ctas_" + randomString(6)
+
+	sql := fmt.Sprintf(`
+spark.sql("CREATE NAMESPACE iceberg.%s")
+print("namespace created")
+
+spark.sql("""
+CREATE TABLE iceberg.%s.%s (
+    id INT,
+    name STRING
+) USING iceberg
+""")
+print("source table created")
+
+spark.sql("""
+INSERT INTO iceberg.%s.%s VALUES
+    (1, 'alpha'),
+    (2, 'beta'),
+    (3, 'gamma')
+""")
+print("data inserted")
+
+# CTAS without explicit LOCATION — the exact pattern that breaks on Trino
+# without #9246, and that iceberg-spark needs the namespace-location
+# property to resolve.
+spark.sql("""
+CREATE TABLE iceberg.%s.%s
+USING iceberg
+AS SELECT * FROM iceberg.%s.%s
+""")
+print("ctas created")
+
+count = spark.sql("SELECT COUNT(*) AS c FROM iceberg.%s.%s").collect()[0]['c']
+print(f"ctas row count: {count}")
+`,
+		namespace,
+		namespace, srcTable,
+		namespace, srcTable,
+		namespace, ctasTable, namespace, srcTable,
+		namespace, ctasTable,
+	)
+
+	output := runSparkPySQL(t, env.sparkContainer, sql, env.icebergRestPort, env.s3Port)
+
+	for _, want := range []string{"namespace created", "source table created", "data inserted", "ctas created", "ctas row count: 3"} {
+		if !strings.Contains(output, want) {
+			t.Fatalf("expected output to contain %q; full output:\n%s", want, output)
+		}
+	}
+
+	// Layout assertion: each table's data must land at the deterministic
+	// <namespace>/<tableName>/ path, with no UUID-suffixed sibling.
+	keys := listSparkBucketKeysUnder(t, env, tableBucket, namespace+"/")
+	expectKeyAtSparkLocation(t, keys, namespace, srcTable)
+	expectKeyAtSparkLocation(t, keys, namespace, ctasTable)
+	rejectUUIDSuffixedSparkSibling(t, keys, namespace, srcTable)
+	rejectUUIDSuffixedSparkSibling(t, keys, namespace, ctasTable)
+}
+
+func listSparkBucketKeysUnder(t *testing.T, env *TestEnvironment, bucket, prefix string) []string {
+	t.Helper()
+	cfg := aws.Config{
+		Region:       "us-east-1",
+		Credentials:  aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")),
+		BaseEndpoint: aws.String(fmt.Sprintf("http://localhost:%d", env.s3Port)),
+	}
+	client := s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true })
+	out, err := client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucket),
+		Prefix: aws.String(prefix),
+	})
+	if err != nil {
+		t.Fatalf("ListObjectsV2 prefix=%s: %v", prefix, err)
+	}
+	keys := make([]string, 0, len(out.Contents))
+	for _, obj := range out.Contents {
+		keys = append(keys, aws.ToString(obj.Key))
+	}
+	return keys
+}
+
+func expectKeyAtSparkLocation(t *testing.T, keys []string, namespace, table string) {
+	t.Helper()
+	want := namespace + "/" + table + "/"
+	for _, k := range keys {
+		if strings.HasPrefix(k, want) {
+			return
+		}
+	}
+	t.Fatalf("expected at least one object under %q; got keys: %v", want, keys)
+}
+
+func rejectUUIDSuffixedSparkSibling(t *testing.T, keys []string, namespace, table string) {
+	t.Helper()
+	prefix := namespace + "/" + table + "-"
+	for _, k := range keys {
+		if strings.HasPrefix(k, prefix) {
+			t.Fatalf("found UUID-suffixed sibling key %q for table %q (Spark should produce no -<uuid> suffix)", k, table)
+		}
+	}
+}

test/s3tables/catalog_spark/spark_operations_test.go

@@ -0,0 +1,356 @@
+package catalog_spark
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/testcontainers/testcontainers-go"
+)
+
+// waitForSparkReady polls Spark to verify it's ready by executing a simple query
+func waitForSparkReady(t *testing.T, container testcontainers.Container, icebergPort int, s3Port int, timeout time.Duration) {
+	t.Helper()
+
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		output := runSparkPySQL(t, container, `
+spark.sql("SELECT 1 as test")
+print("Spark ready")
+`, icebergPort, s3Port)
+		if strings.Contains(output, "Spark ready") {
+			return
+		}
+		time.Sleep(5 * time.Second)
+	}
+	t.Fatalf("Spark did not become ready within %v", timeout)
+}
+
+// setupSparkTestEnv initializes a test environment with SeaweedFS and Spark containers
+func setupSparkTestEnv(t *testing.T) (*TestEnvironment, string, string) {
+	t.Helper()
+
+	env := NewTestEnvironment(t)
+
+	if !env.dockerAvailable {
+		t.Skip("Docker not available, skipping Spark integration test")
+	}
+
+	t.Logf(">>> Starting SeaweedFS...")
+	env.StartSeaweedFS(t)
+	t.Cleanup(func() { env.Cleanup(t) })
+
+	tableBucket := "iceberg-tables"
+	catalogBucket := tableBucket
+	createTableBucket(t, env, tableBucket)
+
+	configDir := env.writeSparkConfig(t, catalogBucket)
+	env.startSparkContainer(t, configDir)
+
+	// Poll for Spark readiness instead of fixed sleep
+	waitForSparkReady(t, env.sparkContainer, env.icebergRestPort, env.s3Port, 10*time.Minute)
+
+	return env, catalogBucket, tableBucket
+}
+
+// TestSparkCatalogBasicOperations tests basic Spark Iceberg catalog operations
+func TestSparkCatalogBasicOperations(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env, _, _ := setupSparkTestEnv(t)
+
+	// Test 1: Create a namespace (database)
+	t.Logf(">>> Test 1: Creating namespace")
+	namespace := "spark_test_" + randomString(6)
+	sparkSQL := fmt.Sprintf(`
+spark.sql("CREATE NAMESPACE iceberg.%s")
+print("Namespace created")
+`, namespace)
+	output := runSparkPySQL(t, env.sparkContainer, sparkSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Namespace created") {
+		t.Fatalf("namespace creation failed, output: %s", output)
+	}
+
+	// Test 2: Create a table
+	t.Logf(">>> Test 2: Creating table")
+	tableName := "test_table_" + randomString(6)
+	createTableSQL := fmt.Sprintf(`
+spark.sql("""
+CREATE TABLE iceberg.%s.%s (
+    id INT,
+    name STRING,
+    age INT
+)
+USING iceberg
+""")
+print("Table created")
+`, namespace, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, createTableSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Table created") {
+		t.Fatalf("table creation failed, output: %s", output)
+	}
+
+	// Test 3: Insert data
+	t.Logf(">>> Test 3: Inserting data")
+	insertDataSQL := fmt.Sprintf(`
+spark.sql("""
+INSERT INTO iceberg.%s.%s VALUES
+    (1, 'Alice', 30),
+    (2, 'Bob', 25),
+    (3, 'Charlie', 35)
+""")
+print("Data inserted")
+`, namespace, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, insertDataSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Data inserted") {
+		t.Fatalf("data insertion failed, output: %s", output)
+	}
+
+	// Test 4: Query data
+	t.Logf(">>> Test 4: Querying data")
+	querySQL := fmt.Sprintf(`
+result = spark.sql("SELECT COUNT(*) as count FROM iceberg.%s.%s")
+result.show()
+count = result.collect()[0]['count']
+print(f"Row count: {count}")
+`, namespace, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, querySQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Row count: 3") {
+		t.Errorf("expected row count 3, got output: %s", output)
+	}
+
+	// Test 5: Update data
+	t.Logf(">>> Test 5: Updating data")
+	updateSQL := fmt.Sprintf(`
+spark.sql("""
+UPDATE iceberg.%s.%s SET age = 31 WHERE id = 1
+""")
+print("Data updated")
+`, namespace, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, updateSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Data updated") {
+		t.Errorf("data update failed, output: %s", output)
+	}
+
+	// Test 6: Delete data
+	t.Logf(">>> Test 6: Deleting data")
+	deleteSQL := fmt.Sprintf(`
+spark.sql("""
+DELETE FROM iceberg.%s.%s WHERE id = 3
+""")
+print("Data deleted")
+`, namespace, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, deleteSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Data deleted") {
+		t.Errorf("data delete failed, output: %s", output)
+	}
+
+	// Verify final count
+	t.Logf(">>> Verifying final data")
+	finalCountSQL := fmt.Sprintf(`
+result = spark.sql("SELECT COUNT(*) as count FROM iceberg.%s.%s")
+result.show()
+count = result.collect()[0]['count']
+print(f"Final row count: {count}")
+`, namespace, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, finalCountSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Final row count: 2") {
+		t.Errorf("expected final row count 2, got output: %s", output)
+	}
+
+	t.Logf(">>> All tests passed")
+}
+
+// TestSparkTimeTravel tests Spark Iceberg time travel capabilities
+func TestSparkTimeTravel(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env, _, _ := setupSparkTestEnv(t)
+
+	namespace := "time_travel_test_" + randomString(6)
+	tableName := "tt_table_" + randomString(6)
+
+	// Create namespace and table
+	setupSQL := fmt.Sprintf(`
+spark.sql("CREATE NAMESPACE iceberg.%s")
+spark.sql("""
+CREATE TABLE iceberg.%s.%s (
+    id INT,
+    value INT
+)
+USING iceberg
+""")
+print("Setup complete")
+`, namespace, namespace, tableName)
+	output := runSparkPySQL(t, env.sparkContainer, setupSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Setup complete") {
+		t.Fatalf("setup failed for namespace %s and table %s, output: %s", namespace, tableName, output)
+	}
+
+	// Insert initial data
+	t.Logf(">>> Inserting initial data")
+	insertSQL := fmt.Sprintf(`
+import time
+from datetime import timedelta
+spark.sql("""
+INSERT INTO iceberg.%s.%s VALUES (1, 10)
+""")
+ts = None
+for _ in range(10):
+    try:
+        ts = spark.sql("SELECT committed_at FROM iceberg.%s.%s.snapshots ORDER BY committed_at DESC LIMIT 1").collect()[0]["committed_at"]
+        if ts is not None:
+            break
+    except Exception as e:
+        print(f"Snapshot query failed: {e}")
+    time.sleep(1)
+if ts is None:
+    raise RuntimeError("Failed to read snapshot committed_at")
+ts_for_time_travel = ts + timedelta(seconds=1)
+print(f"Snapshot timestamp: {ts_for_time_travel.strftime('%%Y-%%m-%%d %%H:%%M:%%S')}")
+`, namespace, tableName, namespace, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, insertSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Snapshot timestamp:") {
+		t.Fatalf("failed to get snapshot timestamp: %s", output)
+	}
+
+	// Extract snapshot timestamp from output - look specifically for the "Snapshot timestamp:" line
+	var snapshotTS string
+	tsRe := regexp.MustCompile(`Snapshot timestamp:\s*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})`)
+	matches := tsRe.FindStringSubmatch(output)
+	if len(matches) > 1 {
+		snapshotTS = matches[1]
+	}
+
+	if snapshotTS == "" {
+		t.Fatalf("could not extract snapshot timestamp from output: %s", output)
+	}
+
+	// Wait to ensure the next insert gets a distinct snapshot timestamp
+	time.Sleep(2 * time.Second)
+
+	// Insert more data
+	t.Logf(">>> Inserting more data")
+	insertMoreSQL := fmt.Sprintf(`
+spark.sql("""
+INSERT INTO iceberg.%s.%s VALUES (2, 20)
+""")
+print("More data inserted")
+`, namespace, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, insertMoreSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "More data inserted") {
+		t.Fatalf("failed to insert more data, output: %s", output)
+	}
+
+	// Verify count increased to 2
+	t.Logf(">>> Verifying row count after second insert")
+	verifySQL := fmt.Sprintf(`
+result = spark.sql("SELECT COUNT(*) as count FROM iceberg.%s.%s")
+count = result.collect()[0]['count']
+print(f"Current row count: {count}")
+`, namespace, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, verifySQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Current row count: 2") {
+		t.Fatalf("expected current row count 2 after second insert, got output: %s", output)
+	}
+
+	// Time travel to first snapshot
+	t.Logf(">>> Time traveling to first snapshot")
+	timeTravelSQL := fmt.Sprintf(`
+result = spark.sql("""
+SELECT COUNT(*) as count FROM iceberg.%s.%s TIMESTAMP AS OF '%s'
+""")
+result.show()
+count = result.collect()[0]['count']
+print(f"Count at snapshot: {count}")
+`, namespace, tableName, snapshotTS)
+	output = runSparkPySQL(t, env.sparkContainer, timeTravelSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Count at snapshot: 1") {
+		t.Errorf("expected count 1 at first snapshot, got: %s", output)
+	}
+
+	t.Logf(">>> Time travel test passed")
+}
+
+// TestSparkMultiLevelNamespace tests that multi-level namespaces produce correct
+// S3 paths (dot-separated) so that Spark can read back the data it writes.
+// Regression test for https://github.com/seaweedfs/seaweedfs/issues/8959
+func TestSparkMultiLevelNamespace(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	env, _, _ := setupSparkTestEnv(t)
+
+	// Use a two-level namespace like "analytics.daily"
+	nsLevel1 := "analytics_" + randomString(4)
+	nsLevel2 := "daily_" + randomString(4)
+	multiNs := fmt.Sprintf("%s.%s", nsLevel1, nsLevel2)
+	tableName := "events_" + randomString(4)
+
+	// Create multi-level namespace
+	t.Logf(">>> Creating multi-level namespace: %s", multiNs)
+	createNsSQL := fmt.Sprintf(`
+spark.sql("CREATE NAMESPACE iceberg.%s")
+print("Namespace created")
+`, multiNs)
+	output := runSparkPySQL(t, env.sparkContainer, createNsSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Namespace created") {
+		t.Fatalf("multi-level namespace creation failed, output: %s", output)
+	}
+
+	// Create table under multi-level namespace
+	t.Logf(">>> Creating table under multi-level namespace")
+	createTableSQL := fmt.Sprintf(`
+spark.sql("""
+CREATE TABLE iceberg.%s.%s (
+    id INT,
+    event STRING,
+    ts TIMESTAMP
+)
+USING iceberg
+""")
+print("Table created")
+`, multiNs, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, createTableSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Table created") {
+		t.Fatalf("table creation under multi-level namespace failed, output: %s", output)
+	}
+
+	// Insert data
+	t.Logf(">>> Inserting data into multi-level namespace table")
+	insertSQL := fmt.Sprintf(`
+spark.sql("""
+INSERT INTO iceberg.%s.%s VALUES
+    (1, 'click', TIMESTAMP '2025-01-01 00:00:00'),
+    (2, 'view',  TIMESTAMP '2025-01-01 01:00:00'),
+    (3, 'click', TIMESTAMP '2025-01-02 00:00:00')
+""")
+print("Data inserted")
+`, multiNs, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, insertSQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Data inserted") {
+		t.Fatalf("data insertion failed, output: %s", output)
+	}
+
+	// Query data back — this is the key test: if the namespace path separator
+	// was wrong (\x1F instead of "."), Spark would not find the data files.
+	t.Logf(">>> Querying data from multi-level namespace table")
+	querySQL := fmt.Sprintf(`
+result = spark.sql("SELECT COUNT(*) as count FROM iceberg.%s.%s")
+count = result.collect()[0]['count']
+print(f"Row count: {count}")
+`, multiNs, tableName)
+	output = runSparkPySQL(t, env.sparkContainer, querySQL, env.icebergRestPort, env.s3Port)
+	if !strings.Contains(output, "Row count: 3") {
+		t.Errorf("expected row count 3 from multi-level namespace table, got output: %s", output)
+	}
+
+	t.Logf(">>> Multi-level namespace test passed")
+}