snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/s3/spark/issue_8234_repro_test.go

@@ -0,0 +1,76 @@
+package spark
+
+import (
+	"strings"
+	"testing"
+)
+
+func setupSparkIssue8234Env(t *testing.T) *TestEnvironment {
+	t.Helper()
+
+	env := NewTestEnvironment()
+	if !env.dockerAvailable {
+		t.Skip("Docker not available, skipping Spark S3 integration test")
+	}
+
+	env.StartSeaweedFS(t)
+	t.Cleanup(func() { env.Cleanup(t) })
+
+	createObjectBucket(t, env, "test")
+	env.startSparkContainer(t)
+	return env
+}
+
+func TestSparkS3AppendIssue8234Regression(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping Spark integration test in short mode")
+	}
+
+	env := setupSparkIssue8234Env(t)
+
+	script := `
+import sys
+import pyspark.sql.types as T
+
+target = "s3a://test/test_table/"
+sc = spark.sparkContext
+df = sc.parallelize([1, 2, 3]).toDF(T.IntegerType())
+
+df.write.format("parquet").mode("append").save(target)
+print("FIRST_APPEND_OK")
+
+try:
+    df.write.format("parquet").mode("append").save(target)
+    print("SECOND_APPEND_OK")
+    spark.stop()
+    sys.exit(0)
+except Exception as err:
+    message = str(err)
+    print("SECOND_APPEND_FAILED")
+    print(message)
+    if "Copy Source must mention the source bucket and key" in message:
+        print("ISSUE_8234_REGRESSION")
+        spark.stop()
+        sys.exit(4)
+    spark.stop()
+    sys.exit(2)
+`
+
+	code, output := runSparkPyScript(t, env.sparkContainer, script, env.s3Port)
+	if code != 0 {
+		if strings.Contains(output, "ISSUE_8234_REGRESSION") {
+			t.Fatalf("issue #8234 regression detected; output:\n%s", output)
+		}
+		t.Fatalf("Spark script exited with code %d; output:\n%s", code, output)
+	}
+
+	if !strings.Contains(output, "FIRST_APPEND_OK") {
+		t.Fatalf("expected first append success marker in output, got:\n%s", output)
+	}
+	if !strings.Contains(output, "SECOND_APPEND_OK") {
+		t.Fatalf("expected second append success marker in output, got:\n%s", output)
+	}
+	if strings.Contains(output, "Copy Source must mention the source bucket and key") {
+		t.Fatalf("unexpected issue #8234 error in output:\n%s", output)
+	}
+}

test/s3/spark/issue_8285_repro_test.go

@@ -0,0 +1,187 @@
+package spark
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+func TestSparkS3TemporaryDirectoryCleanupIssue8285Regression(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping Spark integration test in short mode")
+	}
+
+	env := setupSparkIssue8234Env(t)
+
+	script := `
+import pyspark.sql.functions as F
+
+target = "s3a://test/issue-8285/output"
+
+spark.conf.set("spark.hadoop.fs.s3a.committer.name", "directory")
+spark.conf.set("spark.hadoop.fs.s3a.committer.magic.enabled", "false")
+spark.conf.set("spark.hadoop.fs.s3a.committer.staging.abort.pending.uploads", "true")
+spark.conf.set("spark.hadoop.fs.s3a.committer.staging.conflict-mode", "append")
+spark.conf.set("spark.hadoop.fs.s3a.committer.staging.tmp.path", "/tmp")
+spark.conf.set("spark.hadoop.fs.s3a.directory.marker.retention", "keep")
+
+df = spark.range(0, 200).repartition(12).withColumn("value", F.col("id") * 2)
+df.write.format("parquet").mode("overwrite").save(target)
+count = spark.read.parquet(target).count()
+print("WRITE_COUNT=" + str(count))
+`
+
+	code, output := runSparkPyScript(t, env.sparkContainer, script, env.s3Port)
+	if code != 0 {
+		t.Fatalf("Spark script exited with code %d; output:\n%s", code, output)
+	}
+	if !strings.Contains(output, "WRITE_COUNT=200") {
+		t.Fatalf("expected write/read success marker in output, got:\n%s", output)
+	}
+
+	keys := listObjectKeysByPrefix(t, env, "test", "issue-8285/")
+	var temporaryKeys []string
+	for _, key := range keys {
+		// Skip directory markers (keys ending in "/") — these are 0-byte
+		// metadata objects, not data artifacts. They are verified separately
+		// via HeadObject with a timeout below.
+		if strings.HasSuffix(key, "/") {
+			continue
+		}
+		if hasTemporaryPathSegment(key) {
+			temporaryKeys = append(temporaryKeys, key)
+		}
+	}
+
+	if len(temporaryKeys) > 0 {
+		t.Fatalf("issue #8285 regression detected: found lingering _temporary artifacts: %v\nall keys: %v", temporaryKeys, keys)
+	}
+
+	temporaryCandidates := []string{
+		"issue-8285/output/_temporary/",
+		"issue-8285/output/_temporary/0/",
+		"issue-8285/output/_temporary/0/_temporary/",
+	}
+	// Empty folder cleanup has a 2m default delay + 30s processor interval
+	lingering := waitForObjectsToDisappear(t, env, "test", temporaryCandidates, 3*time.Minute)
+	if len(lingering) > 0 {
+		t.Fatalf("issue #8285 regression detected: lingering temporary directories: %v", lingering)
+	}
+}
+
+func listObjectKeysByPrefix(t *testing.T, env *TestEnvironment, bucketName, prefix string) []string {
+	t.Helper()
+	client := newS3Client(env)
+
+	pager := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String(prefix),
+	})
+
+	var keys []string
+	for pager.HasMorePages() {
+		page, err := pager.NextPage(context.Background())
+		if err != nil {
+			t.Fatalf("failed listing objects for prefix %q: %v", prefix, err)
+		}
+		for _, object := range page.Contents {
+			keys = append(keys, aws.ToString(object.Key))
+		}
+	}
+
+	return keys
+}
+
+func headObjectInfo(t *testing.T, env *TestEnvironment, bucketName, key string) (bool, string, error) {
+	t.Helper()
+
+	client := newS3Client(env)
+	output, err := client.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(key),
+	})
+	if err == nil {
+		return true, aws.ToString(output.ContentType), nil
+	}
+
+	var notFound *s3types.NotFound
+	if strings.Contains(err.Error(), "NotFound") || strings.Contains(err.Error(), "NoSuchKey") || errors.As(err, &notFound) {
+		return false, "", nil
+	}
+	return false, "", err
+}
+
+func waitForObjectsToDisappear(t *testing.T, env *TestEnvironment, bucketName string, keys []string, timeout time.Duration) []string {
+	t.Helper()
+
+	deadline := time.Now().Add(timeout)
+	pending := make(map[string]struct{}, len(keys))
+	details := make(map[string]string, len(keys))
+	for _, key := range keys {
+		pending[key] = struct{}{}
+	}
+
+	for len(pending) > 0 && time.Now().Before(deadline) {
+		for key := range pending {
+			exists, contentType, err := headObjectInfo(t, env, bucketName, key)
+			if err != nil {
+				details[key] = fmt.Sprintf("%s (head_error=%v)", key, err)
+				continue
+			}
+			if !exists {
+				delete(pending, key)
+				delete(details, key)
+				continue
+			}
+			details[key] = fmt.Sprintf("%s (exists=true, contentType=%q)", key, contentType)
+		}
+		if len(pending) > 0 {
+			time.Sleep(2 * time.Second)
+		}
+	}
+
+	if len(pending) == 0 {
+		return nil
+	}
+
+	var lingering []string
+	for _, key := range keys {
+		if _, ok := pending[key]; !ok {
+			continue
+		}
+		if detail, hasDetail := details[key]; hasDetail {
+			lingering = append(lingering, detail)
+		} else {
+			lingering = append(lingering, key)
+		}
+	}
+	return lingering
+}
+
+func newS3Client(env *TestEnvironment) *s3.Client {
+	cfg := aws.Config{
+		Region:       "us-east-1",
+		Credentials:  aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")),
+		BaseEndpoint: aws.String(fmt.Sprintf("http://localhost:%d", env.s3Port)),
+	}
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+	})
+}
+
+func hasTemporaryPathSegment(key string) bool {
+	for _, segment := range strings.Split(strings.TrimSuffix(key, "/"), "/") {
+		if segment == "_temporary" {
+			return true
+		}
+	}
+	return false
+}

test/s3/spark/setup_test.go

@@ -0,0 +1,308 @@
+package spark
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+	"github.com/testcontainers/testcontainers-go"
+)
+
+var (
+	miniProcessMu   sync.Mutex
+	lastMiniProcess *exec.Cmd
+)
+
+func stopPreviousMini() {
+	miniProcessMu.Lock()
+	defer miniProcessMu.Unlock()
+
+	if lastMiniProcess != nil && lastMiniProcess.Process != nil {
+		_ = lastMiniProcess.Process.Kill()
+		_ = lastMiniProcess.Wait()
+	}
+	lastMiniProcess = nil
+}
+
+func registerMiniProcess(cmd *exec.Cmd) {
+	miniProcessMu.Lock()
+	lastMiniProcess = cmd
+	miniProcessMu.Unlock()
+}
+
+func clearMiniProcess(cmd *exec.Cmd) {
+	miniProcessMu.Lock()
+	if lastMiniProcess == cmd {
+		lastMiniProcess = nil
+	}
+	miniProcessMu.Unlock()
+}
+
+type TestEnvironment struct {
+	dockerAvailable  bool
+	weedBinary       string
+	seaweedfsDataDir string
+	weedLogPath      string
+	weedLogFile      *os.File
+	masterPort       int
+	filerPort        int
+	s3Port           int
+	accessKey        string
+	secretKey        string
+	sparkContainer   testcontainers.Container
+	masterProcess    *exec.Cmd
+}
+
+func NewTestEnvironment() *TestEnvironment {
+	env := &TestEnvironment{
+		accessKey: "test",
+		secretKey: "test",
+	}
+
+	cmd := exec.Command("docker", "version")
+	env.dockerAvailable = cmd.Run() == nil
+
+	if weedPath, err := exec.LookPath("weed"); err == nil {
+		env.weedBinary = weedPath
+	}
+
+	return env
+}
+
+func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
+	t.Helper()
+
+	if env.weedBinary == "" {
+		t.Skip("weed binary not found in PATH, skipping Spark S3 integration test")
+	}
+
+	stopPreviousMini()
+
+	var err error
+	env.seaweedfsDataDir, err = os.MkdirTemp("", "seaweed-s3-spark-test-")
+	if err != nil {
+		t.Fatalf("failed to create temp directory: %v", err)
+	}
+
+	ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3"})
+	env.masterPort = ports[0]
+	env.filerPort = ports[1]
+	env.s3Port = ports[2]
+
+	bindIP := testutil.FindBindIP()
+	iamConfigPath, err := testutil.WriteIAMConfig(env.seaweedfsDataDir, env.accessKey, env.secretKey)
+	if err != nil {
+		t.Fatalf("failed to create IAM config: %v", err)
+	}
+
+	env.masterProcess = exec.Command(
+		env.weedBinary, "mini",
+		"-ip", bindIP,
+		"-ip.bind", "0.0.0.0",
+		"-master.port", fmt.Sprintf("%d", env.masterPort),
+		"-filer.port", fmt.Sprintf("%d", env.filerPort),
+		"-s3.port", fmt.Sprintf("%d", env.s3Port),
+		"-s3.config", iamConfigPath,
+		"-dir", env.seaweedfsDataDir,
+	)
+	weedLogPath := filepath.Join(env.seaweedfsDataDir, "weed-mini.log")
+	weedLogFile, err := os.Create(weedLogPath)
+	if err != nil {
+		t.Fatalf("failed to create weed log file: %v", err)
+	}
+	env.weedLogPath = weedLogPath
+	env.weedLogFile = weedLogFile
+	env.masterProcess.Stdout = weedLogFile
+	env.masterProcess.Stderr = weedLogFile
+	env.masterProcess.Env = append(os.Environ(),
+		"AWS_ACCESS_KEY_ID="+env.accessKey,
+		"AWS_SECRET_ACCESS_KEY="+env.secretKey,
+	)
+
+	if err := env.masterProcess.Start(); err != nil {
+		t.Fatalf("failed to start weed mini: %v", err)
+	}
+	registerMiniProcess(env.masterProcess)
+
+	if !testutil.WaitForPort(env.masterPort, testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - master port %d not listening", env.masterPort)
+	}
+	if !testutil.WaitForPort(env.filerPort, testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - filer port %d not listening", env.filerPort)
+	}
+	if !testutil.WaitForService(fmt.Sprintf("http://127.0.0.1:%d/status", env.s3Port), testutil.SeaweedMiniStartupTimeout) {
+		t.Fatalf("weed mini failed to start - s3 endpoint http://127.0.0.1:%d/status not responding", env.s3Port)
+	}
+}
+
+func (env *TestEnvironment) startSparkContainer(t *testing.T) {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+
+	req := testcontainers.ContainerRequest{
+		Image:        "apache/spark:3.5.1",
+		ExposedPorts: []string{"4040/tcp"},
+		Env: map[string]string{
+			"SPARK_LOCAL_IP": "localhost",
+		},
+		ExtraHosts: []string{"host.docker.internal:host-gateway"},
+		Cmd:        []string{"/bin/sh", "-c", "sleep 7200"},
+	}
+
+	container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: req,
+		Started:          true,
+	})
+	if err != nil {
+		t.Fatalf("failed to start spark container: %v", err)
+	}
+	env.sparkContainer = container
+}
+
+func (env *TestEnvironment) Cleanup(t *testing.T) {
+	if env.masterProcess != nil && env.masterProcess.Process != nil {
+		_ = env.masterProcess.Process.Kill()
+		_ = env.masterProcess.Wait()
+	}
+	clearMiniProcess(env.masterProcess)
+	if env.weedLogFile != nil {
+		_ = env.weedLogFile.Close()
+	}
+
+	if t.Failed() && os.Getenv("CI") != "" && env.weedLogPath != "" {
+		logData, err := os.ReadFile(env.weedLogPath)
+		if err != nil {
+			t.Logf("failed to read weed mini log file %s: %v", env.weedLogPath, err)
+		} else {
+			// Print the tail to keep CI output manageable while preserving failure context.
+			const maxTailBytes = 64 * 1024
+			start := 0
+			if len(logData) > maxTailBytes {
+				start = len(logData) - maxTailBytes
+			}
+			t.Logf("weed mini logs (tail, %d bytes):\n%s", len(logData)-start, string(logData[start:]))
+		}
+	}
+
+	if env.sparkContainer != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = env.sparkContainer.Terminate(ctx)
+	}
+
+	if env.seaweedfsDataDir != "" {
+		_ = os.RemoveAll(env.seaweedfsDataDir)
+	}
+}
+
+func runSparkPyScript(t *testing.T, container testcontainers.Container, script string, s3Port int) (int, string) {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
+	defer cancel()
+
+	pythonScript := fmt.Sprintf(`
+import glob
+import os
+import sys
+
+spark_home = os.environ.get("SPARK_HOME", "/opt/spark")
+python_path = os.path.join(spark_home, "python")
+py4j_glob = glob.glob(os.path.join(python_path, "lib", "py4j-*.zip"))
+ivy_dir = "/tmp/ivy"
+os.makedirs(ivy_dir, exist_ok=True)
+os.environ["AWS_REGION"] = "us-east-1"
+os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
+os.environ["AWS_ACCESS_KEY_ID"] = "test"
+os.environ["AWS_SECRET_ACCESS_KEY"] = "test"
+
+if python_path not in sys.path:
+    sys.path.insert(0, python_path)
+if py4j_glob and py4j_glob[0] not in sys.path:
+    sys.path.insert(0, py4j_glob[0])
+
+from pyspark.sql import SparkSession
+
+spark = (SparkSession.builder
+    .master("local[2]")
+    .appName("SeaweedFS S3 Spark Issue 8234 Repro")
+    .config("spark.sql.catalogImplementation", "hive")
+    .config("spark.executor.extraJavaOptions", "-Djdk.tls.client.protocols=TLSv1")
+    .config("spark.sql.parquet.int96RebaseModeInRead", "CORRECTED")
+    .config("spark.sql.parquet.int96RebaseModeInWrite", "CORRECTED")
+    .config("spark.sql.parquet.datetimeRebaseModeInRead", "CORRECTED")
+    .config("spark.sql.parquet.datetimeRebaseModeInWrite", "CORRECTED")
+    .config("spark.sql.parquet.enableVectorizedReader", "false")
+    .config("spark.sql.parquet.mergeSchema", "false")
+    .config("spark.sql.parquet.writeLegacyFormat", "true")
+    .config("spark.sql.broadcastTimeout", "-1")
+    .config("spark.network.timeout", "600")
+    .config("spark.hadoop.hive.metastore.schema.verification", "false")
+    .config("spark.hadoop.hive.metastore.schema.verification.record.version", "false")
+    .config("spark.jars.ivy", ivy_dir)
+    .config("spark.jars.packages", "org.apache.hadoop:hadoop-aws:3.3.2,com.amazonaws:aws-java-sdk-bundle:1.12.262")
+    .config("spark.hadoop.fs.s3a.access.key", "test")
+    .config("spark.hadoop.fs.s3a.secret.key", "test")
+    .config("spark.hadoop.fs.s3a.endpoint", "host.docker.internal:%d")
+    .config("spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version", "1")
+    .config("spark.hadoop.fs.s3a.path.style.access", "true")
+    .config("spark.hadoop.fs.s3a.fast.upload", "true")
+    .config("spark.hadoop.fs.s3a.connection.ssl.enabled", "false")
+    .config("spark.hadoop.fs.s3a.multiobjectdelete.enable", "true")
+    .config("spark.hadoop.fs.s3a.directory.marker.retention", "keep")
+    .config("spark.hadoop.fs.s3a.change.detection.version.required", "false")
+    .config("spark.hadoop.fs.s3a.change.detection.mode", "warn")
+    .config("spark.local.dir", "/tmp/spark-temp")
+    .getOrCreate())
+
+%s
+`, s3Port, script)
+
+	code, out, err := container.Exec(ctx, []string{"python3", "-c", pythonScript})
+	var output string
+	if out != nil {
+		outputBytes, readErr := io.ReadAll(out)
+		if readErr != nil {
+			output = fmt.Sprintf("failed to read container output: %v", readErr)
+		} else {
+			output = string(outputBytes)
+		}
+	}
+	if err != nil {
+		output = output + fmt.Sprintf("\ncontainer exec error: %v\n", err)
+	}
+	return code, output
+}
+
+func createObjectBucket(t *testing.T, env *TestEnvironment, bucketName string) {
+	t.Helper()
+
+	cfg := aws.Config{
+		Region:       "us-east-1",
+		Credentials:  aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(env.accessKey, env.secretKey, "")),
+		BaseEndpoint: aws.String(fmt.Sprintf("http://localhost:%d", env.s3Port)),
+	}
+
+	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+	})
+
+	_, err := client.CreateBucket(context.Background(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		t.Fatalf("failed to create object bucket %s: %v", bucketName, err)
+	}
+}