snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/s3/basic/basic_test.go

@@ -0,0 +1,227 @@
+package basic
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+var (
+	svc *s3.S3
+)
+
+func init() {
+	// Initialize a session in us-west-2 that the SDK will use to load
+	// credentials from the shared credentials file ~/.aws/credentials.
+	sess, err := session.NewSession(&aws.Config{
+		Region:     aws.String("us-west-2"),
+		Endpoint:   aws.String("localhost:8333"),
+		DisableSSL: aws.Bool(true),
+	})
+	if err != nil {
+		exitErrorf("create session, %v", err)
+	}
+
+	// Create S3 service client
+	svc = s3.New(sess)
+}
+
+func TestCreateBucket(t *testing.T) {
+
+	input := &s3.CreateBucketInput{
+		Bucket: aws.String("theBucket"),
+	}
+
+	result, err := svc.CreateBucket(input)
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			case s3.ErrCodeBucketAlreadyExists:
+				fmt.Println(s3.ErrCodeBucketAlreadyExists, aerr.Error())
+			case s3.ErrCodeBucketAlreadyOwnedByYou:
+				fmt.Println(s3.ErrCodeBucketAlreadyOwnedByYou, aerr.Error())
+			default:
+				fmt.Println(aerr.Error())
+			}
+		} else {
+			// Print the error, cast err to awserr.Error to get the Code and
+			// Message from an error.
+			fmt.Println(err.Error())
+		}
+		return
+	}
+
+	fmt.Println(result)
+
+}
+
+func TestPutObject(t *testing.T) {
+
+	input := &s3.PutObjectInput{
+		ACL:    aws.String("authenticated-read"),
+		Body:   aws.ReadSeekCloser(strings.NewReader("filetoupload")),
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("exampleobject"),
+	}
+
+	result, err := svc.PutObject(input)
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			default:
+				fmt.Println(aerr.Error())
+			}
+		} else {
+			// Print the error, cast err to awserr.Error to get the Code and
+			// Message from an error.
+			fmt.Println(err.Error())
+		}
+		return
+	}
+
+	fmt.Println(result)
+
+}
+
+func TestListBucket(t *testing.T) {
+
+	result, err := svc.ListBuckets(nil)
+	if err != nil {
+		exitErrorf("Unable to list buckets, %v", err)
+	}
+
+	fmt.Println("Buckets:")
+
+	for _, b := range result.Buckets {
+		fmt.Printf("* %s created on %s\n",
+			aws.StringValue(b.Name), aws.TimeValue(b.CreationDate))
+	}
+
+}
+
+func TestListObjectV2(t *testing.T) {
+
+	listObj, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
+		Bucket:    aws.String(Bucket),
+		Prefix:    aws.String("foo"),
+		Delimiter: aws.String("/"),
+	})
+	if err != nil {
+		exitErrorf("Unable to list objects, %v", err)
+	}
+	for _, content := range listObj.Contents {
+		fmt.Println(aws.StringValue(content.Key))
+	}
+	fmt.Printf("list: %s\n", listObj)
+
+}
+
+func exitErrorf(msg string, args ...interface{}) {
+	fmt.Fprintf(os.Stderr, msg+"\n", args...)
+	os.Exit(1)
+}
+
+const (
+	Bucket = "theBucket"
+	object = "foo/bar"
+	Data   = "<data>"
+)
+
+func TestObjectOp(t *testing.T) {
+	_, err := svc.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(Bucket),
+	})
+	if err != nil {
+		exitErrorf("Unable to create bucket, %v", err)
+	}
+
+	_, err = svc.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(Bucket),
+		Key:    aws.String(object),
+		Body:   strings.NewReader(Data),
+	})
+	if err != nil {
+		exitErrorf("Unable to put object, %v", err)
+	}
+
+	dest := fmt.Sprintf("%s_bak", object)
+	copyObj, err := svc.CopyObject(&s3.CopyObjectInput{
+		Bucket:     aws.String(Bucket),
+		CopySource: aws.String(fmt.Sprintf("%s/%s", Bucket, object)),
+		Key:        aws.String(dest),
+	})
+	if err != nil {
+		exitErrorf("Unable to copy object, %v", err)
+	}
+	t.Log("copy object result -> ", copyObj.CopyObjectResult)
+
+	getObj, err := svc.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(Bucket),
+		Key:    aws.String(dest),
+	})
+	if err != nil {
+		exitErrorf("Unable to get copy object, %v", err)
+	}
+
+	data, err := io.ReadAll(getObj.Body)
+	if err != nil {
+		exitErrorf("Unable to read object data, %v", err)
+	}
+	if string(data) != Data {
+		t.Error("object data -> ", string(data))
+	}
+
+	listObj, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
+		Bucket: aws.String(Bucket),
+		Prefix: aws.String("foo/"),
+	})
+	if err != nil {
+		exitErrorf("Unable to list objects, %v", err)
+	}
+	count := 0
+	for _, content := range listObj.Contents {
+		key := aws.StringValue(content.Key)
+		if key == dest {
+			count++
+		} else if key == object {
+			count++
+		}
+		if count == 2 {
+			break
+		}
+	}
+	if count != 2 {
+		exitErrorf("Unable to find two objects, %v", listObj.Contents)
+	}
+
+	_, err = svc.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(Bucket),
+		Key:    aws.String(object),
+	})
+	if err != nil {
+		exitErrorf("Unable to delete source object, %v", err)
+	}
+
+	_, err = svc.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(Bucket),
+		Key:    aws.String(dest),
+	})
+	if err != nil {
+		exitErrorf("Unable to delete object, %v", err)
+	}
+
+	_, err = svc.DeleteBucket(&s3.DeleteBucketInput{
+		Bucket: aws.String(Bucket),
+	})
+
+	if err != nil {
+		exitErrorf("Unable to delete bucket, %v", err)
+	}
+}

test/s3/basic/delimiter_test.go

@@ -0,0 +1,169 @@
+package basic
+
+import (
+	"fmt"
+	"math/rand"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestS3ListDelimiterWithDirectoryKeyObjects tests the specific scenario from
+// test_bucket_list_delimiter_not_skip_special where directory key objects
+// should be properly grouped into common prefixes when using delimiters
+func TestS3ListDelimiterWithDirectoryKeyObjects(t *testing.T) {
+	bucketName := fmt.Sprintf("test-delimiter-dir-key-%d", rand.Int31())
+
+	// Create bucket
+	_, err := svc.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+	defer cleanupBucket(t, bucketName)
+
+	// Create objects matching the failing test scenario:
+	// ['0/'] + ['0/1000', '0/1001', '0/1002'] + ['1999', '1999#', '1999+', '2000']
+	objects := []string{
+		"0/",     // Directory key object
+		"0/1000", // Objects under 0/ prefix
+		"0/1001",
+		"0/1002",
+		"1999", // Objects without delimiter
+		"1999#",
+		"1999+",
+		"2000",
+	}
+
+	// Create all objects
+	for _, key := range objects {
+		_, err := svc.PutObject(&s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(key),
+			Body:   strings.NewReader(fmt.Sprintf("content for %s", key)),
+		})
+		require.NoError(t, err, "Failed to create object %s", key)
+	}
+
+	// Test with delimiter='/'
+	resp, err := svc.ListObjects(&s3.ListObjectsInput{
+		Bucket:    aws.String(bucketName),
+		Delimiter: aws.String("/"),
+	})
+	require.NoError(t, err)
+
+	// Extract keys and prefixes
+	var keys []string
+	for _, content := range resp.Contents {
+		keys = append(keys, *content.Key)
+	}
+
+	var prefixes []string
+	for _, prefix := range resp.CommonPrefixes {
+		prefixes = append(prefixes, *prefix.Prefix)
+	}
+
+	// Expected results:
+	// Keys should be: ['1999', '1999#', '1999+', '2000'] (objects without delimiters)
+	// Prefixes should be: ['0/'] (grouping '0/' and all '0/xxxx' objects)
+
+	expectedKeys := []string{"1999", "1999#", "1999+", "2000"}
+	expectedPrefixes := []string{"0/"}
+
+	t.Logf("Actual keys: %v", keys)
+	t.Logf("Actual prefixes: %v", prefixes)
+
+	assert.ElementsMatch(t, expectedKeys, keys, "Keys should only include objects without delimiters")
+	assert.ElementsMatch(t, expectedPrefixes, prefixes, "CommonPrefixes should group directory key object with other objects sharing prefix")
+
+	// Additional validation
+	assert.Equal(t, "/", *resp.Delimiter, "Delimiter should be set correctly")
+	assert.Contains(t, prefixes, "0/", "Directory key object '0/' should be grouped into common prefix '0/'")
+	assert.NotContains(t, keys, "0/", "Directory key object '0/' should NOT appear as individual key when delimiter is used")
+
+	// Verify none of the '0/xxxx' objects appear as individual keys
+	for _, key := range keys {
+		assert.False(t, strings.HasPrefix(key, "0/"), "No object with '0/' prefix should appear as individual key, found: %s", key)
+	}
+}
+
+// TestS3ListWithoutDelimiter tests that directory key objects appear as individual keys when no delimiter is used
+func TestS3ListWithoutDelimiter(t *testing.T) {
+	bucketName := fmt.Sprintf("test-no-delimiter-%d", rand.Int31())
+
+	// Create bucket
+	_, err := svc.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+	defer cleanupBucket(t, bucketName)
+
+	// Create objects
+	objects := []string{"0/", "0/1000", "1999"}
+
+	for _, key := range objects {
+		_, err := svc.PutObject(&s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(key),
+			Body:   strings.NewReader(fmt.Sprintf("content for %s", key)),
+		})
+		require.NoError(t, err)
+	}
+
+	// Test without delimiter
+	resp, err := svc.ListObjects(&s3.ListObjectsInput{
+		Bucket: aws.String(bucketName),
+		// No delimiter specified
+	})
+	require.NoError(t, err)
+
+	// Extract keys
+	var keys []string
+	for _, content := range resp.Contents {
+		keys = append(keys, *content.Key)
+	}
+
+	// When no delimiter is used, all objects should be returned as individual keys
+	expectedKeys := []string{"0/", "0/1000", "1999"}
+	assert.ElementsMatch(t, expectedKeys, keys, "All objects should be individual keys when no delimiter is used")
+
+	// No common prefixes should be present
+	assert.Empty(t, resp.CommonPrefixes, "No common prefixes should be present when no delimiter is used")
+	assert.Contains(t, keys, "0/", "Directory key object '0/' should appear as individual key when no delimiter is used")
+}
+
+func cleanupBucket(t *testing.T, bucketName string) {
+	// Delete all objects
+	resp, err := svc.ListObjects(&s3.ListObjectsInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		t.Logf("Failed to list objects for cleanup: %v", err)
+		return
+	}
+
+	for _, obj := range resp.Contents {
+		_, err := svc.DeleteObject(&s3.DeleteObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    obj.Key,
+		})
+		if err != nil {
+			t.Logf("Failed to delete object %s: %v", *obj.Key, err)
+		}
+	}
+
+	// Give some time for eventual consistency
+	time.Sleep(100 * time.Millisecond)
+
+	// Delete bucket
+	_, err = svc.DeleteBucket(&s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		t.Logf("Failed to delete bucket %s: %v", bucketName, err)
+	}
+}

test/s3/basic/object_tagging_test.go

@@ -0,0 +1,228 @@
+package basic
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+func TestObjectTagging(t *testing.T) {
+
+	input := &s3.PutObjectInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("testDir/testObject"),
+	}
+
+	svc.PutObject(input)
+
+	printTags()
+
+	setTags()
+
+	printTags()
+
+	clearTags()
+
+	printTags()
+
+}
+
+func printTags() {
+	response, err := svc.GetObjectTagging(
+		&s3.GetObjectTaggingInput{
+			Bucket: aws.String("theBucket"),
+			Key:    aws.String("testDir/testObject"),
+		})
+
+	fmt.Println("printTags")
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+
+	fmt.Println(response.TagSet)
+}
+
+func setTags() {
+
+	response, err := svc.PutObjectTagging(&s3.PutObjectTaggingInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("testDir/testObject"),
+		Tagging: &s3.Tagging{
+			TagSet: []*s3.Tag{
+				{
+					Key:   aws.String("kye2"),
+					Value: aws.String("value2"),
+				},
+			},
+		},
+	})
+
+	fmt.Println("setTags")
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+
+	fmt.Println(response.String())
+}
+
+func clearTags() {
+
+	response, err := svc.DeleteObjectTagging(&s3.DeleteObjectTaggingInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("testDir/testObject"),
+	})
+
+	fmt.Println("clearTags")
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+
+	fmt.Println(response.String())
+}
+
+func TestObjectTaggingWithEncodedValues(t *testing.T) {
+	// Test for URL encoded tag values
+	input := &s3.PutObjectInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("testDir/testObjectWithEncodedTags"),
+	}
+
+	svc.PutObject(input)
+
+	// Set tags with encoded values (simulating what would happen with timestamps containing spaces and colons)
+	_, err := svc.PutObjectTagging(&s3.PutObjectTaggingInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("testDir/testObjectWithEncodedTags"),
+		Tagging: &s3.Tagging{
+			TagSet: []*s3.Tag{
+				{
+					Key:   aws.String("Timestamp"),
+					Value: aws.String("2025-07-16 14:40:39"), // This would be URL encoded as "2025-07-16%2014%3A40%3A39" in the header
+				},
+				{
+					Key:   aws.String("Path"),
+					Value: aws.String("/tmp/file.txt"), // This would be URL encoded as "/tmp%2Ffile.txt" in the header
+				},
+			},
+		},
+	})
+
+	if err != nil {
+		t.Fatalf("Failed to set tags with encoded values: %v", err)
+	}
+
+	// Get tags back and verify they are properly decoded
+	response, err := svc.GetObjectTagging(&s3.GetObjectTaggingInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("testDir/testObjectWithEncodedTags"),
+	})
+
+	if err != nil {
+		t.Fatalf("Failed to get tags: %v", err)
+	}
+
+	// Verify that the tags are properly decoded
+	tagMap := make(map[string]string)
+	for _, tag := range response.TagSet {
+		tagMap[*tag.Key] = *tag.Value
+	}
+
+	expectedTimestamp := "2025-07-16 14:40:39"
+	if tagMap["Timestamp"] != expectedTimestamp {
+		t.Errorf("Expected Timestamp tag to be '%s', got '%s'", expectedTimestamp, tagMap["Timestamp"])
+	}
+
+	expectedPath := "/tmp/file.txt"
+	if tagMap["Path"] != expectedPath {
+		t.Errorf("Expected Path tag to be '%s', got '%s'", expectedPath, tagMap["Path"])
+	}
+
+	fmt.Printf("✓ URL encoded tags test passed - Timestamp: %s, Path: %s\n", tagMap["Timestamp"], tagMap["Path"])
+
+	// Clean up
+	svc.DeleteObjectTagging(&s3.DeleteObjectTaggingInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("testDir/testObjectWithEncodedTags"),
+	})
+}
+
+// TestObjectUploadWithEncodedTags tests the specific issue reported in GitHub issue #7040
+// where tags sent via X-Amz-Tagging header during object upload are not URL decoded properly
+func TestObjectUploadWithEncodedTags(t *testing.T) {
+	// This test specifically addresses the issue where tags with special characters
+	// (like spaces, colons, slashes) sent during object upload are not URL decoded
+	// This tests the fix in filer_server_handlers_write_autochunk.go
+
+	objectKey := "testDir/testObjectUploadWithTags"
+
+	// Upload object with tags that contain special characters that would be URL encoded
+	// The AWS SDK will automatically URL encode these when sending the X-Amz-Tagging header
+	// Test edge cases that url.ParseQuery handles better than manual parsing:
+	// - Values containing "=" characters
+	// - Empty values
+	// - Complex special characters
+	_, err := svc.PutObject(&s3.PutObjectInput{
+		Bucket:  aws.String("theBucket"),
+		Key:     aws.String(objectKey),
+		Body:    aws.ReadSeekCloser(strings.NewReader("test content")),
+		Tagging: aws.String("Timestamp=2025-07-16 14:40:39&Path=/tmp/file.txt&Description=A test file with spaces&Equation=x=y+1&EmptyValue=&Complex=A%20tag%20with%20%26%20%3D%20chars"),
+	})
+
+	if err != nil {
+		t.Fatalf("Failed to upload object with tags: %v", err)
+	}
+
+	// Get the tags back to verify they were properly URL decoded during upload
+	response, err := svc.GetObjectTagging(&s3.GetObjectTaggingInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String(objectKey),
+	})
+
+	if err != nil {
+		t.Fatalf("Failed to get object tags: %v", err)
+	}
+
+	// Verify that the tags are properly decoded (not URL encoded)
+	tagMap := make(map[string]string)
+	for _, tag := range response.TagSet {
+		tagMap[*tag.Key] = *tag.Value
+	}
+
+	// Test cases for values that would be URL encoded in the X-Amz-Tagging header
+	testCases := []struct {
+		key           string
+		expectedValue string
+		description   string
+	}{
+		{"Timestamp", "2025-07-16 14:40:39", "timestamp with spaces and colons"},
+		{"Path", "/tmp/file.txt", "file path with slashes"},
+		{"Description", "A test file with spaces", "description with spaces"},
+	}
+
+	for _, tc := range testCases {
+		actualValue, exists := tagMap[tc.key]
+		if !exists {
+			t.Errorf("Expected tag key '%s' not found", tc.key)
+			continue
+		}
+
+		if actualValue != tc.expectedValue {
+			t.Errorf("Tag '%s' (%s): expected '%s', got '%s'",
+				tc.key, tc.description, tc.expectedValue, actualValue)
+		} else {
+			fmt.Printf("✓ Tag '%s' correctly decoded: '%s'\n", tc.key, actualValue)
+		}
+	}
+
+	// Clean up
+	_, err = svc.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String(objectKey),
+	})
+	if err != nil {
+		t.Logf("Warning: Failed to clean up test object: %v", err)
+	}
+}