snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/s3/SDK_COMPATIBILITY.md

@@ -0,0 +1,126 @@
+# S3 SDK Compatibility Testing
+
+This document describes the SDK versions tested against SeaweedFS S3 API and known compatibility considerations.
+
+## Validated SDK Versions
+
+### Go SDKs
+
+| SDK | Version | Test Location | Status |
+|-----|---------|---------------|--------|
+| AWS SDK v2 for Go | 1.20+ | `test/s3/etag/`, `test/s3/copying/` | ✅ Tested |
+| AWS SDK v1 for Go | 1.x | `test/s3/basic/` | ✅ Tested |
+
+### Java SDKs
+
+| SDK | Version | Test Location | Status |
+|-----|---------|---------------|--------|
+| AWS SDK v2 for Java | 2.20.127+ | `other/java/s3copier/` | ✅ Tested |
+| AWS SDK v1 for Java | 1.12.600+ | `other/java/s3copier/` | ✅ Tested |
+
+### Python SDKs
+
+| SDK | Version | Test Location | Status |
+|-----|---------|---------------|--------|
+| boto3 | 1.x | `test/s3/parquet/` | ✅ Tested |
+| PyArrow S3 | 14+ | `test/s3/parquet/` | ✅ Tested |
+
+## SDK-Specific Considerations
+
+### AWS SDK v2 for Java - ETag Validation
+
+**Issue**: [GitHub #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+
+AWS SDK v2 for Java includes strict ETag validation in `ChecksumsEnabledValidator.validatePutObjectChecksum`. It decodes the ETag as a hexadecimal MD5 hash using `Base16Codec.decode()`.
+
+**Impact**: If the ETag contains non-hexadecimal characters (like `-` in composite format), the SDK fails with:
+```text
+java.lang.IllegalArgumentException: Invalid base 16 character: '-'
+```
+
+**Resolution**: SeaweedFS now correctly returns:
+- **PutObject**: Pure MD5 hex ETag (32 characters) regardless of internal chunking
+- **CompleteMultipartUpload**: Composite ETag (`<md5>-<partcount>`)
+
+**Test Coverage**: `test/s3/etag/` and `other/java/s3copier/ETagValidationTest.java`
+
+### AWS SDK v1 vs v2 Differences
+
+| Feature | SDK v1 | SDK v2 |
+|---------|--------|--------|
+| ETag hex validation | No | Yes (strict) |
+| Checksum validation | Basic | Enhanced |
+| Async support | Limited | Full |
+| Default retry behavior | Lenient | Stricter |
+
+### Large File Handling
+
+SeaweedFS auto-chunks files larger than **8MB** for efficient storage. This is transparent to clients, but previously affected ETag format. The current implementation ensures:
+
+1. Regular `PutObject` (any size): Returns pure MD5 ETag
+2. Multipart upload: Returns composite ETag per AWS S3 specification
+
+## Test Categories by File Size
+
+| Category | Size | Chunks | ETag Format |
+|----------|------|--------|-------------|
+| Small | < 256KB | 1 (inline) | Pure MD5 |
+| Medium | 256KB - 8MB | 1 | Pure MD5 |
+| Large | 8MB - 24MB | 2-3 | Pure MD5 |
+| Extra Large | > 24MB | 4+ | Pure MD5 |
+| Multipart | N/A | Per part | Composite |
+
+## Running SDK Compatibility Tests
+
+### Go Tests
+
+```bash
+# Run all ETag tests
+cd test/s3/etag && make test
+
+# Run large file tests only
+cd test/s3/etag && make test-large
+```
+
+### Java Tests
+
+```bash
+# Run all Java SDK tests
+cd other/java/s3copier && mvn test
+
+# Run only ETag validation tests
+cd other/java/s3copier && mvn test -Dtest=ETagValidationTest
+```
+
+### Python Tests
+
+```bash
+# Run PyArrow S3 tests
+cd test/s3/parquet && make test
+```
+
+## Adding New SDK Tests
+
+When adding tests for new SDKs, ensure:
+
+1. **Large file tests (>8MB)**: Critical for verifying ETag format with auto-chunking
+2. **Multipart upload tests**: Verify composite ETag format
+3. **Checksum validation**: Test SDK-specific checksum validation if applicable
+4. **Document SDK version**: Add to this compatibility matrix
+
+## Known Issues and Workarounds
+
+### Issue: Older SDK Versions
+
+Some very old SDK versions (e.g., AWS SDK v1 for Java < 1.11.x) may have different behavior. Testing with the versions listed above is recommended.
+
+### Issue: Custom Checksum Algorithms
+
+AWS SDK v2 supports SHA-256 and CRC32 checksums in addition to MD5. SeaweedFS currently returns MD5-based ETags. For checksums other than MD5, use the `x-amz-checksum-*` headers.
+
+## References
+
+- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
+- [AWS SDK v2 Migration Guide](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/migration.html)
+- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+

test/s3/acl/s3_acl_versioning_test.go

@@ -0,0 +1,335 @@
+package acl
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func getS3Client(t *testing.T) *s3.Client {
+	endpoint := os.Getenv("S3_ENDPOINT")
+	if endpoint == "" {
+		endpoint = "http://localhost:8333"
+	}
+	accessKey := os.Getenv("S3_ACCESS_KEY")
+	if accessKey == "" {
+		accessKey = "some_access_key1"
+	}
+	secretKey := os.Getenv("S3_SECRET_KEY")
+	if secretKey == "" {
+		secretKey = "some_secret_key1"
+	}
+
+	cfg, err := config.LoadDefaultConfig(
+		context.TODO(),
+		config.WithRegion("us-east-1"),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+			accessKey,
+			secretKey,
+			"",
+		)),
+	)
+	require.NoError(t, err)
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+		o.BaseEndpoint = aws.String(endpoint)
+	})
+}
+
+func createVersionedTestBucket(t *testing.T, client *s3.Client) string {
+	bucketName := "test-acl-versioned-" + strings.ToLower(strings.ReplaceAll(time.Now().Format("2006-01-02-15-04-05.000"), ":", "-"))
+	_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+
+	_, err = client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err)
+	time.Sleep(100 * time.Millisecond)
+	return bucketName
+}
+
+func cleanupTestBucket(t *testing.T, client *s3.Client, bucketName string) {
+	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+	})
+	if err == nil {
+		for _, obj := range listResp.Contents {
+			client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    obj.Key,
+			})
+		}
+	}
+
+	listVersionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err == nil {
+		for _, version := range listVersionsResp.Versions {
+			client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+				Bucket:    aws.String(bucketName),
+				Key:       version.Key,
+				VersionId: version.VersionId,
+			})
+		}
+		for _, marker := range listVersionsResp.DeleteMarkers {
+			client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+				Bucket:    aws.String(bucketName),
+				Key:       marker.Key,
+				VersionId: marker.VersionId,
+			})
+		}
+	}
+
+	client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+}
+
+// TestGetObjectAclOnVersionedBucket tests retrieving ACL from versioned objects
+func TestGetObjectAclOnVersionedBucket(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createVersionedTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	objectKey := "versioned-object-acl"
+	putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   strings.NewReader("Hello, ACL World!"),
+	})
+	require.NoError(t, err)
+
+	aclResp, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	assert.NotNil(t, aclResp.Owner)
+
+	t.Logf("Successfully retrieved ACL for versioned object %s (versionId: %s)", objectKey, *putResp.VersionId)
+}
+
+// TestGetObjectAclOnSpecificVersionInVersionedBucket tests retrieving ACL for specific versions
+func TestGetObjectAclOnSpecificVersionInVersionedBucket(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createVersionedTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	objectKey := "multi-version-object-acl"
+
+	version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   strings.NewReader("Version 1"),
+	})
+	require.NoError(t, err)
+
+	time.Sleep(50 * time.Millisecond)
+
+	version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   strings.NewReader("Version 2"),
+	})
+	require.NoError(t, err)
+
+	versionId1 := *version1Resp.VersionId
+	versionId2 := *version2Resp.VersionId
+
+	aclResp1, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(versionId1),
+	})
+	require.NoError(t, err)
+	assert.NotNil(t, aclResp1.Owner)
+
+	aclResp2, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(versionId2),
+	})
+	require.NoError(t, err)
+	assert.NotNil(t, aclResp2.Owner)
+
+	t.Logf("Successfully retrieved ACL for both versions: v1=%s, v2=%s", versionId1, versionId2)
+}
+
+// TestPutObjectAclOnVersionedBucket tests setting ACL on versioned objects
+func TestPutObjectAclOnVersionedBucket(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createVersionedTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	objectKey := "versioned-object-put-acl"
+	putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   strings.NewReader("Hello, Put ACL!"),
+	})
+	require.NoError(t, err)
+
+	_, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		ACL:    types.ObjectCannedACLPublicRead,
+	})
+	require.NoError(t, err)
+
+	aclResp, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	assert.NotNil(t, aclResp.Owner)
+
+	t.Logf("Successfully set and verified ACL for versioned object %s (versionId: %s)", objectKey, *putResp.VersionId)
+}
+
+// TestPutObjectAclOnSpecificVersionInVersionedBucket tests setting ACL on specific versions
+func TestPutObjectAclOnSpecificVersionInVersionedBucket(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createVersionedTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	objectKey := "multi-version-object-put-acl"
+
+	version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   strings.NewReader("Version 1"),
+	})
+	require.NoError(t, err)
+
+	time.Sleep(50 * time.Millisecond)
+
+	version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   strings.NewReader("Version 2"),
+	})
+	require.NoError(t, err)
+
+	versionId1 := *version1Resp.VersionId
+	versionId2 := *version2Resp.VersionId
+
+	_, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(versionId1),
+		ACL:       types.ObjectCannedACLPublicRead,
+	})
+	require.NoError(t, err)
+
+	_, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(versionId2),
+		ACL:       types.ObjectCannedACLPrivate,
+	})
+	require.NoError(t, err)
+
+	aclResp1, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(versionId1),
+	})
+	require.NoError(t, err)
+	assert.NotNil(t, aclResp1.Owner)
+
+	aclResp2, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(versionId2),
+	})
+	require.NoError(t, err)
+	assert.NotNil(t, aclResp2.Owner)
+
+	t.Logf("Successfully set ACL on both versions: v1=%s, v2=%s", versionId1, versionId2)
+}
+
+// TestModifyAclOnDifferentVersionsIndependently tests that ACL changes on one version don't affect others
+func TestModifyAclOnDifferentVersionsIndependently(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createVersionedTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	objectKey := "versioned-independent-acl"
+
+	version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   strings.NewReader("Version 1"),
+	})
+	require.NoError(t, err)
+
+	time.Sleep(50 * time.Millisecond)
+
+	version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   strings.NewReader("Version 2"),
+	})
+	require.NoError(t, err)
+
+	versionId1 := *version1Resp.VersionId
+	versionId2 := *version2Resp.VersionId
+
+	_, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(versionId1),
+		ACL:       types.ObjectCannedACLPublicRead,
+	})
+	require.NoError(t, err)
+
+	_, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(versionId2),
+		ACL:       types.ObjectCannedACLPrivate,
+	})
+	require.NoError(t, err)
+
+	aclRespLatest, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	assert.NotNil(t, aclRespLatest.Owner)
+
+	aclResp1, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(versionId1),
+	})
+	require.NoError(t, err)
+	assert.NotNil(t, aclResp1.Owner)
+
+	aclResp2, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
+		Bucket:    aws.String(bucketName),
+		Key:       aws.String(objectKey),
+		VersionId: aws.String(versionId2),
+	})
+	require.NoError(t, err)
+	assert.NotNil(t, aclResp2.Owner)
+
+	t.Logf("Successfully verified independent ACL management across versions")
+}

test/s3/acl/s3_ownership_controls_test.go

@@ -0,0 +1,93 @@
+package acl
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func createOwnershipTestBucket(t *testing.T, client *s3.Client) string {
+	bucketName := "test-ownership-" + strings.ToLower(strings.ReplaceAll(time.Now().Format("2006-01-02-15-04-05.000"), ":", "-"))
+	_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+	return bucketName
+}
+
+func getObjectOwnership(t *testing.T, client *s3.Client, bucketName string) types.ObjectOwnership {
+	t.Helper()
+	resp, err := client.GetBucketOwnershipControls(context.TODO(), &s3.GetBucketOwnershipControlsInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+	require.NotNil(t, resp.OwnershipControls)
+	require.Len(t, resp.OwnershipControls.Rules, 1)
+	return resp.OwnershipControls.Rules[0].ObjectOwnership
+}
+
+func putObjectOwnership(t *testing.T, client *s3.Client, bucketName string, ownership types.ObjectOwnership) {
+	t.Helper()
+	_, err := client.PutBucketOwnershipControls(context.TODO(), &s3.PutBucketOwnershipControlsInput{
+		Bucket: aws.String(bucketName),
+		OwnershipControls: &types.OwnershipControls{
+			Rules: []types.OwnershipControlsRule{{ObjectOwnership: ownership}},
+		},
+	})
+	require.NoError(t, err)
+}
+
+func TestGetBucketOwnershipControlsDefault(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createOwnershipTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	assert.Equal(t, types.ObjectOwnershipBucketOwnerEnforced, getObjectOwnership(t, client, bucketName))
+}
+
+func TestBucketOwnershipControlsLifecycle(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createOwnershipTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	putObjectOwnership(t, client, bucketName, types.ObjectOwnershipObjectWriter)
+	assert.Equal(t, types.ObjectOwnershipObjectWriter, getObjectOwnership(t, client, bucketName))
+
+	putObjectOwnership(t, client, bucketName, types.ObjectOwnershipBucketOwnerEnforced)
+	assert.Equal(t, types.ObjectOwnershipBucketOwnerEnforced, getObjectOwnership(t, client, bucketName))
+
+	_, err := client.DeleteBucketOwnershipControls(context.TODO(), &s3.DeleteBucketOwnershipControlsInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+
+	assert.Equal(t, types.ObjectOwnershipBucketOwnerEnforced, getObjectOwnership(t, client, bucketName))
+}
+
+// A bucket with nothing stored already reports BucketOwnerEnforced, so this put has
+// to persist anyway or the delete below finds nothing to remove.
+func TestPutBucketOwnershipControlsDefaultOnNewBucket(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createOwnershipTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	putObjectOwnership(t, client, bucketName, types.ObjectOwnershipBucketOwnerEnforced)
+
+	_, err := client.DeleteBucketOwnershipControls(context.TODO(), &s3.DeleteBucketOwnershipControlsInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+
+	_, err = client.DeleteBucketOwnershipControls(context.TODO(), &s3.DeleteBucketOwnershipControlsInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "OwnershipControlsNotFound")
+}

test/s3/basic/basic_test.go

@@ -0,0 +1,227 @@
+package basic
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+var (
+	svc *s3.S3
+)
+
+func init() {
+	// Initialize a session in us-west-2 that the SDK will use to load
+	// credentials from the shared credentials file ~/.aws/credentials.
+	sess, err := session.NewSession(&aws.Config{
+		Region:     aws.String("us-west-2"),
+		Endpoint:   aws.String("localhost:8333"),
+		DisableSSL: aws.Bool(true),
+	})
+	if err != nil {
+		exitErrorf("create session, %v", err)
+	}
+
+	// Create S3 service client
+	svc = s3.New(sess)
+}
+
+func TestCreateBucket(t *testing.T) {
+
+	input := &s3.CreateBucketInput{
+		Bucket: aws.String("theBucket"),
+	}
+
+	result, err := svc.CreateBucket(input)
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			case s3.ErrCodeBucketAlreadyExists:
+				fmt.Println(s3.ErrCodeBucketAlreadyExists, aerr.Error())
+			case s3.ErrCodeBucketAlreadyOwnedByYou:
+				fmt.Println(s3.ErrCodeBucketAlreadyOwnedByYou, aerr.Error())
+			default:
+				fmt.Println(aerr.Error())
+			}
+		} else {
+			// Print the error, cast err to awserr.Error to get the Code and
+			// Message from an error.
+			fmt.Println(err.Error())
+		}
+		return
+	}
+
+	fmt.Println(result)
+
+}
+
+func TestPutObject(t *testing.T) {
+
+	input := &s3.PutObjectInput{
+		ACL:    aws.String("authenticated-read"),
+		Body:   aws.ReadSeekCloser(strings.NewReader("filetoupload")),
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("exampleobject"),
+	}
+
+	result, err := svc.PutObject(input)
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			default:
+				fmt.Println(aerr.Error())
+			}
+		} else {
+			// Print the error, cast err to awserr.Error to get the Code and
+			// Message from an error.
+			fmt.Println(err.Error())
+		}
+		return
+	}
+
+	fmt.Println(result)
+
+}
+
+func TestListBucket(t *testing.T) {
+
+	result, err := svc.ListBuckets(nil)
+	if err != nil {
+		exitErrorf("Unable to list buckets, %v", err)
+	}
+
+	fmt.Println("Buckets:")
+
+	for _, b := range result.Buckets {
+		fmt.Printf("* %s created on %s\n",
+			aws.StringValue(b.Name), aws.TimeValue(b.CreationDate))
+	}
+
+}
+
+func TestListObjectV2(t *testing.T) {
+
+	listObj, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
+		Bucket:    aws.String(Bucket),
+		Prefix:    aws.String("foo"),
+		Delimiter: aws.String("/"),
+	})
+	if err != nil {
+		exitErrorf("Unable to list objects, %v", err)
+	}
+	for _, content := range listObj.Contents {
+		fmt.Println(aws.StringValue(content.Key))
+	}
+	fmt.Printf("list: %s\n", listObj)
+
+}
+
+func exitErrorf(msg string, args ...interface{}) {
+	fmt.Fprintf(os.Stderr, msg+"\n", args...)
+	os.Exit(1)
+}
+
+const (
+	Bucket = "theBucket"
+	object = "foo/bar"
+	Data   = "<data>"
+)
+
+func TestObjectOp(t *testing.T) {
+	_, err := svc.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(Bucket),
+	})
+	if err != nil {
+		exitErrorf("Unable to create bucket, %v", err)
+	}
+
+	_, err = svc.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(Bucket),
+		Key:    aws.String(object),
+		Body:   strings.NewReader(Data),
+	})
+	if err != nil {
+		exitErrorf("Unable to put object, %v", err)
+	}
+
+	dest := fmt.Sprintf("%s_bak", object)
+	copyObj, err := svc.CopyObject(&s3.CopyObjectInput{
+		Bucket:     aws.String(Bucket),
+		CopySource: aws.String(fmt.Sprintf("%s/%s", Bucket, object)),
+		Key:        aws.String(dest),
+	})
+	if err != nil {
+		exitErrorf("Unable to copy object, %v", err)
+	}
+	t.Log("copy object result -> ", copyObj.CopyObjectResult)
+
+	getObj, err := svc.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(Bucket),
+		Key:    aws.String(dest),
+	})
+	if err != nil {
+		exitErrorf("Unable to get copy object, %v", err)
+	}
+
+	data, err := io.ReadAll(getObj.Body)
+	if err != nil {
+		exitErrorf("Unable to read object data, %v", err)
+	}
+	if string(data) != Data {
+		t.Error("object data -> ", string(data))
+	}
+
+	listObj, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
+		Bucket: aws.String(Bucket),
+		Prefix: aws.String("foo/"),
+	})
+	if err != nil {
+		exitErrorf("Unable to list objects, %v", err)
+	}
+	count := 0
+	for _, content := range listObj.Contents {
+		key := aws.StringValue(content.Key)
+		if key == dest {
+			count++
+		} else if key == object {
+			count++
+		}
+		if count == 2 {
+			break
+		}
+	}
+	if count != 2 {
+		exitErrorf("Unable to find two objects, %v", listObj.Contents)
+	}
+
+	_, err = svc.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(Bucket),
+		Key:    aws.String(object),
+	})
+	if err != nil {
+		exitErrorf("Unable to delete source object, %v", err)
+	}
+
+	_, err = svc.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(Bucket),
+		Key:    aws.String(dest),
+	})
+	if err != nil {
+		exitErrorf("Unable to delete object, %v", err)
+	}
+
+	_, err = svc.DeleteBucket(&s3.DeleteBucketInput{
+		Bucket: aws.String(Bucket),
+	})
+
+	if err != nil {
+		exitErrorf("Unable to delete bucket, %v", err)
+	}
+}

test/s3/basic/delimiter_test.go

@@ -0,0 +1,169 @@
+package basic
+
+import (
+	"fmt"
+	"math/rand"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestS3ListDelimiterWithDirectoryKeyObjects tests the specific scenario from
+// test_bucket_list_delimiter_not_skip_special where directory key objects
+// should be properly grouped into common prefixes when using delimiters
+func TestS3ListDelimiterWithDirectoryKeyObjects(t *testing.T) {
+	bucketName := fmt.Sprintf("test-delimiter-dir-key-%d", rand.Int31())
+
+	// Create bucket
+	_, err := svc.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+	defer cleanupBucket(t, bucketName)
+
+	// Create objects matching the failing test scenario:
+	// ['0/'] + ['0/1000', '0/1001', '0/1002'] + ['1999', '1999#', '1999+', '2000']
+	objects := []string{
+		"0/",     // Directory key object
+		"0/1000", // Objects under 0/ prefix
+		"0/1001",
+		"0/1002",
+		"1999", // Objects without delimiter
+		"1999#",
+		"1999+",
+		"2000",
+	}
+
+	// Create all objects
+	for _, key := range objects {
+		_, err := svc.PutObject(&s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(key),
+			Body:   strings.NewReader(fmt.Sprintf("content for %s", key)),
+		})
+		require.NoError(t, err, "Failed to create object %s", key)
+	}
+
+	// Test with delimiter='/'
+	resp, err := svc.ListObjects(&s3.ListObjectsInput{
+		Bucket:    aws.String(bucketName),
+		Delimiter: aws.String("/"),
+	})
+	require.NoError(t, err)
+
+	// Extract keys and prefixes
+	var keys []string
+	for _, content := range resp.Contents {
+		keys = append(keys, *content.Key)
+	}
+
+	var prefixes []string
+	for _, prefix := range resp.CommonPrefixes {
+		prefixes = append(prefixes, *prefix.Prefix)
+	}
+
+	// Expected results:
+	// Keys should be: ['1999', '1999#', '1999+', '2000'] (objects without delimiters)
+	// Prefixes should be: ['0/'] (grouping '0/' and all '0/xxxx' objects)
+
+	expectedKeys := []string{"1999", "1999#", "1999+", "2000"}
+	expectedPrefixes := []string{"0/"}
+
+	t.Logf("Actual keys: %v", keys)
+	t.Logf("Actual prefixes: %v", prefixes)
+
+	assert.ElementsMatch(t, expectedKeys, keys, "Keys should only include objects without delimiters")
+	assert.ElementsMatch(t, expectedPrefixes, prefixes, "CommonPrefixes should group directory key object with other objects sharing prefix")
+
+	// Additional validation
+	assert.Equal(t, "/", *resp.Delimiter, "Delimiter should be set correctly")
+	assert.Contains(t, prefixes, "0/", "Directory key object '0/' should be grouped into common prefix '0/'")
+	assert.NotContains(t, keys, "0/", "Directory key object '0/' should NOT appear as individual key when delimiter is used")
+
+	// Verify none of the '0/xxxx' objects appear as individual keys
+	for _, key := range keys {
+		assert.False(t, strings.HasPrefix(key, "0/"), "No object with '0/' prefix should appear as individual key, found: %s", key)
+	}
+}
+
+// TestS3ListWithoutDelimiter tests that directory key objects appear as individual keys when no delimiter is used
+func TestS3ListWithoutDelimiter(t *testing.T) {
+	bucketName := fmt.Sprintf("test-no-delimiter-%d", rand.Int31())
+
+	// Create bucket
+	_, err := svc.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+	defer cleanupBucket(t, bucketName)
+
+	// Create objects
+	objects := []string{"0/", "0/1000", "1999"}
+
+	for _, key := range objects {
+		_, err := svc.PutObject(&s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(key),
+			Body:   strings.NewReader(fmt.Sprintf("content for %s", key)),
+		})
+		require.NoError(t, err)
+	}
+
+	// Test without delimiter
+	resp, err := svc.ListObjects(&s3.ListObjectsInput{
+		Bucket: aws.String(bucketName),
+		// No delimiter specified
+	})
+	require.NoError(t, err)
+
+	// Extract keys
+	var keys []string
+	for _, content := range resp.Contents {
+		keys = append(keys, *content.Key)
+	}
+
+	// When no delimiter is used, all objects should be returned as individual keys
+	expectedKeys := []string{"0/", "0/1000", "1999"}
+	assert.ElementsMatch(t, expectedKeys, keys, "All objects should be individual keys when no delimiter is used")
+
+	// No common prefixes should be present
+	assert.Empty(t, resp.CommonPrefixes, "No common prefixes should be present when no delimiter is used")
+	assert.Contains(t, keys, "0/", "Directory key object '0/' should appear as individual key when no delimiter is used")
+}
+
+func cleanupBucket(t *testing.T, bucketName string) {
+	// Delete all objects
+	resp, err := svc.ListObjects(&s3.ListObjectsInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		t.Logf("Failed to list objects for cleanup: %v", err)
+		return
+	}
+
+	for _, obj := range resp.Contents {
+		_, err := svc.DeleteObject(&s3.DeleteObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    obj.Key,
+		})
+		if err != nil {
+			t.Logf("Failed to delete object %s: %v", *obj.Key, err)
+		}
+	}
+
+	// Give some time for eventual consistency
+	time.Sleep(100 * time.Millisecond)
+
+	// Delete bucket
+	_, err = svc.DeleteBucket(&s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		t.Logf("Failed to delete bucket %s: %v", bucketName, err)
+	}
+}

test/s3/basic/object_tagging_test.go

@@ -0,0 +1,228 @@
+package basic
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+func TestObjectTagging(t *testing.T) {
+
+	input := &s3.PutObjectInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("testDir/testObject"),
+	}
+
+	svc.PutObject(input)
+
+	printTags()
+
+	setTags()
+
+	printTags()
+
+	clearTags()
+
+	printTags()
+
+}
+
+func printTags() {
+	response, err := svc.GetObjectTagging(
+		&s3.GetObjectTaggingInput{
+			Bucket: aws.String("theBucket"),
+			Key:    aws.String("testDir/testObject"),
+		})
+
+	fmt.Println("printTags")
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+
+	fmt.Println(response.TagSet)
+}
+
+func setTags() {
+
+	response, err := svc.PutObjectTagging(&s3.PutObjectTaggingInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("testDir/testObject"),
+		Tagging: &s3.Tagging{
+			TagSet: []*s3.Tag{
+				{
+					Key:   aws.String("kye2"),
+					Value: aws.String("value2"),
+				},
+			},
+		},
+	})
+
+	fmt.Println("setTags")
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+
+	fmt.Println(response.String())
+}
+
+func clearTags() {
+
+	response, err := svc.DeleteObjectTagging(&s3.DeleteObjectTaggingInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("testDir/testObject"),
+	})
+
+	fmt.Println("clearTags")
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+
+	fmt.Println(response.String())
+}
+
+func TestObjectTaggingWithEncodedValues(t *testing.T) {
+	// Test for URL encoded tag values
+	input := &s3.PutObjectInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("testDir/testObjectWithEncodedTags"),
+	}
+
+	svc.PutObject(input)
+
+	// Set tags with encoded values (simulating what would happen with timestamps containing spaces and colons)
+	_, err := svc.PutObjectTagging(&s3.PutObjectTaggingInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("testDir/testObjectWithEncodedTags"),
+		Tagging: &s3.Tagging{
+			TagSet: []*s3.Tag{
+				{
+					Key:   aws.String("Timestamp"),
+					Value: aws.String("2025-07-16 14:40:39"), // This would be URL encoded as "2025-07-16%2014%3A40%3A39" in the header
+				},
+				{
+					Key:   aws.String("Path"),
+					Value: aws.String("/tmp/file.txt"), // This would be URL encoded as "/tmp%2Ffile.txt" in the header
+				},
+			},
+		},
+	})
+
+	if err != nil {
+		t.Fatalf("Failed to set tags with encoded values: %v", err)
+	}
+
+	// Get tags back and verify they are properly decoded
+	response, err := svc.GetObjectTagging(&s3.GetObjectTaggingInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("testDir/testObjectWithEncodedTags"),
+	})
+
+	if err != nil {
+		t.Fatalf("Failed to get tags: %v", err)
+	}
+
+	// Verify that the tags are properly decoded
+	tagMap := make(map[string]string)
+	for _, tag := range response.TagSet {
+		tagMap[*tag.Key] = *tag.Value
+	}
+
+	expectedTimestamp := "2025-07-16 14:40:39"
+	if tagMap["Timestamp"] != expectedTimestamp {
+		t.Errorf("Expected Timestamp tag to be '%s', got '%s'", expectedTimestamp, tagMap["Timestamp"])
+	}
+
+	expectedPath := "/tmp/file.txt"
+	if tagMap["Path"] != expectedPath {
+		t.Errorf("Expected Path tag to be '%s', got '%s'", expectedPath, tagMap["Path"])
+	}
+
+	fmt.Printf("✓ URL encoded tags test passed - Timestamp: %s, Path: %s\n", tagMap["Timestamp"], tagMap["Path"])
+
+	// Clean up
+	svc.DeleteObjectTagging(&s3.DeleteObjectTaggingInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String("testDir/testObjectWithEncodedTags"),
+	})
+}
+
+// TestObjectUploadWithEncodedTags tests the specific issue reported in GitHub issue #7040
+// where tags sent via X-Amz-Tagging header during object upload are not URL decoded properly
+func TestObjectUploadWithEncodedTags(t *testing.T) {
+	// This test specifically addresses the issue where tags with special characters
+	// (like spaces, colons, slashes) sent during object upload are not URL decoded
+	// This tests the fix in filer_server_handlers_write_autochunk.go
+
+	objectKey := "testDir/testObjectUploadWithTags"
+
+	// Upload object with tags that contain special characters that would be URL encoded
+	// The AWS SDK will automatically URL encode these when sending the X-Amz-Tagging header
+	// Test edge cases that url.ParseQuery handles better than manual parsing:
+	// - Values containing "=" characters
+	// - Empty values
+	// - Complex special characters
+	_, err := svc.PutObject(&s3.PutObjectInput{
+		Bucket:  aws.String("theBucket"),
+		Key:     aws.String(objectKey),
+		Body:    aws.ReadSeekCloser(strings.NewReader("test content")),
+		Tagging: aws.String("Timestamp=2025-07-16 14:40:39&Path=/tmp/file.txt&Description=A test file with spaces&Equation=x=y+1&EmptyValue=&Complex=A%20tag%20with%20%26%20%3D%20chars"),
+	})
+
+	if err != nil {
+		t.Fatalf("Failed to upload object with tags: %v", err)
+	}
+
+	// Get the tags back to verify they were properly URL decoded during upload
+	response, err := svc.GetObjectTagging(&s3.GetObjectTaggingInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String(objectKey),
+	})
+
+	if err != nil {
+		t.Fatalf("Failed to get object tags: %v", err)
+	}
+
+	// Verify that the tags are properly decoded (not URL encoded)
+	tagMap := make(map[string]string)
+	for _, tag := range response.TagSet {
+		tagMap[*tag.Key] = *tag.Value
+	}
+
+	// Test cases for values that would be URL encoded in the X-Amz-Tagging header
+	testCases := []struct {
+		key           string
+		expectedValue string
+		description   string
+	}{
+		{"Timestamp", "2025-07-16 14:40:39", "timestamp with spaces and colons"},
+		{"Path", "/tmp/file.txt", "file path with slashes"},
+		{"Description", "A test file with spaces", "description with spaces"},
+	}
+
+	for _, tc := range testCases {
+		actualValue, exists := tagMap[tc.key]
+		if !exists {
+			t.Errorf("Expected tag key '%s' not found", tc.key)
+			continue
+		}
+
+		if actualValue != tc.expectedValue {
+			t.Errorf("Tag '%s' (%s): expected '%s', got '%s'",
+				tc.key, tc.description, tc.expectedValue, actualValue)
+		} else {
+			fmt.Printf("✓ Tag '%s' correctly decoded: '%s'\n", tc.key, actualValue)
+		}
+	}
+
+	// Clean up
+	_, err = svc.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String("theBucket"),
+		Key:    aws.String(objectKey),
+	})
+	if err != nil {
+		t.Logf("Warning: Failed to clean up test object: %v", err)
+	}
+}

test/s3/benchmark/s3_benchmark.go

@@ -0,0 +1,223 @@
+// Command s3_benchmark drives concurrent PUT and GET load against an S3
+// gateway and reports throughput and latency. It is a self-contained load
+// generator for the performance CI, using the aws-sdk-go-v2 client the rest
+// of the S3 tests already depend on.
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+func main() {
+	endpoint := flag.String("endpoint", "http://localhost:8000", "S3 gateway endpoint")
+	accessKey := flag.String("access-key", "some_access_key1", "S3 access key")
+	secretKey := flag.String("secret-key", "some_secret_key1", "S3 secret key")
+	region := flag.String("region", "us-east-1", "S3 region")
+	bucket := flag.String("bucket", "perf-benchmark", "bucket to write into")
+	objects := flag.Int("objects", 10000, "number of objects")
+	size := flag.Int("size", 1024, "object size in bytes")
+	concurrency := flag.Int("concurrency", 16, "concurrent workers")
+	mode := flag.String("mode", "both", "write, read, or both")
+	flag.Parse()
+
+	cfg, err := config.LoadDefaultConfig(context.Background(),
+		config.WithRegion(*region),
+		config.WithRetryMaxAttempts(1),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(*accessKey, *secretKey, "")),
+	)
+	if err != nil {
+		fatalf("load aws config: %v", err)
+	}
+	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.BaseEndpoint = aws.String(*endpoint)
+		o.UsePathStyle = true
+	})
+
+	if err := ensureBucket(client, *bucket); err != nil {
+		fatalf("ensure bucket: %v", err)
+	}
+
+	payload := make([]byte, *size)
+	for i := range payload {
+		payload[i] = byte(i)
+	}
+
+	doWrite := *mode == "both" || *mode == "write"
+	doRead := *mode == "both" || *mode == "read"
+
+	if doWrite {
+		report("S3 WRITE", *concurrency, *size, run(*concurrency, *objects, func(key string) error {
+			_, err := client.PutObject(context.Background(), &s3.PutObjectInput{
+				Bucket:        bucket,
+				Key:           aws.String(key),
+				Body:          bytes.NewReader(payload),
+				ContentLength: aws.Int64(int64(*size)),
+			})
+			return err
+		}))
+	}
+
+	if doRead {
+		report("S3 READ", *concurrency, *size, run(*concurrency, *objects, func(key string) error {
+			out, err := client.GetObject(context.Background(), &s3.GetObjectInput{
+				Bucket: bucket,
+				Key:    aws.String(key),
+			})
+			if err != nil {
+				return err
+			}
+			defer out.Body.Close()
+			_, err = io.Copy(io.Discard, out.Body)
+			return err
+		}))
+	}
+}
+
+type result struct {
+	completed int64
+	failed    int64
+	elapsed   time.Duration
+	latencies []time.Duration
+	firstErr  error
+}
+
+// run fans `total` keyed operations across `workers` goroutines, timing each.
+func run(workers, total int, op func(key string) error) result {
+	var (
+		completed, failed, nextKey atomic.Int64
+		mu                         sync.Mutex
+		firstErr                   error
+		latencies                  = make([]time.Duration, 0, total)
+		wg                         sync.WaitGroup
+	)
+	start := time.Now()
+	for w := 0; w < workers; w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			local := make([]time.Duration, 0, total/workers+1)
+			for {
+				i := nextKey.Add(1) - 1
+				if i >= int64(total) {
+					break
+				}
+				key := fmt.Sprintf("obj-%d", i)
+				opStart := time.Now()
+				err := op(key)
+				local = append(local, time.Since(opStart))
+				if err != nil {
+					failed.Add(1)
+					mu.Lock()
+					if firstErr == nil {
+						firstErr = err
+					}
+					mu.Unlock()
+					continue
+				}
+				completed.Add(1)
+			}
+			mu.Lock()
+			latencies = append(latencies, local...)
+			mu.Unlock()
+		}()
+	}
+	wg.Wait()
+	return result{
+		completed: completed.Load(),
+		failed:    failed.Load(),
+		elapsed:   time.Since(start),
+		latencies: latencies,
+		firstErr:  firstErr,
+	}
+}
+
+func report(label string, concurrency, size int, r result) {
+	secs := r.elapsed.Seconds()
+	transferred := r.completed * int64(size)
+	rps := 0.0
+	kbps := 0.0
+	if secs > 0 {
+		rps = float64(r.completed) / secs
+		kbps = float64(transferred) / 1024 / secs
+	}
+	fmt.Printf("\n%s results:\n", label)
+	fmt.Printf("Concurrency Level:      %d\n", concurrency)
+	fmt.Printf("Time taken for tests:   %.3f seconds\n", secs)
+	fmt.Printf("Completed requests:     %d\n", r.completed)
+	fmt.Printf("Failed requests:        %d\n", r.failed)
+	fmt.Printf("Total transferred:      %d bytes\n", transferred)
+	fmt.Printf("Requests per second:    %.2f [#/sec]\n", rps)
+	fmt.Printf("Transfer rate:          %.2f [Kbytes/sec]\n", kbps)
+	p := percentiles(r.latencies)
+	fmt.Printf("Latency (ms):           avg=%.2f p50=%.2f p90=%.2f p99=%.2f max=%.2f\n",
+		ms(p.avg), ms(p.p50), ms(p.p90), ms(p.p99), ms(p.max))
+	if r.failed > 0 && r.firstErr != nil {
+		fmt.Printf("First error:            %v\n", r.firstErr)
+		os.Exit(1)
+	}
+}
+
+type latencyStats struct {
+	avg, p50, p90, p99, max time.Duration
+}
+
+func percentiles(d []time.Duration) latencyStats {
+	if len(d) == 0 {
+		return latencyStats{}
+	}
+	slices.Sort(d)
+	var sum time.Duration
+	for _, v := range d {
+		sum += v
+	}
+	at := func(q float64) time.Duration {
+		idx := int(q * float64(len(d)))
+		if idx >= len(d) {
+			idx = len(d) - 1
+		}
+		return d[idx]
+	}
+	return latencyStats{
+		avg: sum / time.Duration(len(d)),
+		p50: at(0.50),
+		p90: at(0.90),
+		p99: at(0.99),
+		max: d[len(d)-1],
+	}
+}
+
+func ms(d time.Duration) float64 { return float64(d) / float64(time.Millisecond) }
+
+func ensureBucket(client *s3.Client, bucket string) error {
+	_, err := client.CreateBucket(context.Background(), &s3.CreateBucketInput{Bucket: aws.String(bucket)})
+	if err == nil {
+		return nil
+	}
+	var owned *types.BucketAlreadyOwnedByYou
+	var exists *types.BucketAlreadyExists
+	if errors.As(err, &owned) || errors.As(err, &exists) {
+		return nil
+	}
+	return err
+}
+
+func fatalf(format string, args ...any) {
+	fmt.Fprintf(os.Stderr, format+"\n", args...)
+	os.Exit(1)
+}

test/s3/checksum/.gitignore

@@ -0,0 +1,3 @@
+test-volume-data/
+weed-test.log
+weed-server.pid

test/s3/checksum/Makefile

@@ -0,0 +1,100 @@
+# S3 Checksum Integration Tests
+# Covers flexible-checksum behavior on presigned uploads and multipart copy.
+
+.PHONY: help build-weed check-deps start-server stop-server test test-with-server logs clean health-check
+
+WEED_BINARY := ../../../weed/weed_binary
+S3_PORT := 8333
+ACCESS_KEY ?= some_access_key1
+SECRET_KEY ?= some_secret_key1
+TEST_TIMEOUT := 10m
+TEST_PATTERN ?= .
+SERVER_DIR := ./test-volume-data/server-data
+
+help:
+	@echo "S3 Checksum Integration Tests"
+	@echo ""
+	@echo "Targets:"
+	@echo "  build-weed         Build the SeaweedFS binary"
+	@echo "  start-server       Start a local SeaweedFS S3 server for testing"
+	@echo "  stop-server        Stop the local SeaweedFS server"
+	@echo "  test               Run checksum tests (server must already be running)"
+	@echo "  test-with-server   Start server, run tests, stop server"
+	@echo "  logs               Tail server log"
+	@echo "  clean              Remove test artifacts"
+
+build-weed:
+	@echo "Building SeaweedFS binary..."
+	@cd ../../../weed && go build -o weed_binary .
+	@chmod +x $(WEED_BINARY)
+
+check-deps: build-weed
+	@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1)
+	@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1)
+	@go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1)
+	@go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1)
+
+start-server: check-deps
+	@echo "Starting SeaweedFS server..."
+	@rm -f weed-server.pid
+	@mkdir -p $(SERVER_DIR)
+	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \
+		-dir=$(SERVER_DIR) \
+		-s3.port=$(S3_PORT) \
+		> weed-test.log 2>&1 & \
+		echo $$! > weed-server.pid
+	@for i in $$(seq 1 90); do \
+		if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
+			echo "✅ SeaweedFS server started on port $(S3_PORT) after $$i seconds"; \
+			exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "❌ Server failed to start within 90 seconds"; \
+	if [ -f weed-server.pid ]; then \
+		PID=$$(cat weed-server.pid); \
+		if ps -p $$PID >/dev/null 2>&1; then \
+			kill -TERM $$PID 2>/dev/null || true; \
+			sleep 1; \
+			ps -p $$PID >/dev/null 2>&1 && kill -KILL $$PID 2>/dev/null || true; \
+		fi; \
+		rm -f weed-server.pid; \
+	fi; \
+	if [ -f weed-test.log ]; then tail -100 weed-test.log; fi; \
+	exit 1
+
+stop-server:
+	@if [ -f weed-server.pid ]; then \
+		PID=$$(cat weed-server.pid); \
+		if ps -p $$PID >/dev/null 2>&1; then \
+			kill -TERM $$PID 2>/dev/null || true; \
+			sleep 2; \
+			ps -p $$PID >/dev/null 2>&1 && kill -KILL $$PID 2>/dev/null || true; \
+		fi; \
+		rm -f weed-server.pid; \
+	fi
+	@echo "✅ SeaweedFS server stopped"
+
+test: check-deps
+	@echo "Running checksum tests (pattern: $(TEST_PATTERN))..."
+	@S3_ENDPOINT=http://localhost:$(S3_PORT) \
+		S3_ACCESS_KEY=$(ACCESS_KEY) \
+		S3_SECRET_KEY=$(SECRET_KEY) \
+		go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" .
+
+test-with-server: start-server
+	@sleep 3
+	@$(MAKE) test || (echo "Tests failed, stopping server..." && $(MAKE) stop-server && exit 1)
+	@$(MAKE) stop-server
+	@echo "✅ Checksum tests completed"
+
+logs:
+	@if [ -f weed-test.log ]; then tail -f weed-test.log; else echo "No log file"; fi
+
+health-check:
+	@curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1 && echo "✅ S3 on $(S3_PORT)" || (echo "❌ S3 not up" && exit 1)
+
+clean:
+	@$(MAKE) stop-server
+	@rm -f weed-test.log weed-server.pid
+	@rm -rf ./test-volume-data

test/s3/checksum/s3_checksum_multipart_test.go

@@ -0,0 +1,249 @@
+package checksum_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/http/httptest"
+	"net/http/httputil"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/require"
+)
+
+// newTrailerChecksumClient returns a client that can send flexible checksums.
+// aws-sdk-go-v2 sends them as unsigned streaming trailers, which it refuses over
+// plain HTTP, so front the HTTP endpoint with a TLS proxy.
+func newTrailerChecksumClient(t *testing.T) *s3.Client {
+	t.Helper()
+
+	target, err := url.Parse(defaultConfig.Endpoint)
+	require.NoError(t, err)
+
+	proxy := httputil.NewSingleHostReverseProxy(target)
+	server := httptest.NewTLSServer(proxy)
+	t.Cleanup(server.Close)
+
+	cfg, err := config.LoadDefaultConfig(context.Background(),
+		config.WithRegion(defaultConfig.Region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+			defaultConfig.AccessKey, defaultConfig.SecretKey, "")),
+		config.WithHTTPClient(server.Client()),
+	)
+	require.NoError(t, err)
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.BaseEndpoint = aws.String(server.URL)
+		o.UsePathStyle = true
+	})
+}
+
+func TestMultipartCopyPreservesChecksum(t *testing.T) {
+	client := newTrailerChecksumClient(t)
+
+	bucket := uniqueBucket()
+	createBucket(t, client, bucket)
+	defer cleanupBucket(t, client, bucket)
+
+	cases := []struct {
+		algorithm types.ChecksumAlgorithm
+		srcSum    func(*s3.PutObjectOutput) *string
+		partSum   func(*types.CopyPartResult) *string
+		setPart   func(*types.CompletedPart, *string)
+	}{
+		{
+			algorithm: types.ChecksumAlgorithmCrc32,
+			srcSum:    func(o *s3.PutObjectOutput) *string { return o.ChecksumCRC32 },
+			partSum:   func(r *types.CopyPartResult) *string { return r.ChecksumCRC32 },
+			setPart:   func(p *types.CompletedPart, v *string) { p.ChecksumCRC32 = v },
+		},
+		{
+			algorithm: types.ChecksumAlgorithmCrc32c,
+			srcSum:    func(o *s3.PutObjectOutput) *string { return o.ChecksumCRC32C },
+			partSum:   func(r *types.CopyPartResult) *string { return r.ChecksumCRC32C },
+			setPart:   func(p *types.CompletedPart, v *string) { p.ChecksumCRC32C = v },
+		},
+		{
+			algorithm: types.ChecksumAlgorithmCrc64nvme,
+			srcSum:    func(o *s3.PutObjectOutput) *string { return o.ChecksumCRC64NVME },
+			partSum:   func(r *types.CopyPartResult) *string { return r.ChecksumCRC64NVME },
+			setPart:   func(p *types.CompletedPart, v *string) { p.ChecksumCRC64NVME = v },
+		},
+		{
+			algorithm: types.ChecksumAlgorithmSha1,
+			srcSum:    func(o *s3.PutObjectOutput) *string { return o.ChecksumSHA1 },
+			partSum:   func(r *types.CopyPartResult) *string { return r.ChecksumSHA1 },
+			setPart:   func(p *types.CompletedPart, v *string) { p.ChecksumSHA1 = v },
+		},
+		{
+			algorithm: types.ChecksumAlgorithmSha256,
+			srcSum:    func(o *s3.PutObjectOutput) *string { return o.ChecksumSHA256 },
+			partSum:   func(r *types.CopyPartResult) *string { return r.ChecksumSHA256 },
+			setPart:   func(p *types.CompletedPart, v *string) { p.ChecksumSHA256 = v },
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(string(tc.algorithm), func(t *testing.T) {
+			sourceKey := "source-" + string(tc.algorithm)
+			source, err := client.PutObject(context.Background(), &s3.PutObjectInput{
+				Bucket:            aws.String(bucket),
+				Key:               aws.String(sourceKey),
+				Body:              bytes.NewBufferString("trailing checksum"),
+				ChecksumAlgorithm: tc.algorithm,
+			})
+			require.NoError(t, err)
+
+			key := "multipart-copy-" + string(tc.algorithm)
+			create, err := client.CreateMultipartUpload(context.Background(), &s3.CreateMultipartUploadInput{
+				Bucket:            aws.String(bucket),
+				Key:               aws.String(key),
+				ChecksumAlgorithm: tc.algorithm,
+			})
+			require.NoError(t, err)
+
+			part, err := client.UploadPartCopy(context.Background(), &s3.UploadPartCopyInput{
+				Bucket:     aws.String(bucket),
+				CopySource: aws.String(fmt.Sprintf("%s/%s", bucket, sourceKey)),
+				Key:        aws.String(key),
+				UploadId:   create.UploadId,
+				PartNumber: aws.Int32(1),
+			})
+			require.NoError(t, err)
+			require.Equal(t, aws.ToString(tc.srcSum(source)), aws.ToString(tc.partSum(part.CopyPartResult)))
+
+			completed := types.CompletedPart{ETag: part.CopyPartResult.ETag, PartNumber: aws.Int32(1)}
+			tc.setPart(&completed, tc.partSum(part.CopyPartResult))
+			_, err = client.CompleteMultipartUpload(context.Background(), &s3.CompleteMultipartUploadInput{
+				Bucket:          aws.String(bucket),
+				Key:             aws.String(key),
+				UploadId:        create.UploadId,
+				MultipartUpload: &types.CompletedMultipartUpload{Parts: []types.CompletedPart{completed}},
+			})
+			require.NoError(t, err)
+		})
+	}
+}
+
+// A multipart upload that asked for a checksum must report it in the
+// CompleteMultipartUpload response, alongside the algorithm and type that
+// CreateMultipartUpload echoed back.
+func TestMultipartUploadReturnsObjectChecksum(t *testing.T) {
+	client := newTrailerChecksumClient(t)
+
+	bucket := uniqueBucket()
+	createBucket(t, client, bucket)
+	defer cleanupBucket(t, client, bucket)
+
+	cases := []struct {
+		algorithm    types.ChecksumAlgorithm
+		expectedType types.ChecksumType
+		completeSum  func(*s3.CompleteMultipartUploadOutput) *string
+		headSum      func(*s3.HeadObjectOutput) *string
+		partSum      func(*s3.UploadPartOutput) *string
+		setPart      func(*types.CompletedPart, *string)
+	}{
+		{
+			algorithm:    types.ChecksumAlgorithmCrc32,
+			expectedType: types.ChecksumTypeComposite,
+			completeSum:  func(o *s3.CompleteMultipartUploadOutput) *string { return o.ChecksumCRC32 },
+			headSum:      func(o *s3.HeadObjectOutput) *string { return o.ChecksumCRC32 },
+			partSum:      func(o *s3.UploadPartOutput) *string { return o.ChecksumCRC32 },
+			setPart:      func(p *types.CompletedPart, v *string) { p.ChecksumCRC32 = v },
+		},
+		{
+			algorithm:    types.ChecksumAlgorithmCrc32c,
+			expectedType: types.ChecksumTypeComposite,
+			completeSum:  func(o *s3.CompleteMultipartUploadOutput) *string { return o.ChecksumCRC32C },
+			headSum:      func(o *s3.HeadObjectOutput) *string { return o.ChecksumCRC32C },
+			partSum:      func(o *s3.UploadPartOutput) *string { return o.ChecksumCRC32C },
+			setPart:      func(p *types.CompletedPart, v *string) { p.ChecksumCRC32C = v },
+		},
+		{
+			algorithm:    types.ChecksumAlgorithmCrc64nvme,
+			expectedType: types.ChecksumTypeFullObject,
+			completeSum:  func(o *s3.CompleteMultipartUploadOutput) *string { return o.ChecksumCRC64NVME },
+			headSum:      func(o *s3.HeadObjectOutput) *string { return o.ChecksumCRC64NVME },
+			partSum:      func(o *s3.UploadPartOutput) *string { return o.ChecksumCRC64NVME },
+			setPart:      func(p *types.CompletedPart, v *string) { p.ChecksumCRC64NVME = v },
+		},
+		{
+			algorithm:    types.ChecksumAlgorithmSha1,
+			expectedType: types.ChecksumTypeComposite,
+			completeSum:  func(o *s3.CompleteMultipartUploadOutput) *string { return o.ChecksumSHA1 },
+			headSum:      func(o *s3.HeadObjectOutput) *string { return o.ChecksumSHA1 },
+			partSum:      func(o *s3.UploadPartOutput) *string { return o.ChecksumSHA1 },
+			setPart:      func(p *types.CompletedPart, v *string) { p.ChecksumSHA1 = v },
+		},
+		{
+			algorithm:    types.ChecksumAlgorithmSha256,
+			expectedType: types.ChecksumTypeComposite,
+			completeSum:  func(o *s3.CompleteMultipartUploadOutput) *string { return o.ChecksumSHA256 },
+			headSum:      func(o *s3.HeadObjectOutput) *string { return o.ChecksumSHA256 },
+			partSum:      func(o *s3.UploadPartOutput) *string { return o.ChecksumSHA256 },
+			setPart:      func(p *types.CompletedPart, v *string) { p.ChecksumSHA256 = v },
+		},
+	}
+
+	// Every part but the last has to reach the 5MB multipart minimum.
+	parts := [][]byte{bytes.Repeat([]byte("a"), 5*1024*1024), []byte("tail")}
+
+	for _, tc := range cases {
+		t.Run(string(tc.algorithm), func(t *testing.T) {
+			key := "multipart-" + string(tc.algorithm)
+			create, err := client.CreateMultipartUpload(context.Background(), &s3.CreateMultipartUploadInput{
+				Bucket:            aws.String(bucket),
+				Key:               aws.String(key),
+				ChecksumAlgorithm: tc.algorithm,
+			})
+			require.NoError(t, err)
+			require.Equal(t, tc.algorithm, create.ChecksumAlgorithm)
+			require.Equal(t, tc.expectedType, create.ChecksumType)
+
+			var completed []types.CompletedPart
+			for i, data := range parts {
+				part, err := client.UploadPart(context.Background(), &s3.UploadPartInput{
+					Bucket:            aws.String(bucket),
+					Key:               aws.String(key),
+					UploadId:          create.UploadId,
+					PartNumber:        aws.Int32(int32(i + 1)),
+					Body:              bytes.NewReader(data),
+					ChecksumAlgorithm: tc.algorithm,
+				})
+				require.NoError(t, err)
+				require.NotEmpty(t, aws.ToString(tc.partSum(part)))
+
+				entry := types.CompletedPart{ETag: part.ETag, PartNumber: aws.Int32(int32(i + 1))}
+				tc.setPart(&entry, tc.partSum(part))
+				completed = append(completed, entry)
+			}
+
+			done, err := client.CompleteMultipartUpload(context.Background(), &s3.CompleteMultipartUploadInput{
+				Bucket:          aws.String(bucket),
+				Key:             aws.String(key),
+				UploadId:        create.UploadId,
+				MultipartUpload: &types.CompletedMultipartUpload{Parts: completed},
+			})
+			require.NoError(t, err)
+			require.NotEmpty(t, aws.ToString(tc.completeSum(done)))
+			require.Equal(t, tc.expectedType, done.ChecksumType)
+			if tc.expectedType == types.ChecksumTypeComposite {
+				require.True(t, strings.HasSuffix(aws.ToString(tc.completeSum(done)), fmt.Sprintf("-%d", len(parts))))
+			}
+
+			head, err := client.HeadObject(context.Background(), &s3.HeadObjectInput{
+				Bucket:       aws.String(bucket),
+				Key:          aws.String(key),
+				ChecksumMode: types.ChecksumModeEnabled,
+			})
+			require.NoError(t, err)
+			require.Equal(t, aws.ToString(tc.completeSum(done)), aws.ToString(tc.headSum(head)))
+		})
+	}
+}

test/s3/checksum/s3_checksum_presigned_test.go

@@ -0,0 +1,256 @@
+package checksum_test
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Integration test for https://github.com/seaweedfs/seaweedfs/issues/9075:
+// presigned PUT URLs that request a flexible-checksum algorithm (SHA256, SHA1, ...)
+// must cause the server to compute and persist that checksum, and HEAD/GET must
+// return the x-amz-checksum-* header when the caller asks for it.
+//
+// AWS SDK presigners hoist headers like x-amz-sdk-checksum-algorithm into the
+// signed URL's query string, so we deliberately upload the body with a plain
+// http.Client to ensure the server sees the checksum algorithm via the query
+// string — which is exactly what a browser / curl / non-SDK client does.
+
+type s3TestConfig struct {
+	Endpoint     string
+	AccessKey    string
+	SecretKey    string
+	Region       string
+	BucketPrefix string
+}
+
+var defaultConfig = &s3TestConfig{
+	Endpoint:     "http://localhost:8333",
+	AccessKey:    "some_access_key1",
+	SecretKey:    "some_secret_key1",
+	Region:       "us-east-1",
+	BucketPrefix: "test-checksum-",
+}
+
+func getenvAny(keys ...string) string {
+	for _, k := range keys {
+		if v := os.Getenv(k); v != "" {
+			return v
+		}
+	}
+	return ""
+}
+
+func init() {
+	if v := getenvAny("S3_ENDPOINT"); v != "" {
+		defaultConfig.Endpoint = v
+	}
+	if v := getenvAny("S3_ACCESS_KEY", "AWS_ACCESS_KEY_ID"); v != "" {
+		defaultConfig.AccessKey = v
+	}
+	if v := getenvAny("S3_SECRET_KEY", "AWS_SECRET_ACCESS_KEY"); v != "" {
+		defaultConfig.SecretKey = v
+	}
+	if v := getenvAny("AWS_REGION", "AWS_DEFAULT_REGION"); v != "" {
+		defaultConfig.Region = v
+	}
+}
+
+// presignedHTTPClient is used for non-SDK PUTs to a presigned URL. A fixed
+// timeout keeps tests from hanging forever if the server stalls.
+var presignedHTTPClient = &http.Client{Timeout: 30 * time.Second}
+
+func getS3Client(t *testing.T) *s3.Client {
+	cfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithRegion(defaultConfig.Region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+			defaultConfig.AccessKey, defaultConfig.SecretKey, "")),
+		config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
+			func(service, region string, _ ...interface{}) (aws.Endpoint, error) {
+				return aws.Endpoint{
+					URL:               defaultConfig.Endpoint,
+					SigningRegion:     defaultConfig.Region,
+					HostnameImmutable: true,
+				}, nil
+			})),
+	)
+	require.NoError(t, err)
+	return s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true })
+}
+
+func uniqueBucket() string {
+	return fmt.Sprintf("%s%d", defaultConfig.BucketPrefix, time.Now().UnixNano())
+}
+
+func createBucket(t *testing.T, client *s3.Client, name string) {
+	_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{Bucket: aws.String(name)})
+	require.NoError(t, err)
+}
+
+func cleanupBucket(t *testing.T, client *s3.Client, name string) {
+	t.Helper()
+	objs, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{Bucket: aws.String(name)})
+	if err == nil {
+		for _, o := range objs.Contents {
+			_, _ = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+				Bucket: aws.String(name), Key: o.Key,
+			})
+		}
+	}
+	_, _ = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{Bucket: aws.String(name)})
+}
+
+// uploadViaPresignedURL PUTs body to the given presigned URL using a plain
+// http.Client, mirroring what a browser / curl / non-SDK client would do.
+// Crucially this path does NOT run any AWS SDK middleware, so the server must
+// compute and store the checksum based on the algorithm parameter that was
+// hoisted into the query string by the presigner.
+//
+// extraHeaders are additional HTTP headers the presigner signed (e.g.
+// Content-MD5). Their values must exactly match what was signed or SigV4
+// verification will return SignatureDoesNotMatch.
+func uploadViaPresignedURL(t *testing.T, url string, body []byte, extraHeaders map[string]string) {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(body))
+	require.NoError(t, err)
+	req.ContentLength = int64(len(body))
+	for k, v := range extraHeaders {
+		req.Header.Set(k, v)
+	}
+	resp, err := presignedHTTPClient.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	respBody, err := io.ReadAll(resp.Body)
+	require.NoError(t, err, "read presigned PUT response body")
+	require.Equalf(t, http.StatusOK, resp.StatusCode,
+		"presigned PUT failed: %d %s", resp.StatusCode, string(respBody))
+}
+
+// presignPutURL builds a presigned S3 PutObject URL using the low-level
+// SigV4 presigner, with an optional x-amz-sdk-checksum-algorithm query
+// parameter baked into the signed canonical query string.
+//
+// We use the raw signer instead of s3.PresignClient because the SDK's
+// flexible-checksum middleware tries to inject a Content-MD5 header for
+// flexible-checksum PutObject calls, and at presign time (no body) it seeds
+// MD5-of-empty, which then mismatches any real body uploaded through a plain
+// http.Client. The raw signer has no such middleware and produces exactly
+// the kind of URL a browser, curl caller, or custom client would receive.
+func presignPutURL(t *testing.T, bucket, key, checksumAlgorithm string) string {
+	t.Helper()
+	putURL, err := url.Parse(fmt.Sprintf("%s/%s/%s", strings.TrimRight(defaultConfig.Endpoint, "/"), bucket, key))
+	require.NoError(t, err)
+	q := putURL.Query()
+	// PresignHTTP does not add X-Amz-Expires on its own; the caller must
+	// seed it so the signer picks it up into the canonical query string.
+	q.Set("X-Amz-Expires", "600")
+	if checksumAlgorithm != "" {
+		q.Set("x-amz-sdk-checksum-algorithm", checksumAlgorithm)
+	}
+	putURL.RawQuery = q.Encode()
+
+	req, err := http.NewRequest(http.MethodPut, putURL.String(), nil)
+	require.NoError(t, err)
+
+	signer := v4.NewSigner()
+	creds := aws.Credentials{
+		AccessKeyID:     defaultConfig.AccessKey,
+		SecretAccessKey: defaultConfig.SecretKey,
+	}
+	// For presigned URLs the AWS convention is UNSIGNED-PAYLOAD so the
+	// signer doesn't require a body-hash up front.
+	signedURL, _, err := signer.PresignHTTP(context.TODO(), creds, req,
+		"UNSIGNED-PAYLOAD", "s3", defaultConfig.Region, time.Now(),
+		func(o *v4.SignerOptions) {
+			o.DisableURIPathEscaping = true
+		})
+	require.NoError(t, err)
+	return signedURL
+}
+
+// TestPresignedPutWithChecksumSHA256 reproduces issue #9075: a presigned PUT
+// URL that carries x-amz-sdk-checksum-algorithm=SHA256 in its query string
+// must cause the object to be stored with an x-amz-checksum-sha256 attribute,
+// visible via HEAD when the caller requests ChecksumMode=ENABLED.
+func TestPresignedPutWithChecksumSHA256(t *testing.T) {
+	client := getS3Client(t)
+	bucket := uniqueBucket()
+	createBucket(t, client, bucket)
+	defer cleanupBucket(t, client, bucket)
+
+	key := "presigned-sha256.txt"
+	body := []byte("hello seaweedfs checksum")
+	sha256Sum := sha256.Sum256(body)
+	expected := base64.StdEncoding.EncodeToString(sha256Sum[:])
+
+	signedURL := presignPutURL(t, bucket, key, "SHA256")
+	uploadViaPresignedURL(t, signedURL, body, nil)
+
+	head, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
+		Bucket:       aws.String(bucket),
+		Key:          aws.String(key),
+		ChecksumMode: types.ChecksumModeEnabled,
+	})
+	require.NoError(t, err)
+	require.NotNil(t, head.ChecksumSHA256, "x-amz-checksum-sha256 missing from HEAD response")
+	assert.Equal(t, expected, aws.ToString(head.ChecksumSHA256),
+		"stored SHA256 does not match body")
+
+	// GET with ChecksumMode=ENABLED should also return the header and the body should match.
+	getOut, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
+		Bucket:       aws.String(bucket),
+		Key:          aws.String(key),
+		ChecksumMode: types.ChecksumModeEnabled,
+	})
+	require.NoError(t, err)
+	defer getOut.Body.Close()
+	require.NotNil(t, getOut.ChecksumSHA256, "x-amz-checksum-sha256 missing from GET response")
+	assert.Equal(t, expected, aws.ToString(getOut.ChecksumSHA256))
+	got, err := io.ReadAll(getOut.Body)
+	require.NoError(t, err)
+	assert.Equal(t, body, got)
+}
+
+// TestPresignedPutWithoutChecksumAlgorithm is a negative control: when the
+// caller doesn't request a checksum algorithm, HEAD should not return one.
+func TestPresignedPutWithoutChecksumAlgorithm(t *testing.T) {
+	client := getS3Client(t)
+	bucket := uniqueBucket()
+	createBucket(t, client, bucket)
+	defer cleanupBucket(t, client, bucket)
+
+	key := "presigned-nosum.txt"
+	body := []byte("no checksum requested")
+
+	signedURL := presignPutURL(t, bucket, key, "")
+	uploadViaPresignedURL(t, signedURL, body, nil)
+
+	head, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
+		Bucket:       aws.String(bucket),
+		Key:          aws.String(key),
+		ChecksumMode: types.ChecksumModeEnabled,
+	})
+	require.NoError(t, err)
+	assert.Nil(t, head.ChecksumSHA256)
+	assert.Nil(t, head.ChecksumSHA1)
+	assert.Nil(t, head.ChecksumCRC32)
+	assert.Nil(t, head.ChecksumCRC32C)
+}

test/s3/compatibility/.gitignore

@@ -0,0 +1,8 @@
+# Test run scratch data
+/s3-tests
+/tmp
+
+# Test run outputs
+weed.log
+compat.raw.txt
+compat.summary.txt

test/s3/compatibility/Dockerfile

@@ -0,0 +1,23 @@
+# the tests only support python 3.6, not newer
+#FROM ubuntu:latest
+FROM python:3.6.15-slim-bullseye
+
+# Installed required system deps
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y -qq git-core sudo tzdata ca-certificates libxml2-dev libxslt1-dev zlib1g-dev python3-lxml gcc python3-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install python deps
+RUN pip install virtualenv
+
+# Clone Ceph S3 tests
+COPY fix_s3_tests_bucket_conflicts.py /tmp/fix_s3_tests_bucket_conflicts.py
+ENV S3_TESTS_PATH=s3-tests
+RUN git clone https://github.com/ceph/s3-tests.git \
+    && python3 /tmp/fix_s3_tests_bucket_conflicts.py
+
+WORKDIR s3-tests
+
+RUN pip install -r requirements.txt \
+    && pip install tox \
+    && pip install -e .

test/s3/compatibility/Makefile

@@ -0,0 +1,9 @@
+.PHONY: all build-image test
+
+all: build-image test
+
+build-image:
+	./prepare.sh
+
+test:
+	./run.sh

test/s3/compatibility/README.md

@@ -0,0 +1,245 @@
+# Running S3 Compatibility tests against SeaweedFS
+
+This is using [the tests from CephFS][s3-tests].
+
+[s3-tests]: https://github.com/ceph/s3-tests
+
+## Prerequisites
+
+- have [Docker][docker] installed
+- this has been executed on Mac. On Linux, the hostname in `s3tests.conf`  needs to be adjusted.
+
+[docker]: https://docs.docker.com
+
+## Running tests
+
+To build the docker image that is used for the tests:
+
+```console
+./prepare.sh
+```
+
+To execute all tests:
+
+```console
+./run.sh
+```
+To see debug output including all commands run by the script:
+
+```console
+DEBUG=y ./run.sh
+```
+
+> [!WARNING]
+>
+> If your output does *not* look like the content in [`results.summary.txt`](./results.summary.txt)
+> and it is full of HTTP level exceptions, there is likely an error contacting the `weed` server from
+> the container that is running the S3 compatibility tests.
+>
+> There are at least a couple ways to solve this:
+>
+> - Modify your `docker` setup to ensure `host.docker.internal` is connected to your host running `weed`
+> - Use `--net=host` and modify `host` in `s3tests.conf` to `localhost`
+>
+> The `--net=host` solution is potentially *unsafe*, as the container running [s3-tests][s3-tests] could
+> visit unexpected websites or use the host-passthrough internet access maliciously.
+>
+> If you are OK with the risk of allowing `--net=host`:
+>
+> - Set `host = localhost` in `s3tests.conf`
+> - Set `DOCKER_NET_HOST=y` when running `run.sh`
+
+## Most recent results
+
+See [`results.summary.txt`](./results.summary.txt) for the latest results of compatibility testing (with the caveat that `s3-tests` is pinned to [`ceph/s3-tests` @ 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e](https://github.com/ceph/s3-tests/commit/9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e)).
+
+The file is reproduced below for ease of access:
+
+```
+/s3-tests/virtualenv/lib/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/
+  warnings.warn(warning, PythonDeprecationWarning)
+s3tests_boto3.functional.test_s3.test_bucket_list_return_data ... ERROR
+s3tests_boto3.functional.test_s3.test_object_write_to_nonexist_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_read_not_exist ... ok
+s3tests_boto3.functional.test_s3.test_object_requestid_matches_header_on_error ... FAIL
+s3tests_boto3.functional.test_s3.test_multi_object_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_object_head_zero_bytes ... ok
+s3tests_boto3.functional.test_s3.test_object_write_check_etag ... ok
+s3tests_boto3.functional.test_s3.test_object_write_cache_control ... ok
+s3tests_boto3.functional.test_s3.test_object_write_expires ... ok
+s3tests_boto3.functional.test_s3.test_object_write_read_update_read_delete ... ok
+s3tests_boto3.functional.test_s3.test_object_metadata_replaced_on_put ... ok
+s3tests_boto3.functional.test_s3.test_object_write_file ... ok
+s3tests_boto3.functional.test_s3.test_post_object_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_no_content_type ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_invalid_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_upload_larger_than_chunk ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_key_from_filename ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_ignored_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_case_insensitive_condition_fields ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_escaped_field_values ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_success_redirect_action ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_signature ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_date_format ... ok
+s3tests_boto3.functional.test_s3.test_post_object_no_key_specified ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_signature ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_user_specified_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_condition_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expires_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expired_policy ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_missing_expires_condition ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_conditions_list ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_limit_exceeded ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_invalid_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_below_minimum ... ok
+s3tests_boto3.functional.test_s3.test_post_object_empty_conditions ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_nonexisted_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_response_headers ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_not_expired ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_range_zero ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_max_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_positive_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put_write_access ... ok
+s3tests_boto3.functional.test_s3.test_object_put_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_put_authenticated_expired ... FAIL
+s3tests_boto3.functional.test_s3.test_object_acl_canned_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_write ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_writeacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_read ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_readacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_header_acl_grants ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_bucket_create_special_key_names ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_zero_size ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_same_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_verify_contenttype ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself_with_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_diff_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_object_bucket ... ERROR
+s3tests_boto3.functional.test_s3.test_object_copy_canned_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_retaining_metadata ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_replacing_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_bucket_not_found ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_key_not_found ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_empty ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_small ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_copy_small ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_improper_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_without_range ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_special_names ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_upload_resend_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_size_too_small ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_upload_contents ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_overwrite_existing_object ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload_not_found ... ok
+s3tests_boto3.functional.test_s3.test_list_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_missing_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_incorrect_etag ... ok
+s3tests_boto3.functional.test_s3.test_100_continue ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_read_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_write_bucket_gone ... ok
+s3tests_boto3.functional.test_s3.test_atomic_multipart_upload_write ... ok
+s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_big_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_skip_leading_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_return_trailing_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_invalid_range ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_empty_object ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_tagging ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_head_tagging ... ok
+s3tests_boto3.functional.test_s3.test_put_max_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_max_kvsize_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_key_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_val_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_modify_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_delete_tags ... ok
+s3tests_boto3.functional.test_s3.test_post_object_tags_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_tags_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_put_obj_with_tags ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention ... ERROR
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass ... FAIL
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_good ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_object_read_unreadable ... FAIL
+ERROR
+
+======================================================================
+ERROR: s3tests_boto3.functional.test_s3.test_bucket_list_return_data
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/s3-tests/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest
+    self.test(*self.arg)
+  File "/s3-tests/s3tests_boto3/functional/test_s3.py", line 1669, in test_bucket_list_return_data
+    'DisplayName': acl_response['Owner']['DisplayName'],
+KeyError: 'Owner'
+-------------------- >> begin captured logging << --------------------
+```

test/s3/compatibility/fix_s3_tests_bucket_conflicts.py

@@ -0,0 +1,290 @@
+#!/usr/bin/env python3
+"""
+Patch Ceph s3-tests helpers to avoid bucket name mismatches and make bucket
+creation idempotent when a fixed bucket name is provided.
+
+Why:
+- Some tests call get_new_bucket() to get a name, then call
+  get_new_bucket_resource(name=<that name>) which unconditionally calls
+  CreateBucket again. If the bucket already exists, boto3 raises a
+  ClientError. We want to treat that as idempotent and reuse the bucket.
+- We must NOT silently generate a different bucket name when a name is
+  explicitly provided, otherwise subsequent test steps still reference the
+  original string and read from the wrong (empty) bucket.
+
+What this does:
+- get_new_bucket_resource(name=...):
+  - Try to create the exact bucket name.
+  - If error code is BucketAlreadyOwnedByYou OR BucketAlreadyExists, simply
+    reuse and return the bucket object for that SAME name.
+  - Only when name is None, generate a new unique name with retries.
+- get_new_bucket(client=None, name=None):
+  - If name is None, generate unique names with retries until creation
+    succeeds, and return the actual name string to the caller.
+
+This keeps bucket names consistent across the test helper calls and prevents
+404s or KeyErrors later in the tests that depend on that bucket name.
+"""
+
+import os
+import sys
+
+
+def patch_s3_tests_init_file(file_path: str) -> bool:
+    if not os.path.exists(file_path):
+        print(f"Error: File {file_path} not found")
+        return False
+
+    print(f"Patching {file_path}...")
+    with open(file_path, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # If already patched, skip
+    if "max_retries = 10" in content and "BucketAlreadyOwnedByYou" in content and "BucketAlreadyExists" in content:
+        print("Already patched. Skipping.")
+        return True
+
+    old_resource_func = '''def get_new_bucket_resource(name=None):
+    """
+    Get a bucket that exists and is empty.
+
+    Always recreates a bucket from scratch. This is useful to also
+    reset ACLs and such.
+    """
+    s3 = boto3.resource('s3',
+                        aws_access_key_id=config.main_access_key,
+                        aws_secret_access_key=config.main_secret_key,
+                        endpoint_url=config.default_endpoint,
+                        use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify)
+    if name is None:
+        name = get_new_bucket_name()
+    bucket = s3.Bucket(name)
+    bucket_location = bucket.create()
+    return bucket'''
+
+    new_resource_func = '''def get_new_bucket_resource(name=None):
+    """
+    Get a bucket that exists and is empty.
+
+    Always recreates a bucket from scratch. This is useful to also
+    reset ACLs and such.
+    """
+    s3 = boto3.resource('s3',
+                        aws_access_key_id=config.main_access_key,
+                        aws_secret_access_key=config.main_secret_key,
+                        endpoint_url=config.default_endpoint,
+                        use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify)
+
+    from botocore.exceptions import ClientError
+
+    # If a name is provided, do not change it. Reuse that exact bucket name.
+    if name is not None:
+        bucket = s3.Bucket(name)
+        try:
+            bucket.create()
+        except ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'):
+                # Treat as idempotent create for an explicitly provided name.
+                # We must not change the name or tests will read from the wrong bucket.
+                return bucket
+            # Other errors should surface
+            raise
+        else:
+            return bucket
+
+    # Only generate unique names when no name was provided
+    max_retries = 10
+    for attempt in range(max_retries):
+        gen_name = get_new_bucket_name()
+        bucket = s3.Bucket(gen_name)
+        try:
+            bucket.create()
+            return bucket
+        except ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyExists', 'BucketAlreadyOwnedByYou'):
+                if attempt == max_retries - 1:
+                    raise Exception(f"Failed to create unique bucket after {max_retries} attempts")
+                continue
+            else:
+                raise'''
+
+    old_client_func = '''def get_new_bucket(client=None, name=None):
+    """
+    Get a bucket that exists and is empty.
+
+    Always recreates a bucket from scratch. This is useful to also
+    reset ACLs and such.
+    """
+    if client is None:
+        client = get_client()
+    if name is None:
+        name = get_new_bucket_name()
+
+    client.create_bucket(Bucket=name)
+    return name'''
+
+    new_client_func = '''def get_new_bucket(client=None, name=None):
+    """
+    Get a bucket that exists and is empty.
+
+    Always recreates a bucket from scratch. This is useful to also
+    reset ACLs and such.
+    """
+    if client is None:
+        client = get_client()
+
+    from botocore.exceptions import ClientError
+
+    # If a name is provided, just try to create it once and fall back to idempotent reuse
+    if name is not None:
+        try:
+            client.create_bucket(Bucket=name)
+        except ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'):
+                return name
+            raise
+        else:
+            return name
+
+    # Otherwise, generate a unique name with retries and return the actual name string
+    max_retries = 10
+    for attempt in range(max_retries):
+        gen_name = get_new_bucket_name()
+        try:
+            client.create_bucket(Bucket=gen_name)
+            return gen_name
+        except ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyExists', 'BucketAlreadyOwnedByYou'):
+                if attempt == max_retries - 1:
+                    raise Exception(f"Failed to create unique bucket after {max_retries} attempts")
+                continue
+            else:
+                raise'''
+
+    updated = content
+    updated = updated.replace(old_resource_func, new_resource_func)
+    updated = updated.replace(old_client_func, new_client_func)
+
+    if updated == content:
+        print("Patterns not found; appending override implementations to end of file.")
+        append_patch = '''
+
+# --- SeaweedFS override start ---
+from botocore.exceptions import ClientError as _Sw_ClientError
+
+
+# Idempotent create for provided name; generate unique only when no name given
+# Keep the bucket name stable when provided by the caller
+
+def _sw_get_new_bucket_resource(name=None):
+    s3 = boto3.resource('s3',
+                        aws_access_key_id=config.main_access_key,
+                        aws_secret_access_key=config.main_secret_key,
+                        endpoint_url=config.default_endpoint,
+                        use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify)
+    if name is not None:
+        bucket = s3.Bucket(name)
+        try:
+            bucket.create()
+        except _Sw_ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'):
+                return bucket
+            raise
+        else:
+            return bucket
+    # name not provided: generate unique
+    max_retries = 10
+    for attempt in range(max_retries):
+        gen_name = get_new_bucket_name()
+        bucket = s3.Bucket(gen_name)
+        try:
+            bucket.create()
+            return bucket
+        except _Sw_ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyExists', 'BucketAlreadyOwnedByYou'):
+                if attempt == max_retries - 1:
+                    raise Exception(f"Failed to create unique bucket after {max_retries} attempts")
+                continue
+            else:
+                raise
+
+
+from botocore.exceptions import ClientError as _Sw2_ClientError
+
+
+def _sw_get_new_bucket(client=None, name=None):
+    if client is None:
+        client = get_client()
+    if name is not None:
+        try:
+            client.create_bucket(Bucket=name)
+        except _Sw2_ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'):
+                return name
+            raise
+        else:
+            return name
+    max_retries = 10
+    for attempt in range(max_retries):
+        gen_name = get_new_bucket_name()
+        try:
+            client.create_bucket(Bucket=gen_name)
+            return gen_name
+        except _Sw2_ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyExists', 'BucketAlreadyOwnedByYou'):
+                if attempt == max_retries - 1:
+                    raise Exception(f"Failed to create unique bucket after {max_retries} attempts")
+                continue
+            else:
+                raise
+
+# Override original helper functions
+get_new_bucket_resource = _sw_get_new_bucket_resource
+get_new_bucket = _sw_get_new_bucket
+# --- SeaweedFS override end ---
+'''
+        with open(file_path, "a", encoding="utf-8") as f:
+            f.write(append_patch)
+        print("Appended override implementations.")
+        return True
+
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(updated)
+
+    print("Successfully patched s3-tests helpers.")
+    return True
+
+
+def main() -> int:
+    s3_tests_path = os.environ.get("S3_TESTS_PATH", "s3-tests")
+    init_file_path = os.path.join(s3_tests_path, "s3tests", "functional", "__init__.py")
+    print("Applying s3-tests patch for bucket creation idempotency...")
+    print(f"Target repo path: {s3_tests_path}")
+    if not os.path.exists(s3_tests_path):
+        print(f"Warning: s3-tests directory not found at {s3_tests_path}")
+        print("Skipping patch - directory structure may have changed in the upstream repository")
+        return 0  # Return success to not break CI
+    if not os.path.exists(init_file_path):
+        print(f"Warning: Target file {init_file_path} not found")
+        print("This may indicate the s3-tests repository structure has changed.")
+        print("Skipping patch - tests may still work without it")
+        return 0  # Return success to not break CI
+    ok = patch_s3_tests_init_file(init_file_path)
+    return 0 if ok else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+
+

test/s3/compatibility/prepare.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -ex
+
+docker build  --progress=plain  -t s3tests .

test/s3/compatibility/results.summary.txt

@@ -0,0 +1,186 @@
+/s3-tests/virtualenv/lib/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/
+  warnings.warn(warning, PythonDeprecationWarning)
+s3tests_boto3.functional.test_s3.test_bucket_list_return_data ... ERROR
+s3tests_boto3.functional.test_s3.test_object_write_to_nonexist_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_read_not_exist ... ok
+s3tests_boto3.functional.test_s3.test_object_requestid_matches_header_on_error ... FAIL
+s3tests_boto3.functional.test_s3.test_multi_object_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_object_head_zero_bytes ... ok
+s3tests_boto3.functional.test_s3.test_object_write_check_etag ... ok
+s3tests_boto3.functional.test_s3.test_object_write_cache_control ... ok
+s3tests_boto3.functional.test_s3.test_object_write_expires ... ok
+s3tests_boto3.functional.test_s3.test_object_write_read_update_read_delete ... ok
+s3tests_boto3.functional.test_s3.test_object_metadata_replaced_on_put ... ok
+s3tests_boto3.functional.test_s3.test_object_write_file ... ok
+s3tests_boto3.functional.test_s3.test_post_object_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_no_content_type ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_invalid_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_upload_larger_than_chunk ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_key_from_filename ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_ignored_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_case_insensitive_condition_fields ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_escaped_field_values ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_success_redirect_action ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_signature ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_date_format ... ok
+s3tests_boto3.functional.test_s3.test_post_object_no_key_specified ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_signature ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_user_specified_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_condition_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expires_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expired_policy ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_missing_expires_condition ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_conditions_list ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_limit_exceeded ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_invalid_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_below_minimum ... ok
+s3tests_boto3.functional.test_s3.test_post_object_empty_conditions ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_nonexisted_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_response_headers ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_not_expired ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_range_zero ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_max_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_positive_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put_write_access ... ok
+s3tests_boto3.functional.test_s3.test_object_put_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_put_authenticated_expired ... FAIL
+s3tests_boto3.functional.test_s3.test_object_acl_canned_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_write ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_writeacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_read ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_readacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_header_acl_grants ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_bucket_create_special_key_names ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_zero_size ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_same_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_verify_contenttype ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself_with_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_diff_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_object_bucket ... ERROR
+s3tests_boto3.functional.test_s3.test_object_copy_canned_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_retaining_metadata ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_replacing_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_bucket_not_found ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_key_not_found ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_empty ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_small ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_copy_small ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_improper_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_without_range ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_special_names ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_upload_resend_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_size_too_small ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_upload_contents ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_overwrite_existing_object ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload_not_found ... ok
+s3tests_boto3.functional.test_s3.test_list_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_missing_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_incorrect_etag ... ok
+s3tests_boto3.functional.test_s3.test_100_continue ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_read_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_write_bucket_gone ... ok
+s3tests_boto3.functional.test_s3.test_atomic_multipart_upload_write ... ok
+s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_big_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_skip_leading_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_return_trailing_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_invalid_range ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_empty_object ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_tagging ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_head_tagging ... ok
+s3tests_boto3.functional.test_s3.test_put_max_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_max_kvsize_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_key_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_val_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_modify_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_delete_tags ... ok
+s3tests_boto3.functional.test_s3.test_post_object_tags_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_tags_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_put_obj_with_tags ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention ... ERROR
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass ... FAIL
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_good ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_object_read_unreadable ... FAIL
+ERROR
+
+======================================================================
+ERROR: s3tests_boto3.functional.test_s3.test_bucket_list_return_data
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/s3-tests/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest
+    self.test(*self.arg)
+  File "/s3-tests/s3tests_boto3/functional/test_s3.py", line 1669, in test_bucket_list_return_data
+    'DisplayName': acl_response['Owner']['DisplayName'],
+KeyError: 'Owner'
+-------------------- >> begin captured logging << --------------------

test/s3/compatibility/run.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+CONTAINER_NAME=${CONTAINER_NAME:-s3test-instance}
+CONF_FILE=${CONF_FILE:-s3tests.conf}
+WEED_BIN=${WEED_BIN:-../../../weed/weed}
+TEST_RAW_OUTPUT_FILE=${TEST_RAW_OUTPUT_FILE:-compat.raw.txt}
+TEST_PROCESSED_OUTPUT_FILE=${TEST_PROCESSED_OUTPUT_FILE:-compat.summary.txt}
+
+# Set up debugging for this bash script if DEBUG is set
+if [ -n "${DEBUG}" ]; then
+    echo -e "DEBUG set [${DEBUG}], enabling debugging output...";
+    set -ex
+fi
+
+# Reset from possible previous test run
+killall -9 weed || echo "already stopped"
+rm -Rf tmp
+mkdir tmp
+docker stop $CONTAINER_NAME || echo "already stopped"
+
+# Ensure ulimit is set to reasonable value
+ulimit -n 10000
+
+# Start weed w/ filer + s3 in the background
+$WEED_BIN mini \
+          -master.volumeSizeLimitMB 5 \
+          -dir "$(pwd)/tmp" \
+          1>&2>weed.log &
+
+# Wait for master to start up
+echo -e "\n[info] waiting for master @ 9333...";
+until curl --output /dev/null --silent --head --fail http://127.0.0.1:9333; do
+    printf '.';
+    sleep 5;
+done
+sleep 3;
+
+# Wait for s3 to start up
+echo -e "\n[info] waiting for S3 @ 8333...";
+until curl --output /dev/null --silent --fail http://127.0.0.1:8333; do
+    printf '.';
+    sleep 5;
+done
+sleep 3;
+
+# Determine whether docker net
+DOCKER_NET_HOST_ARGS=""
+if [ -n "${DOCKER_NET_HOST}" ]; then
+    DOCKER_NET_HOST_ARGS="--net=host"
+    echo -e "\n[info] setting docker to het nost"
+fi
+
+echo -e "\n[warn] You may have to run with UNFILTERED=y to disable output filtering, if you get the broken pipe error";
+echo -e "\n[info] running tests with unfiltered output...";
+docker run \
+       --name $CONTAINER_NAME \
+       --rm \
+       ${DOCKER_NET_HOST_ARGS} \
+       -e S3TEST_CONF=$CONF_FILE \
+       -v "$(pwd)"/$CONF_FILE:/s3-tests/s3tests.conf \
+       -it \
+       s3tests \
+       ./virtualenv/bin/nosetests \
+       s3tests_boto3/functional/test_s3.py \
+       -v \
+       -a 'resource=object,!bucket-policy,!versioning,!encryption' \
+    | tee ${TEST_RAW_OUTPUT_FILE}
+
+# If the summary logs are present, process them
+if [ -f "${TEST_RAW_OUTPUT_FILE}" ]; then
+    cat ${TEST_RAW_OUTPUT_FILE} | sed -n -e '/botocore.hooks/!p;//q' | tee ${TEST_PROCESSED_OUTPUT_FILE}
+    echo -e "\n[info] ✅ Successfully wrote processed output @ [${TEST_PROCESSED_OUTPUT_FILE}]";
+    if [ -z "${TEST_KEEP_RAW_OUTPUT}" ]; then
+        echo -e "\n[info] removing test raw output file @ [${TEST_RAW_OUTPUT_FILE}] (to disable this, set TEST_KEEP_RAW_OUTPUT=y)...";
+        rm -rf ${TEST_RAW_OUTPUT_FILE};
+    fi
+else
+    echo -e "\n[warn] failed to find raw output @ [${TEST_RAW_OUTPUT_FILE}]";
+fi
+
+echo -e "\n[info] stopping [${CONTAINER_NAME}] container...";
+docker stop $CONTAINER_NAME || echo "[info] already stopped";
+
+echo -e "\n[info] stopping seaweedfs processes (all, via kill -9)...";
+killall -9 weed;

test/s3/compatibility/s3tests.conf

@@ -0,0 +1,117 @@
+[DEFAULT]
+## this section is just used for host, port and bucket_prefix
+
+# host set for rgw in vstart.sh
+host = host.docker.internal
+
+## NOTE: if running docker --net=host (e.x. `DOCKER_NET_HOST=y ./run.sh`)
+## then use 'localhost', so the container can more easily access your *local* weed instance
+##
+## This is *unsafe* -- it is possible that this script will do malicious
+## things with host-level access. only run this if you trust the programmatic content of
+## ceph/s3-tests (https://github.com/ceph/s3-tests) which will be using this file
+#host = localhost
+
+# port set for rgw in vstart.sh
+port = 8333
+
+## say "False" to disable TLS
+is_secure = False
+
+## say "False" to disable SSL Verify
+ssl_verify = False
+
+[fixtures]
+## all the buckets created will start with this prefix;
+## {random} will be filled with random characters to pad
+## the prefix to 30 characters long, and avoid collisions
+bucket prefix = yournamehere-{random}-
+
+[s3 main]
+# main display_name set in vstart.sh
+display_name = M. Tester
+
+# main user_idname set in vstart.sh
+user_id = testid
+
+# main email set in vstart.sh
+email = tester@ceph.com
+
+# zonegroup api_name for bucket location
+api_name = default
+
+## main AWS access key
+access_key = 0555b35654ad1656d804
+
+## main AWS secret key
+secret_key = h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==
+
+## replace with key id obtained when secret is created, or delete if KMS not tested
+#kms_keyid = 01234567-89ab-cdef-0123-456789abcdef
+
+[s3 alt]
+# alt display_name set in vstart.sh
+display_name = john.doe
+## alt email set in vstart.sh
+email = john.doe@example.com
+
+# alt user_id set in vstart.sh
+user_id = 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234
+
+# alt AWS access key set in vstart.sh
+access_key = NOPQRSTUVWXYZABCDEFG
+
+# alt AWS secret key set in vstart.sh
+secret_key = nopqrstuvwxyzabcdefghijklmnabcdefghijklm
+
+[s3 tenant]
+# tenant display_name set in vstart.sh
+display_name = testx$tenanteduser
+
+# tenant user_id set in vstart.sh
+user_id = 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+
+# tenant AWS secret key set in vstart.sh
+access_key = HIJKLMNOPQRSTUVWXYZA
+
+# tenant AWS secret key set in vstart.sh
+secret_key = opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab
+
+# tenant email set in vstart.sh
+email = tenanteduser@example.com
+
+#following section needs to be added for all sts-tests
+[iam]
+#used for iam operations in sts-tests
+#email from vstart.sh
+email = s3@example.com
+
+#user_id from vstart.sh
+user_id = 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+
+#access_key from vstart.sh
+access_key = ABCDEFGHIJKLMNOPQRST
+
+#secret_key vstart.sh
+secret_key = abcdefghijklmnopqrstuvwxyzabcdefghijklmn
+
+#display_name from vstart.sh
+display_name = youruseridhere
+
+#following section needs to be added when you want to run Assume Role With Webidentity test
+[webidentity]
+#used for assume role with web identity test in sts-tests
+#all parameters will be obtained from ceph/qa/tasks/keycloak.py
+token=<access_token>
+
+aud=<obtained after introspecting token>
+
+sub=<obtained after introspecting token>
+
+azp=<obtained after introspecting token>
+
+user_token=<access token for a user, with attribute Department=[Engineering, Marketing>]
+
+thumbprint=<obtained from x509 certificate>
+
+KC_REALM=<name of the realm>

test/s3/copying/Makefile

@@ -0,0 +1,217 @@
+# Makefile for S3 Copying Tests
+# This Makefile provides targets for running comprehensive S3 copying tests
+
+# Default values
+SEAWEEDFS_BINARY ?= weed
+S3_PORT ?= 8333
+FILER_PORT ?= 8888
+VOLUME_PORT ?= 8080
+MASTER_PORT ?= 9333
+TEST_TIMEOUT ?= 10m
+BUCKET_PREFIX ?= test-copying-
+ACCESS_KEY ?= some_access_key1
+SECRET_KEY ?= some_secret_key1
+VOLUME_MAX_SIZE_MB ?= 50
+
+# Let the tests force-drop collections at the master they actually run against
+export MASTER_ENDPOINT ?= http://127.0.0.1:$(MASTER_PORT)
+
+# Test directory
+TEST_DIR := $(shell pwd)
+SEAWEEDFS_ROOT := $(shell cd ../../../ && pwd)
+
+# Colors for output
+RED := \033[0;31m
+GREEN := \033[0;32m
+YELLOW := \033[1;33m
+NC := \033[0m # No Color
+
+.PHONY: all test clean start-seaweedfs stop-seaweedfs check-binary help
+
+all: test-basic
+
+help:
+	@echo "SeaweedFS S3 Copying Tests"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  test-basic    - Run basic S3 put/get tests first"
+	@echo "  test          - Run all S3 copying tests"
+	@echo "  test-quick    - Run quick tests only"
+	@echo "  test-full     - Run full test suite including large files"
+	@echo "  start-seaweedfs - Start SeaweedFS server for testing"
+	@echo "  stop-seaweedfs - Stop SeaweedFS server"
+	@echo "  clean         - Clean up test artifacts"
+	@echo "  check-binary  - Check if SeaweedFS binary exists"
+	@echo ""
+	@echo "Configuration:"
+	@echo "  SEAWEEDFS_BINARY=$(SEAWEEDFS_BINARY)"
+	@echo "  S3_PORT=$(S3_PORT)"
+	@echo "  FILER_PORT=$(FILER_PORT)"
+	@echo "  VOLUME_PORT=$(VOLUME_PORT)"
+	@echo "  MASTER_PORT=$(MASTER_PORT)"
+	@echo "  TEST_TIMEOUT=$(TEST_TIMEOUT)"
+	@echo "  VOLUME_MAX_SIZE_MB=$(VOLUME_MAX_SIZE_MB)"
+
+check-binary:
+	@if ! command -v $(SEAWEEDFS_BINARY) > /dev/null 2>&1; then \
+		echo "$(RED)Error: SeaweedFS binary '$(SEAWEEDFS_BINARY)' not found in PATH$(NC)"; \
+		echo "Please build SeaweedFS first by running 'make' in the root directory"; \
+		exit 1; \
+	fi
+	@echo "$(GREEN)SeaweedFS binary found: $$(which $(SEAWEEDFS_BINARY))$(NC)"
+
+start-seaweedfs: check-binary
+	@echo "$(YELLOW)Starting SeaweedFS server...$(NC)"
+	@pkill -f "weed master" || true
+	@pkill -f "weed volume" || true
+	@pkill -f "weed filer" || true
+	@pkill -f "weed s3" || true
+	@pkill -f "weed mini" || true
+	@sleep 2
+	@mkdir -p /tmp/seaweedfs-test-copying
+	@echo "Starting weed mini with dir=/tmp/seaweedfs-test-copying"
+	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) nohup $(SEAWEEDFS_BINARY) mini \
+		-dir=/tmp/seaweedfs-test-copying \
+		-s3.port=$(S3_PORT) \
+		-ip=127.0.0.1 \
+		> /tmp/seaweedfs-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid
+	@echo "$(YELLOW)Waiting for S3 service to be ready...$(NC)"
+	@for i in $$(seq 1 30); do \
+		if curl -s -f http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \
+			echo "$(GREEN)S3 service is ready$(NC)"; \
+			break; \
+		fi; \
+		echo "Waiting for S3 service... ($$i/30)"; \
+		sleep 1; \
+	done
+	@sleep 2
+	@echo "$(GREEN)SeaweedFS server started successfully$(NC)"
+	@echo "Mini Log: /tmp/seaweedfs-mini.log"
+	@echo "S3: http://localhost:$(S3_PORT)"
+
+stop-seaweedfs:
+	@echo "$(YELLOW)Stopping SeaweedFS server...$(NC)"
+	@pkill -f "weed master" || true
+	@pkill -f "weed volume" || true
+	@pkill -f "weed filer" || true
+	@pkill -f "weed s3" || true
+	@pkill -f "weed mini" || true
+	@sleep 2
+	@rm -f /tmp/weed-mini.pid
+	@echo "$(GREEN)SeaweedFS server stopped$(NC)"
+
+clean:
+	@echo "$(YELLOW)Cleaning up test artifacts...$(NC)"
+	@rm -rf /tmp/seaweedfs-test-copying-*
+	@rm -f /tmp/seaweedfs-*.log
+	@rm -f /tmp/seaweedfs-s3.json
+	@rm -f /tmp/weed-mini.pid
+	@echo "$(GREEN)Cleanup completed$(NC)"
+
+test-basic: check-binary
+	@echo "$(YELLOW)Running basic S3 put/get tests...$(NC)"
+	@$(MAKE) start-seaweedfs
+	@sleep 5
+	@echo "$(GREEN)Starting basic tests...$(NC)"
+	@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestBasic" ./test/s3/copying || (echo "$(RED)Basic tests failed$(NC)" && $(MAKE) stop-seaweedfs && exit 1)
+	@$(MAKE) stop-seaweedfs
+	@echo "$(GREEN)Basic tests completed successfully!$(NC)"
+
+test: test-basic
+	@echo "$(YELLOW)Running S3 copying tests...$(NC)"
+	@$(MAKE) start-seaweedfs
+	@sleep 5
+	@echo "$(GREEN)Starting tests...$(NC)"
+	@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "Test.*" ./test/s3/copying || (echo "$(RED)Tests failed$(NC)" && $(MAKE) stop-seaweedfs && exit 1)
+	@$(MAKE) stop-seaweedfs
+	@echo "$(GREEN)All tests completed successfully!$(NC)"
+
+test-quick: check-binary
+	@echo "$(YELLOW)Running quick S3 copying tests...$(NC)"
+	@$(MAKE) start-seaweedfs
+	@sleep 5
+	@echo "$(GREEN)Starting quick tests...$(NC)"
+	@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectCopy|TestCopyObjectIf" ./test/s3/copying || (echo "$(RED)Tests failed$(NC)" && $(MAKE) stop-seaweedfs && exit 1)
+	@$(MAKE) stop-seaweedfs
+	@echo "$(GREEN)Quick tests completed successfully!$(NC)"
+
+test-full: check-binary
+	@echo "$(YELLOW)Running full S3 copying test suite...$(NC)"
+	@$(MAKE) start-seaweedfs
+	@sleep 5
+	@echo "$(GREEN)Starting full test suite...$(NC)"
+	@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=30m -run "Test.*" ./test/s3/copying || (echo "$(RED)Tests failed$(NC)" && $(MAKE) stop-seaweedfs && exit 1)
+	@$(MAKE) stop-seaweedfs
+	@echo "$(GREEN)Full test suite completed successfully!$(NC)"
+
+test-multipart: check-binary
+	@echo "$(YELLOW)Running multipart copying tests...$(NC)"
+	@$(MAKE) start-seaweedfs
+	@sleep 5
+	@echo "$(GREEN)Starting multipart tests...$(NC)"
+	@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestMultipart" ./test/s3/copying || (echo "$(RED)Tests failed$(NC)" && $(MAKE) stop-seaweedfs && exit 1)
+	@$(MAKE) stop-seaweedfs
+	@echo "$(GREEN)Multipart tests completed successfully!$(NC)"
+
+test-conditional: check-binary
+	@echo "$(YELLOW)Running conditional copying tests...$(NC)"
+	@$(MAKE) start-seaweedfs
+	@sleep 5
+	@echo "$(GREEN)Starting conditional tests...$(NC)"
+	@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestCopyObjectIf" ./test/s3/copying || (echo "$(RED)Tests failed$(NC)" && $(MAKE) stop-seaweedfs && exit 1)
+	@$(MAKE) stop-seaweedfs
+	@echo "$(GREEN)Conditional tests completed successfully!$(NC)"
+
+# Debug targets
+debug-logs:
+	@echo "$(YELLOW)=== Master Log ===$(NC)"
+	@tail -n 50 /tmp/seaweedfs-master.log || echo "No master log found"
+	@echo "$(YELLOW)=== Volume Log ===$(NC)"
+	@tail -n 50 /tmp/seaweedfs-volume.log || echo "No volume log found"
+	@echo "$(YELLOW)=== Filer Log ===$(NC)"
+	@tail -n 50 /tmp/seaweedfs-filer.log || echo "No filer log found"
+	@echo "$(YELLOW)=== S3 Log ===$(NC)"
+	@tail -n 50 /tmp/seaweedfs-s3.log || echo "No S3 log found"
+
+debug-status:
+	@echo "$(YELLOW)=== Process Status ===$(NC)"
+	@ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found"
+	@echo "$(YELLOW)=== Port Status ===$(NC)"
+	@netstat -an | grep -E "($(MASTER_PORT)|$(VOLUME_PORT)|$(FILER_PORT)|$(S3_PORT))" || echo "No ports in use"
+
+# Manual test targets for development
+manual-start: start-seaweedfs
+	@echo "$(GREEN)SeaweedFS is now running for manual testing$(NC)"
+	@echo "Run 'make manual-stop' when finished"
+
+manual-stop: stop-seaweedfs clean
+
+# CI/CD targets
+ci-test: test-quick
+
+# Benchmark targets
+benchmark: check-binary
+	@echo "$(YELLOW)Running S3 copying benchmarks...$(NC)"
+	@$(MAKE) start-seaweedfs
+	@sleep 5
+	@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=30m -bench=. -run=Benchmark ./test/s3/copying || (echo "$(RED)Benchmarks failed$(NC)" && $(MAKE) stop-seaweedfs && exit 1)
+	@$(MAKE) stop-seaweedfs
+	@echo "$(GREEN)Benchmarks completed!$(NC)"
+
+# Stress test
+stress: check-binary
+	@echo "$(YELLOW)Running S3 copying stress tests...$(NC)"
+	@$(MAKE) start-seaweedfs
+	@sleep 5
+	@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=60m -run="TestMultipartCopyMultipleSizes" -count=10 ./test/s3/copying || (echo "$(RED)Stress tests failed$(NC)" && $(MAKE) stop-seaweedfs && exit 1)
+	@$(MAKE) stop-seaweedfs
+	@echo "$(GREEN)Stress tests completed!$(NC)"
+
+# Performance test with larger files
+perf: check-binary
+	@echo "$(YELLOW)Running S3 copying performance tests...$(NC)"
+	@$(MAKE) start-seaweedfs
+	@sleep 5
+	@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=60m -run="TestMultipartCopyMultipleSizes" ./test/s3/copying || (echo "$(RED)Performance tests failed$(NC)" && $(MAKE) stop-seaweedfs && exit 1)
+	@$(MAKE) stop-seaweedfs
+	@echo "$(GREEN)Performance tests completed!$(NC)" 

test/s3/copying/README.md

@@ -0,0 +1,325 @@
+# SeaweedFS S3 Copying Tests
+
+This directory contains comprehensive Go tests for SeaweedFS S3 copying functionality, converted from the failing Python tests in the s3-tests repository.
+
+## Overview
+
+These tests verify that SeaweedFS correctly implements S3 operations, starting with basic put/get operations and progressing to advanced copy operations, including:
+- **Basic S3 Operations**: Put/Get operations, bucket management, and metadata handling
+- **Basic object copying**: within the same bucket
+- **Cross-bucket copying**: across different buckets
+- **Multipart copy operations**: for large files
+- **Conditional copy operations**: ETag-based conditional copying
+- **Metadata handling**: during copy operations
+- **ACL handling**: during copy operations
+
+## Test Coverage
+
+### Basic S3 Operations (Run First)
+- **TestBasicPutGet**: Tests fundamental S3 put/get operations with various object types
+- **TestBasicBucketOperations**: Tests bucket creation, listing, and deletion
+- **TestBasicLargeObject**: Tests handling of larger objects (up to 10MB)
+
+### Basic Copy Operations
+- **TestObjectCopySameBucket**: Tests copying objects within the same bucket
+- **TestObjectCopyDiffBucket**: Tests copying objects to different buckets
+- **TestObjectCopyCannedAcl**: Tests copying with ACL settings
+- **TestObjectCopyRetainingMetadata**: Tests metadata preservation during copy
+
+### Multipart Copy Operations
+- **TestMultipartCopySmall**: Tests multipart copying of small files
+- **TestMultipartCopyWithoutRange**: Tests multipart copying without range specification
+- **TestMultipartCopySpecialNames**: Tests multipart copying with special character names
+- **TestMultipartCopyMultipleSizes**: Tests multipart copying with various file sizes
+
+### Conditional Copy Operations
+- **TestCopyObjectIfMatchGood**: Tests copying with matching ETag condition
+- **TestCopyObjectIfMatchFailed**: Tests copying with non-matching ETag condition (should fail)
+- **TestCopyObjectIfNoneMatchFailed**: Tests copying with non-matching ETag condition (should succeed)
+- **TestCopyObjectIfNoneMatchGood**: Tests copying with matching ETag condition (should fail)
+
+## Requirements
+
+1. **Go 1.19+**: Required for AWS SDK v2 and modern Go features
+2. **SeaweedFS Binary**: Built from source (`../../../weed/weed`)
+3. **Free Ports**: 8333 (S3), 8888 (Filer), 8080 (Volume), 9333 (Master)
+4. **Dependencies**: Uses the main repository's go.mod with existing AWS SDK v2 and testify dependencies
+
+## Quick Start
+
+### 1. Build SeaweedFS
+```bash
+cd ../../../
+make
+```
+
+### 2. Run Tests
+```bash
+# Run basic S3 operations first (recommended)
+make test-basic
+
+# Run all tests (starts with basic, then copy tests)
+make test
+
+# Run quick tests only
+make test-quick
+
+# Run multipart tests only
+make test-multipart
+
+# Run conditional tests only
+make test-conditional
+```
+
+## Available Make Targets
+
+### Basic Test Execution
+- `make test-basic` - Run basic S3 put/get operations (recommended first)
+- `make test` - Run all S3 tests (starts with basic, then copying)
+- `make test-quick` - Run quick tests only (basic copying)
+- `make test-full` - Run full test suite including large files
+- `make test-multipart` - Run multipart copying tests only
+- `make test-conditional` - Run conditional copying tests only
+
+### Server Management
+- `make start-seaweedfs` - Start SeaweedFS server for testing
+- `make stop-seaweedfs` - Stop SeaweedFS server
+- `make manual-start` - Start server for manual testing
+- `make manual-stop` - Stop server and clean up
+
+### Debugging
+- `make debug-logs` - Show recent log entries from all services
+- `make debug-status` - Show process and port status
+- `make check-binary` - Verify SeaweedFS binary exists
+
+### Performance Testing
+- `make benchmark` - Run performance benchmarks
+- `make stress` - Run stress tests with multiple iterations
+- `make perf` - Run performance tests with large files
+
+### Cleanup
+- `make clean` - Clean up test artifacts and temporary files
+
+## Configuration
+
+The tests use the following default configuration:
+
+```json
+{
+  "endpoint": "http://localhost:8333",
+  "access_key": "some_access_key1",
+  "secret_key": "some_secret_key1",
+  "region": "us-east-1",
+  "bucket_prefix": "test-copying-",
+  "use_ssl": false,
+  "skip_verify_ssl": true
+}
+```
+
+You can modify these values in `test_config.json` or by setting environment variables:
+
+```bash
+export SEAWEEDFS_BINARY=/path/to/weed
+export S3_PORT=8333
+export FILER_PORT=8888
+export VOLUME_PORT=8080
+export MASTER_PORT=9333
+export TEST_TIMEOUT=10m
+export VOLUME_MAX_SIZE_MB=50
+```
+
+**Note**: The volume size limit is set to 50MB to ensure proper testing of volume boundaries and multipart operations.
+
+## Test Details
+
+### TestBasicPutGet
+- Tests fundamental S3 put/get operations with various object types:
+  - Simple text objects
+  - Empty objects
+  - Binary objects (1KB random data)
+  - Objects with metadata and content-type
+- Verifies ETag consistency between put and get operations
+- Tests metadata preservation
+
+### TestBasicBucketOperations
+- Tests bucket creation and existence verification
+- Tests object listing in buckets
+- Tests object creation and listing with directory-like prefixes
+- Tests bucket deletion and cleanup
+- Verifies proper error handling for operations on non-existent buckets
+
+### TestBasicLargeObject
+- Tests handling of progressively larger objects:
+  - 1KB, 10KB, 100KB, 1MB, 5MB, 10MB
+- Verifies data integrity for large objects
+- Tests memory handling and streaming for large files
+- Ensures proper handling up to the 50MB volume limit
+
+### TestObjectCopySameBucket
+- Creates a bucket with a source object
+- Copies the object to a different key within the same bucket
+- Verifies the copied object has the same content
+
+### TestObjectCopyDiffBucket
+- Creates source and destination buckets
+- Copies an object from source to destination bucket
+- Verifies the copied object has the same content
+
+### TestObjectCopyCannedAcl
+- Tests copying with ACL settings (`public-read`)
+- Tests metadata replacement during copy with ACL
+- Verifies both basic copying and metadata handling
+
+### TestObjectCopyRetainingMetadata
+- Tests with different file sizes (3 bytes, 1MB)
+- Verifies metadata and content-type preservation
+- Checks that all metadata is correctly copied
+
+### TestMultipartCopySmall
+- Tests multipart copy with 1-byte files
+- Uses range-based copying (`bytes=0-0`)
+- Verifies multipart upload completion
+
+### TestMultipartCopyWithoutRange
+- Tests multipart copy without specifying range
+- Should copy entire source object
+- Verifies correct content length and data
+
+### TestMultipartCopySpecialNames
+- Tests with special character names: `" "`, `"_"`, `"__"`, `"?versionId"`
+- Verifies proper URL encoding and handling
+- Each special name is tested in isolation
+
+### TestMultipartCopyMultipleSizes
+- Tests with various copy sizes:
+  - 5MB (single part)
+  - 5MB + 100KB (multi-part)
+  - 5MB + 600KB (multi-part)
+  - 10MB + 100KB (multi-part)
+  - 10MB + 600KB (multi-part)
+  - 10MB (exact multi-part boundary)
+- Uses 5MB part size for all copies
+- Verifies data integrity across all sizes
+
+### TestCopyObjectIfMatchGood
+- Tests conditional copy with matching ETag
+- Should succeed when ETag matches
+- Verifies successful copy operation
+
+### TestCopyObjectIfMatchFailed
+- Tests conditional copy with non-matching ETag
+- Should fail with precondition error
+- Verifies proper error handling
+
+### TestCopyObjectIfNoneMatchFailed
+- Tests conditional copy with non-matching ETag for IfNoneMatch
+- Should succeed when ETag doesn't match
+- Verifies successful copy operation
+
+### TestCopyObjectIfNoneMatchGood
+- Tests conditional copy with matching ETag for IfNoneMatch
+- Should fail with precondition error
+- Verifies proper error handling
+
+## Expected Behavior
+
+These tests verify that SeaweedFS correctly implements:
+
+1. **Basic S3 Operations**: Standard `PutObject`, `GetObject`, `ListBuckets`, `ListObjects` APIs
+2. **Bucket Management**: Bucket creation, deletion, and listing
+3. **Object Storage**: Binary and text data storage with metadata
+4. **Large Object Handling**: Efficient storage and retrieval of large files
+5. **Basic S3 Copy Operations**: Standard `CopyObject` API
+6. **Multipart Copy Operations**: `UploadPartCopy` API with range support
+7. **Conditional Operations**: ETag-based conditional copying
+8. **Metadata Handling**: Proper metadata preservation and replacement
+9. **ACL Handling**: Access control list management during copy
+10. **Error Handling**: Proper error responses for invalid operations
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Port Already in Use**
+   ```bash
+   make stop-seaweedfs
+   make clean
+   ```
+
+2. **SeaweedFS Binary Not Found**
+   ```bash
+   cd ../../../
+   make
+   ```
+
+3. **Test Timeouts**
+   ```bash
+   export TEST_TIMEOUT=30m
+   make test
+   ```
+
+4. **Permission Denied**
+   ```bash
+   sudo make clean
+   ```
+
+### Debug Information
+
+```bash
+# Check server status
+make debug-status
+
+# View recent logs
+make debug-logs
+
+# Manual server start for investigation
+make manual-start
+# ... perform manual testing ...
+make manual-stop
+```
+
+### Log Locations
+
+When running tests, logs are stored in:
+- Master: `/tmp/seaweedfs-master.log`
+- Volume: `/tmp/seaweedfs-volume.log`
+- Filer: `/tmp/seaweedfs-filer.log`
+- S3: `/tmp/seaweedfs-s3.log`
+
+## Contributing
+
+When adding new tests:
+
+1. Follow the existing naming convention (`TestXxxYyy`)
+2. Use the helper functions for common operations
+3. Add cleanup with `defer deleteBucket(t, client, bucketName)`
+4. Include error checking with `require.NoError(t, err)`
+5. Use assertions with `assert.Equal(t, expected, actual)`
+6. Add the test to the appropriate Make target
+
+## Performance Notes
+
+- **TestMultipartCopyMultipleSizes** is the most resource-intensive test
+- Large file tests may take several minutes to complete
+- Memory usage scales with file sizes being tested
+- Network latency affects multipart copy performance
+
+## Integration with CI/CD
+
+For automated testing:
+
+```bash
+# Basic validation (recommended first)
+make test-basic
+
+# Quick validation
+make ci-test
+
+# Full validation
+make test-full
+
+# Performance validation
+make perf
+```
+
+The tests are designed to be self-contained and can run in containerized environments. 

test/s3/copying/s3_rename_test.go

@@ -0,0 +1,333 @@
+package copying_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// createRenameSource builds the x-amz-rename-source value the way the AWS CLI,
+// Java and Rust examples do: the bare source key, URL encoded.
+func createRenameSource(key string) string {
+	return url.PathEscape(key)
+}
+
+// createQualifiedRenameSource builds the alternate bucket/key form the second
+// AWS CLI example and the boto3 conditional example use.
+func createQualifiedRenameSource(bucketName, key string) string {
+	return fmt.Sprintf("%s/%s", bucketName, url.PathEscape(key))
+}
+
+func renameObject(t *testing.T, client *s3.Client, bucketName, srcKey, dstKey string) {
+	t.Helper()
+	_, err := client.RenameObject(context.TODO(), &s3.RenameObjectInput{
+		Bucket:       aws.String(bucketName),
+		Key:          aws.String(dstKey),
+		RenameSource: aws.String(createRenameSource(srcKey)),
+	})
+	require.NoError(t, err)
+}
+
+// requireRenameStatus asserts err carries the given HTTP status code.
+func requireRenameStatus(t *testing.T, err error, status int) {
+	t.Helper()
+	require.Error(t, err)
+	var respErr *smithyhttp.ResponseError
+	require.True(t, errors.As(err, &respErr), "expected an HTTP response error, got %v", err)
+	assert.Equal(t, status, respErr.HTTPStatusCode(), "unexpected error: %v", err)
+}
+
+func objectExists(t *testing.T, client *s3.Client, bucketName, key string) bool {
+	t.Helper()
+	_, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(key),
+	})
+	return err == nil
+}
+
+// TestRenameObject renames an object and checks the bytes, metadata and ETag
+// arrive under the new key while the old key disappears.
+func TestRenameObject(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	content := "rename me"
+	put := putObjectWithMetadata(t, client, bucketName, "source.txt", content,
+		map[string]string{"origin": "source"}, "text/plain")
+
+	renameObject(t, client, bucketName, "source.txt", "renamed/target.txt")
+
+	assert.False(t, objectExists(t, client, bucketName, "source.txt"), "source should be gone")
+
+	resp := getObject(t, client, bucketName, "renamed/target.txt")
+	assert.Equal(t, content, getObjectBody(t, resp))
+	assert.Equal(t, "text/plain", aws.ToString(resp.ContentType))
+	assert.Equal(t, "source", resp.Metadata["origin"])
+	assert.Equal(t, aws.ToString(put.ETag), aws.ToString(resp.ETag), "ETag must survive the rename")
+}
+
+// TestRenameObjectOverwritesDestination: without a conditional header a rename
+// replaces whatever the destination key held.
+func TestRenameObjectOverwritesDestination(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	putObject(t, client, bucketName, "source.txt", "new content")
+	putObject(t, client, bucketName, "target.txt", "old content")
+
+	renameObject(t, client, bucketName, "source.txt", "target.txt")
+
+	resp := getObject(t, client, bucketName, "target.txt")
+	assert.Equal(t, "new content", getObjectBody(t, resp))
+}
+
+// TestRenameObjectIfNoneMatch: If-None-Match: * protects an existing destination.
+func TestRenameObjectIfNoneMatch(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	putObject(t, client, bucketName, "source.txt", "new content")
+	putObject(t, client, bucketName, "target.txt", "old content")
+
+	_, err := client.RenameObject(context.TODO(), &s3.RenameObjectInput{
+		Bucket:                 aws.String(bucketName),
+		Key:                    aws.String("target.txt"),
+		RenameSource:           aws.String(createRenameSource("source.txt")),
+		DestinationIfNoneMatch: aws.String("*"),
+	})
+	requireRenameStatus(t, err, 412)
+
+	resp := getObject(t, client, bucketName, "target.txt")
+	assert.Equal(t, "old content", getObjectBody(t, resp))
+	assert.True(t, objectExists(t, client, bucketName, "source.txt"), "a failed rename must leave the source alone")
+
+	// The same rename onto a free key succeeds.
+	_, err = client.RenameObject(context.TODO(), &s3.RenameObjectInput{
+		Bucket:                 aws.String(bucketName),
+		Key:                    aws.String("free.txt"),
+		RenameSource:           aws.String(createRenameSource("source.txt")),
+		DestinationIfNoneMatch: aws.String("*"),
+	})
+	require.NoError(t, err)
+}
+
+// TestRenameObjectSourceIfMatch gates the rename on the source's ETag.
+func TestRenameObjectSourceIfMatch(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	put := putObject(t, client, bucketName, "source.txt", "content")
+
+	_, err := client.RenameObject(context.TODO(), &s3.RenameObjectInput{
+		Bucket:        aws.String(bucketName),
+		Key:           aws.String("target.txt"),
+		RenameSource:  aws.String(createRenameSource("source.txt")),
+		SourceIfMatch: aws.String("\"00000000000000000000000000000000\""),
+	})
+	requireRenameStatus(t, err, 412)
+	assert.True(t, objectExists(t, client, bucketName, "source.txt"))
+
+	_, err = client.RenameObject(context.TODO(), &s3.RenameObjectInput{
+		Bucket:        aws.String(bucketName),
+		Key:           aws.String("target.txt"),
+		RenameSource:  aws.String(createRenameSource("source.txt")),
+		SourceIfMatch: put.ETag,
+	})
+	require.NoError(t, err)
+	assert.False(t, objectExists(t, client, bucketName, "source.txt"))
+	assert.True(t, objectExists(t, client, bucketName, "target.txt"))
+}
+
+// TestRenameObjectOntoDirectory: S3 keys are flat, so a key that other keys are
+// nested under is still a key of its own. The rename writes it without disturbing
+// them - it does not replace the directory, it stores the object on it.
+func TestRenameObjectOntoDirectory(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	putObject(t, client, bucketName, "source.txt", "content")
+	putObject(t, client, bucketName, "target/child.txt", "child")
+
+	_, err := client.RenameObject(context.TODO(), &s3.RenameObjectInput{
+		Bucket:       aws.String(bucketName),
+		Key:          aws.String("target"),
+		RenameSource: aws.String(createRenameSource("source.txt")),
+	})
+	require.NoError(t, err)
+	assert.False(t, objectExists(t, client, bucketName, "source.txt"))
+	assert.Equal(t, "content", getObjectBody(t, getObject(t, client, bucketName, "target")))
+	assert.Equal(t, "child", getObjectBody(t, getObject(t, client, bucketName, "target/child.txt")))
+}
+
+// TestRenameObjectDirectorySource: a directory can be named without a trailing
+// slash, and renaming one would move a whole subtree. It is not an object, so it
+// is a missing key.
+func TestRenameObjectDirectorySource(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	putObject(t, client, bucketName, "source/child.txt", "child")
+
+	for _, src := range []string{"source", "source/"} {
+		_, err := client.RenameObject(context.TODO(), &s3.RenameObjectInput{
+			Bucket:       aws.String(bucketName),
+			Key:          aws.String("target.txt"),
+			RenameSource: aws.String(createRenameSource(src)),
+		})
+		requireRenameStatus(t, err, 404)
+	}
+	assert.True(t, objectExists(t, client, bucketName, "source/child.txt"))
+	assert.False(t, objectExists(t, client, bucketName, "target.txt"))
+}
+
+// TestRenameObjectMissingSource reports a missing source as NoSuchKey.
+func TestRenameObjectMissingSource(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	_, err := client.RenameObject(context.TODO(), &s3.RenameObjectInput{
+		Bucket:       aws.String(bucketName),
+		Key:          aws.String("target.txt"),
+		RenameSource: aws.String(createRenameSource("absent.txt")),
+	})
+	requireRenameStatus(t, err, 404)
+}
+
+// TestRenameObjectQualifiedSource: the bucket/key form AWS's second CLI example
+// and the boto3 conditional example use resolves to the same object as the bare
+// key.
+func TestRenameObjectQualifiedSource(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	putObject(t, client, bucketName, "dir/source.txt", "content")
+
+	_, err := client.RenameObject(context.TODO(), &s3.RenameObjectInput{
+		Bucket:       aws.String(bucketName),
+		Key:          aws.String("target.txt"),
+		RenameSource: aws.String(createQualifiedRenameSource(bucketName, "dir/source.txt")),
+	})
+	require.NoError(t, err)
+	assert.False(t, objectExists(t, client, bucketName, "dir/source.txt"))
+	assert.Equal(t, "content", getObjectBody(t, getObject(t, client, bucketName, "target.txt")))
+}
+
+// TestRenameObjectSourceShadowingTheBucketName: a key whose own first segment is
+// the bucket name is a real key, and must win over reading the same value as a
+// bucket-qualified source.
+func TestRenameObjectSourceShadowingTheBucketName(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	shadowed := bucketName + "/source.txt"
+	putObject(t, client, bucketName, shadowed, "shadowed")
+	putObject(t, client, bucketName, "source.txt", "bare")
+
+	_, err := client.RenameObject(context.TODO(), &s3.RenameObjectInput{
+		Bucket:       aws.String(bucketName),
+		Key:          aws.String("target.txt"),
+		RenameSource: aws.String(createQualifiedRenameSource(bucketName, "source.txt")),
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "shadowed", getObjectBody(t, getObject(t, client, bucketName, "target.txt")))
+	assert.False(t, objectExists(t, client, bucketName, shadowed))
+	assert.True(t, objectExists(t, client, bucketName, "source.txt"), "the bare key must be left alone")
+}
+
+// TestRenameObjectSourceShadowedByADirectory: the literal reading of the source
+// names a directory here, and the bucket-qualified reading names a live object.
+// Naming a directory is an error about that directory — falling through to the
+// other reading would rename a different object than the one asked for.
+func TestRenameObjectSourceShadowedByADirectory(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	putObject(t, client, bucketName, bucketName+"/source.txt/child.txt", "child")
+	putObject(t, client, bucketName, "source.txt", "bare")
+
+	_, err := client.RenameObject(context.TODO(), &s3.RenameObjectInput{
+		Bucket:       aws.String(bucketName),
+		Key:          aws.String("target.txt"),
+		RenameSource: aws.String(createQualifiedRenameSource(bucketName, "source.txt")),
+	})
+	requireRenameStatus(t, err, 404)
+	assert.True(t, objectExists(t, client, bucketName, "source.txt"), "the bare key must be left alone")
+	assert.False(t, objectExists(t, client, bucketName, "target.txt"))
+}
+
+// TestRenameObjectCrossBucket: RenameObject moves within one bucket, so another
+// bucket's name in the source is just part of a key this bucket does not hold.
+func TestRenameObjectCrossBucket(t *testing.T) {
+	client := getS3Client(t)
+	srcBucket := getNewBucketName()
+	createBucket(t, client, srcBucket)
+	defer deleteBucket(t, client, srcBucket)
+	dstBucket := getNewBucketName()
+	createBucket(t, client, dstBucket)
+	defer deleteBucket(t, client, dstBucket)
+
+	putObject(t, client, srcBucket, "source.txt", "content")
+
+	_, err := client.RenameObject(context.TODO(), &s3.RenameObjectInput{
+		Bucket:       aws.String(dstBucket),
+		Key:          aws.String("target.txt"),
+		RenameSource: aws.String(createQualifiedRenameSource(srcBucket, "source.txt")),
+	})
+	requireRenameStatus(t, err, 404)
+	assert.True(t, objectExists(t, client, srcBucket, "source.txt"))
+}
+
+// TestRenameObjectVersionedBucket: versioned buckets are not supported yet, and
+// must say so rather than silently dropping versions.
+func TestRenameObjectVersionedBucket(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := getNewBucketName()
+	createBucket(t, client, bucketName)
+	defer deleteBucket(t, client, bucketName)
+
+	_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket:                  aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{Status: types.BucketVersioningStatusEnabled},
+	})
+	require.NoError(t, err)
+
+	putObject(t, client, bucketName, "source.txt", "content")
+
+	_, err = client.RenameObject(context.TODO(), &s3.RenameObjectInput{
+		Bucket:       aws.String(bucketName),
+		Key:          aws.String("target.txt"),
+		RenameSource: aws.String(createRenameSource("source.txt")),
+	})
+	requireRenameStatus(t, err, 501)
+	assert.True(t, objectExists(t, client, bucketName, "source.txt"))
+}

test/s3/copying/test_config.json

@@ -0,0 +1,9 @@
+{
+  "endpoint": "http://localhost:8333",
+  "access_key": "some_access_key1",
+  "secret_key": "some_secret_key1",
+  "region": "us-east-1",
+  "bucket_prefix": "test-copying-",
+  "use_ssl": false,
+  "skip_verify_ssl": true
+} 

test/s3/cors/Makefile

@@ -0,0 +1,343 @@
+# CORS Integration Tests Makefile
+# This Makefile provides comprehensive targets for running CORS integration tests
+
+.PHONY: help build-weed setup-server start-server stop-server test-cors test-cors-quick test-cors-comprehensive test-all clean logs check-deps
+
+# Configuration
+WEED_BINARY := ../../../weed/weed_binary
+S3_PORT := 8333
+MASTER_PORT := 9333
+VOLUME_PORT := 8080
+FILER_PORT := 8888
+TEST_TIMEOUT := 10m
+TEST_PATTERN := TestCORS
+SERVER_DIR := test-mini-data
+S3_CONFIG := s3_test_config.json
+
+# Default target
+help:
+	@echo "CORS Integration Tests Makefile"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  help                       - Show this help message"
+	@echo "  build-weed                - Build the SeaweedFS binary"
+	@echo "  check-deps                - Check dependencies and build binary if needed"
+	@echo "  start-server              - Start SeaweedFS server for testing"
+	@echo "  start-server-simple       - Start server without process cleanup (for CI)"
+	@echo "  stop-server               - Stop SeaweedFS server"
+	@echo "  test-cors                 - Run all CORS tests"
+	@echo "  test-cors-quick           - Run core CORS tests only"
+	@echo "  test-cors-simple          - Run tests without server management"
+	@echo "  test-cors-comprehensive   - Run comprehensive CORS tests"
+	@echo "  test-with-server          - Start server, run tests, stop server"
+	@echo "  logs                      - Show server logs"
+	@echo "  clean                     - Clean up test artifacts and stop server"
+	@echo "  health-check              - Check if server is accessible"
+	@echo ""
+	@echo "Configuration:"
+	@echo "  S3_PORT=${S3_PORT}"
+	@echo "  TEST_TIMEOUT=${TEST_TIMEOUT}"
+
+# Build the SeaweedFS binary
+build-weed:
+	@echo "Building SeaweedFS binary..."
+	@cd ../../../weed && go build -o weed_binary .
+	@chmod +x $(WEED_BINARY)
+	@echo "OK SeaweedFS binary built at $(WEED_BINARY)"
+
+check-deps: build-weed
+	@echo "Checking dependencies..."
+	@echo "DEBUG: Checking Go installation..."
+	@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1)
+	@echo "DEBUG: Go version: $$(go version)"
+	@echo "DEBUG: Checking binary at $(WEED_BINARY)..."
+	@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1)
+	@echo "🔍 DEBUG: Binary size: $$(ls -lh $(WEED_BINARY) | awk '{print $$5}')"
+	@echo "🔍 DEBUG: Binary permissions: $$(ls -la $(WEED_BINARY) | awk '{print $$1}')"
+	@echo "🔍 DEBUG: Checking Go module dependencies..."
+	@go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1)
+	@go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1)
+	@echo "OK All dependencies are available"
+
+# Start SeaweedFS server for testing
+start-server: check-deps
+	@echo "Starting SeaweedFS server..."
+	@echo "🔍 DEBUG: Current working directory: $$(pwd)"
+	@echo "🔍 DEBUG: Checking for existing weed processes..."
+	@ps aux | grep weed | grep -v grep || echo "No existing weed processes found"
+	@echo "🔍 DEBUG: Cleaning up any existing PID file..."
+	@rm -f weed-server.pid
+	@echo "🔍 DEBUG: Checking for port conflicts..."
+	@if netstat -tlnp 2>/dev/null | grep $(S3_PORT) >/dev/null; then \
+		echo "⚠️  Port $(S3_PORT) is already in use, trying to find the process..."; \
+		netstat -tlnp 2>/dev/null | grep $(S3_PORT) || true; \
+	else \
+		echo "✅ Port $(S3_PORT) is available"; \
+	fi
+	@echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)"
+	@ls -la $(WEED_BINARY) || (echo "❌ Binary not found!" && exit 1)
+	@echo "🔍 DEBUG: Checking config file at ../../../docker/compose/s3.json"
+	@ls -la ../../../docker/compose/s3.json || echo "⚠️  Config file not found, continuing without it"
+	@echo "🔍 DEBUG: Creating volume directory..."
+	@mkdir -p $(SERVER_DIR)
+	@echo "🔍 DEBUG: Launching SeaweedFS S3 server in background..."
+	@echo "🔍 DEBUG: Command: AWS_ACCESS_KEY_ID=some_access_key1 AWS_SECRET_ACCESS_KEY=some_secret_key1 $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
+	@env AWS_ACCESS_KEY_ID=some_access_key1 \
+	 AWS_SECRET_ACCESS_KEY=some_secret_key1 \
+	 $(WEED_BINARY) mini \
+		-dir=$(SERVER_DIR) \
+		-s3.port=$(S3_PORT) \
+		-s3.config=$(S3_CONFIG) \
+		> weed-test.log 2>&1 & \
+		echo $$! > weed-server.pid
+	
+	@echo "Waiting for S3 server to be ready..."
+	@for i in $$(seq 1 30); do \
+		if echo | nc -z localhost $(S3_PORT); then \
+			echo "S3 server is ready!"; \
+			exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "S3 server failed to start"; \
+	exit 1
+	@echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
+	@echo "🔍 DEBUG: Checking if PID is still running..."
+	@sleep 2
+	@if [ -f weed-server.pid ]; then \
+		SERVER_PID=$$(cat weed-server.pid); \
+		ps -p $$SERVER_PID || echo "⚠️  Server PID $$SERVER_PID not found after 2 seconds"; \
+	else \
+		echo "⚠️  PID file not found"; \
+	fi
+	@echo "🔍 DEBUG: Waiting for server to start (up to 90 seconds)..."
+	@for i in $$(seq 1 90); do \
+		echo "🔍 DEBUG: Attempt $$i/90 - checking port $(S3_PORT)"; \
+		if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
+			echo "✅ SeaweedFS server started successfully on port $(S3_PORT) after $$i seconds"; \
+			exit 0; \
+		fi; \
+		if [ $$i -eq 5 ]; then \
+			echo "🔍 DEBUG: After 5 seconds, checking process and logs..."; \
+			ps aux | grep weed | grep -v grep || echo "No weed processes found"; \
+			if [ -f weed-test.log ]; then \
+				echo "=== First server logs ==="; \
+				head -20 weed-test.log; \
+			fi; \
+		fi; \
+		if [ $$i -eq 15 ]; then \
+			echo "🔍 DEBUG: After 15 seconds, checking port bindings..."; \
+			netstat -tlnp 2>/dev/null | grep $(S3_PORT) || echo "Port $(S3_PORT) not bound"; \
+			netstat -tlnp 2>/dev/null | grep 9333 || echo "Port 9333 not bound"; \
+			netstat -tlnp 2>/dev/null | grep 8080 || echo "Port 8080 not bound"; \
+		fi; \
+		if [ $$i -eq 30 ]; then \
+			echo "⚠️  Server taking longer than expected (30s), checking logs..."; \
+			if [ -f weed-test.log ]; then \
+				echo "=== Recent server logs ==="; \
+				tail -20 weed-test.log; \
+			fi; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "❌ Server failed to start within 90 seconds"; \
+	echo "🔍 DEBUG: Final process check:"; \
+	ps aux | grep weed | grep -v grep || echo "No weed processes found"; \
+	echo "🔍 DEBUG: Final port check:"; \
+	netstat -tlnp 2>/dev/null | grep -E "(8333|9333|8080)" || echo "No ports bound"; \
+	echo "=== Full server logs ==="; \
+	if [ -f weed-test.log ]; then \
+		cat weed-test.log; \
+	else \
+		echo "No log file found"; \
+	fi; \
+	exit 1
+
+# Stop SeaweedFS server
+stop-server:
+	@echo "Stopping SeaweedFS server..."
+	@if [ -f weed-server.pid ]; then \
+		SERVER_PID=$$(cat weed-server.pid); \
+		echo "Killing server PID $$SERVER_PID"; \
+		if ps -p $$SERVER_PID >/dev/null 2>&1; then \
+			kill -TERM $$SERVER_PID 2>/dev/null || true; \
+			sleep 2; \
+			if ps -p $$SERVER_PID >/dev/null 2>&1; then \
+				echo "Process still running, sending KILL signal..."; \
+				kill -KILL $$SERVER_PID 2>/dev/null || true; \
+				sleep 1; \
+			fi; \
+		else \
+			echo "Process $$SERVER_PID not found (already stopped)"; \
+		fi; \
+		rm -f weed-server.pid; \
+	else \
+		echo "No PID file found, checking for running processes..."; \
+		echo "⚠️  Skipping automatic process cleanup to avoid CI issues"; \
+		echo "Note: Any remaining weed processes should be cleaned up by the CI environment"; \
+	fi
+	@echo "✅ SeaweedFS server stopped"
+
+# Show server logs
+logs:
+	@if test -f weed-test.log; then \
+		echo "=== SeaweedFS Server Logs ==="; \
+		tail -f weed-test.log; \
+	else \
+		echo "No log file found. Server may not be running."; \
+	fi
+
+# Core CORS tests (basic functionality)
+test-cors-quick: check-deps
+	@echo "Running core CORS tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestCORSConfigurationManagement|TestCORSPreflightRequest|TestCORSActualRequest" .
+	@echo "✅ Core CORS tests completed"
+
+# All CORS tests (comprehensive)
+test-cors: check-deps
+	@echo "Running all CORS tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" .
+	@echo "✅ All CORS tests completed"
+
+# Comprehensive CORS tests (all features)
+test-cors-comprehensive: check-deps
+	@echo "Running comprehensive CORS tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestCORS" .
+	@echo "✅ Comprehensive CORS tests completed"
+
+# All tests without server management
+test-cors-simple: check-deps
+	@echo "Running CORS tests (assuming server is already running)..."
+	@go test -v -timeout=$(TEST_TIMEOUT) .
+	@echo "✅ All CORS tests completed"
+
+# Start server, run tests, stop server
+test-with-server: start-server
+	@echo "Running CORS tests with managed server..."
+	@sleep 5  # Give server time to fully start
+	@make test-cors-comprehensive || (echo "Tests failed, stopping server..." && make stop-server && exit 1)
+	@make stop-server
+	@echo "✅ All tests completed with managed server"
+
+# Health check
+health-check:
+	@echo "Checking server health..."
+	@if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
+		echo "✅ Server is accessible on port $(S3_PORT)"; \
+	else \
+		echo "❌ Server is not accessible on port $(S3_PORT)"; \
+		exit 1; \
+	fi
+
+# Clean up
+clean:
+	@echo "Cleaning up test artifacts..."
+	@make stop-server
+	@rm -f weed-test.log
+	@rm -f weed-server.pid
+	@rm -rf ./test-volume-data
+	@rm -f cors.test
+	@go clean -testcache
+	@echo "✅ Cleanup completed"
+
+# Individual test targets for specific functionality
+test-basic-cors:
+	@echo "Running basic CORS tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestCORSConfigurationManagement" .
+
+test-preflight-cors:
+	@echo "Running preflight CORS tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestCORSPreflightRequest" .
+
+test-actual-cors:
+	@echo "Running actual CORS request tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestCORSActualRequest" .
+
+test-origin-matching:
+	@echo "Running origin matching tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestCORSOriginMatching" .
+
+test-header-matching:
+	@echo "Running header matching tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestCORSHeaderMatching" .
+
+test-method-matching:
+	@echo "Running method matching tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestCORSMethodMatching" .
+
+test-multiple-rules:
+	@echo "Running multiple rules tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestCORSMultipleRulesMatching" .
+
+test-validation:
+	@echo "Running validation tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestCORSValidation" .
+
+test-caching:
+	@echo "Running caching tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestCORSCaching" .
+
+test-error-handling:
+	@echo "Running error handling tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestCORSErrorHandling" .
+
+# Development targets
+dev-start: start-server
+	@echo "Development server started. Access S3 API at http://localhost:$(S3_PORT)"
+	@echo "To stop: make stop-server"
+
+dev-test: check-deps
+	@echo "Running tests in development mode..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -run "TestCORSConfigurationManagement" .
+
+# CI targets
+ci-test: check-deps
+	@echo "Running tests in CI mode..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -race .
+
+# All targets
+test-all: test-cors test-cors-comprehensive
+	@echo "✅ All CORS tests completed"
+
+# Benchmark targets
+benchmark-cors:
+	@echo "Running CORS performance benchmarks..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -bench=. -benchmem .
+
+# Coverage targets
+coverage:
+	@echo "Running tests with coverage..."
+	@go test -v -timeout=$(TEST_TIMEOUT) -coverprofile=coverage.out .
+	@go tool cover -html=coverage.out -o coverage.html
+	@echo "Coverage report generated: coverage.html"
+
+# Format and lint
+fmt:
+	@echo "Formatting Go code..."
+	@go fmt .
+
+lint:
+	@echo "Running linter..."
+	@golint . || echo "golint not available, skipping..."
+
+# Install dependencies for development
+install-deps:
+	@echo "Installing Go dependencies..."
+	@go mod tidy
+	@go mod download
+
+# Show current configuration
+show-config:
+	@echo "Current configuration:"
+	@echo "  WEED_BINARY: $(WEED_BINARY)"
+	@echo "  S3_PORT: $(S3_PORT)"
+	@echo "  TEST_TIMEOUT: $(TEST_TIMEOUT)"
+	@echo "  TEST_PATTERN: $(TEST_PATTERN)"
+
+# Legacy targets for backward compatibility
+test: test-with-server
+test-verbose: test-cors-comprehensive
+test-single: test-basic-cors
+test-clean: clean
+build: check-deps
+setup: check-deps 

test/s3/cors/README.md

@@ -0,0 +1,362 @@
+# CORS Integration Tests for SeaweedFS S3 API
+
+This directory contains comprehensive integration tests for the CORS (Cross-Origin Resource Sharing) functionality in SeaweedFS S3 API.
+
+## Overview
+
+The CORS integration tests validate the complete CORS implementation including:
+- CORS configuration management (PUT/GET/DELETE)
+- CORS rule validation
+- CORS middleware behavior
+- Caching functionality
+- Error handling
+- Real-world CORS scenarios
+
+## Prerequisites
+
+1. **Go 1.19+**: For building SeaweedFS and running tests
+2. **Network Access**: Tests use `localhost:8333` by default
+3. **System Dependencies**: `curl` and `netstat` for health checks
+
+## Quick Start
+
+The tests now automatically start their own SeaweedFS server, so you don't need to manually start one.
+
+### 1. Run All Tests with Managed Server
+
+```bash
+# Run all tests with automatic server management
+make test-with-server
+
+# Run core CORS tests only
+make test-cors-quick
+
+# Run comprehensive CORS tests
+make test-cors-comprehensive
+```
+
+### 2. Manual Server Management
+
+If you prefer to manage the server manually:
+
+```bash
+# Start server
+make start-server
+
+# Run tests (assuming server is running)
+make test-cors-simple
+
+# Stop server
+make stop-server
+```
+
+### 3. Individual Test Categories
+
+```bash
+# Run specific test types
+make test-basic-cors           # Basic CORS configuration
+make test-preflight-cors       # Preflight OPTIONS requests
+make test-actual-cors          # Actual CORS request handling
+make test-origin-matching      # Origin matching logic
+make test-header-matching      # Header matching logic
+make test-method-matching      # Method matching logic
+make test-multiple-rules       # Multiple CORS rules
+make test-validation           # CORS validation
+make test-caching              # CORS caching behavior
+make test-error-handling       # Error handling
+```
+
+## Test Server Management
+
+The tests use a comprehensive server management system similar to other SeaweedFS integration tests:
+
+### Server Configuration
+
+- **S3 Port**: 8333 (configurable via `S3_PORT`)
+- **Master Port**: 9333
+- **Volume Port**: 8080
+- **Filer Port**: 8888
+- **Metrics Port**: 9324
+- **Data Directory**: `./test-volume-data` (auto-created)
+- **Log File**: `weed-test.log`
+
+### Server Lifecycle
+
+1. **Build**: Automatically builds `../../../weed/weed_binary`
+2. **Start**: Launches SeaweedFS with S3 API enabled
+3. **Health Check**: Waits up to 90 seconds for server to be ready
+4. **Test**: Runs the requested tests
+5. **Stop**: Gracefully shuts down the server
+6. **Cleanup**: Removes temporary files and data
+
+### Available Commands
+
+```bash
+# Server management
+make start-server              # Start SeaweedFS server
+make stop-server               # Stop SeaweedFS server
+make health-check              # Check server health
+make logs                      # View server logs
+
+# Test execution
+make test-with-server          # Full test cycle with server management
+make test-cors-simple          # Run tests without server management
+make test-cors-quick           # Run core tests only
+make test-cors-comprehensive   # Run all tests
+
+# Development
+make dev-start                 # Start server for development
+make dev-test                  # Run development tests
+make build-weed                # Build SeaweedFS binary
+make check-deps                # Check dependencies
+
+# Maintenance
+make clean                     # Clean up all artifacts
+make coverage                  # Generate coverage report
+make fmt                       # Format code
+make lint                      # Run linter
+```
+
+## Test Configuration
+
+### Default Configuration
+
+The tests use these default settings (configurable via environment variables):
+
+```bash
+WEED_BINARY=../../../weed/weed_binary
+S3_PORT=8333
+TEST_TIMEOUT=10m
+TEST_PATTERN=TestCORS
+```
+
+### Configuration File
+
+The `test_config.json` file contains S3 client configuration:
+
+```json
+{
+  "endpoint": "http://localhost:8333",
+  "access_key": "some_access_key1",
+  "secret_key": "some_secret_key1",
+  "region": "us-east-1",
+  "bucket_prefix": "test-cors-",
+  "use_ssl": false,
+  "skip_verify_ssl": true
+}
+```
+
+## Troubleshooting
+
+### Compilation Issues
+
+If you encounter compilation errors, the most common issues are:
+
+1. **AWS SDK v2 Type Mismatches**: The `MaxAgeSeconds` field in `types.CORSRule` expects `int32`, not `*int32`. Use direct values like `3600` instead of `aws.Int32(3600)`.
+
+2. **Field Name Issues**: The `GetBucketCorsOutput` type has a `CORSRules` field directly, not a `CORSConfiguration` field.
+
+Example fix:
+```go
+// ❌ Incorrect
+MaxAgeSeconds: aws.Int32(3600),
+assert.Len(t, getResp.CORSConfiguration.CORSRules, 1)
+
+// ✅ Correct
+MaxAgeSeconds: 3600,
+assert.Len(t, getResp.CORSRules, 1)
+```
+
+### Server Issues
+
+1. **Server Won't Start**
+   ```bash
+   # Check for port conflicts
+   netstat -tlnp | grep 8333
+   
+   # View server logs
+   make logs
+   
+   # Force cleanup
+   make clean
+   ```
+
+2. **Test Failures**
+   ```bash
+   # Run with server management
+   make test-with-server
+   
+   # Run specific test
+   make test-basic-cors
+   
+   # Check server health
+   make health-check
+   ```
+
+3. **Connection Issues**
+   ```bash
+   # Verify server is running
+   curl -s http://localhost:8333
+   
+   # Check server logs
+   tail -f weed-test.log
+   ```
+
+### Performance Issues
+
+If tests are slow or timing out:
+
+```bash
+# Increase timeout
+export TEST_TIMEOUT=30m
+make test-with-server
+
+# Run quick tests only
+make test-cors-quick
+
+# Check server resources
+make debug-status
+```
+
+## Test Coverage
+
+### Core Functionality Tests
+
+#### 1. CORS Configuration Management (`TestCORSConfigurationManagement`)
+- PUT CORS configuration
+- GET CORS configuration
+- DELETE CORS configuration
+- Configuration updates
+- Error handling for non-existent configurations
+
+#### 2. Multiple CORS Rules (`TestCORSMultipleRules`)
+- Multiple rules in single configuration
+- Rule precedence and ordering
+- Complex rule combinations
+
+#### 3. CORS Validation (`TestCORSValidation`)
+- Invalid HTTP methods
+- Empty origins validation
+- Negative MaxAge validation
+- Rule limit validation
+
+#### 4. Wildcard Support (`TestCORSWithWildcards`)
+- Wildcard origins (`*`, `https://*.example.com`)
+- Wildcard headers (`*`)
+- Wildcard expose headers
+
+#### 5. Rule Limits (`TestCORSRuleLimit`)
+- Maximum 100 rules per configuration
+- Rule limit enforcement
+- Large configuration handling
+
+#### 6. Error Handling (`TestCORSErrorHandling`)
+- Non-existent bucket operations
+- Invalid configurations
+- Malformed requests
+
+### HTTP-Level Tests
+
+#### 1. Preflight Requests (`TestCORSPreflightRequest`)
+- OPTIONS request handling
+- CORS headers in preflight responses
+- Access-Control-Request-Method validation
+- Access-Control-Request-Headers validation
+
+#### 2. Actual Requests (`TestCORSActualRequest`)
+- CORS headers in actual responses
+- Origin validation for real requests
+- Proper expose headers handling
+
+#### 3. Origin Matching (`TestCORSOriginMatching`)
+- Exact origin matching
+- Wildcard origin matching (`*`)
+- Subdomain wildcard matching (`https://*.example.com`)
+- Non-matching origins (should be rejected)
+
+#### 4. Header Matching (`TestCORSHeaderMatching`)
+- Wildcard header matching (`*`)
+- Specific header matching
+- Case-insensitive matching
+- Disallowed headers
+
+#### 5. Method Matching (`TestCORSMethodMatching`)
+- Allowed methods verification
+- Disallowed methods rejection
+- Method-specific CORS behavior
+
+#### 6. Multiple Rules (`TestCORSMultipleRulesMatching`)
+- Rule precedence and selection
+- Multiple rules with different configurations
+- Complex rule interactions
+
+### Integration Tests
+
+#### 1. Caching (`TestCORSCaching`)
+- CORS configuration caching
+- Cache invalidation
+- Cache performance
+
+#### 2. Object Operations (`TestCORSObjectOperations`)
+- CORS with actual S3 operations
+- PUT/GET/DELETE objects with CORS
+- CORS headers in object responses
+
+#### 3. Without Configuration (`TestCORSWithoutConfiguration`)
+- Behavior when no CORS configuration exists
+- Default CORS behavior
+- Graceful degradation
+
+## Development
+
+### Running Tests During Development
+
+```bash
+# Start server for development
+make dev-start
+
+# Run quick test
+make dev-test
+
+# View logs in real-time
+make logs
+```
+
+### Adding New Tests
+
+1. Follow the existing naming convention (`TestCORSXxxYyy`)
+2. Use the helper functions (`getS3Client`, `createTestBucket`, etc.)
+3. Add cleanup with `defer cleanupTestBucket(t, client, bucketName)`
+4. Include proper error checking with `require.NoError(t, err)`
+5. Use assertions with `assert.Equal(t, expected, actual)`
+6. Add the test to the appropriate Makefile target
+
+### Code Quality
+
+```bash
+# Format code
+make fmt
+
+# Run linter
+make lint
+
+# Generate coverage report
+make coverage
+```
+
+## Performance Notes
+
+- Tests create and destroy buckets for each test case
+- Large configuration tests may take several minutes
+- Server startup typically takes 15-30 seconds
+- Tests run in parallel where possible for efficiency
+
+## Integration with SeaweedFS
+
+These tests validate the CORS implementation in:
+- `weed/s3api/cors/` - Core CORS package
+- `weed/s3api/s3api_bucket_cors_handlers.go` - HTTP handlers
+- `weed/s3api/s3api_server.go` - Router integration
+- `weed/s3api/s3api_bucket_config.go` - Configuration management
+
+The tests ensure AWS S3 API compatibility and proper CORS behavior across all supported scenarios. 

test/s3/cors/s3_cors_http_test.go

@@ -0,0 +1,633 @@
+package cors
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestCORSPreflightRequest tests CORS preflight OPTIONS requests
+func TestCORSPreflightRequest(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	// Set up CORS configuration
+	corsConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"Content-Type", "Authorization"},
+				AllowedMethods: []string{"GET", "POST", "PUT", "DELETE"},
+				AllowedOrigins: []string{"https://example.com"},
+				ExposeHeaders:  []string{"ETag", "Content-Length"},
+				MaxAgeSeconds:  aws.Int32(3600),
+			},
+		},
+	}
+
+	_, err := client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: corsConfig,
+	})
+	require.NoError(t, err, "Should be able to put CORS configuration")
+
+	// Wait for metadata subscription to update cache
+	time.Sleep(50 * time.Millisecond)
+
+	// Test preflight request with raw HTTP
+	httpClient := &http.Client{Timeout: 10 * time.Second}
+
+	// Create OPTIONS request
+	req, err := http.NewRequest("OPTIONS", fmt.Sprintf("%s/%s/test-object", getDefaultConfig().Endpoint, bucketName), nil)
+	require.NoError(t, err, "Should be able to create OPTIONS request")
+
+	// Add CORS preflight headers
+	req.Header.Set("Origin", "https://example.com")
+	req.Header.Set("Access-Control-Request-Method", "PUT")
+	req.Header.Set("Access-Control-Request-Headers", "Content-Type, Authorization")
+
+	// Send the request
+	resp, err := httpClient.Do(req)
+	require.NoError(t, err, "Should be able to send OPTIONS request")
+	defer resp.Body.Close()
+
+	// Verify CORS headers in response
+	assert.Equal(t, "https://example.com", resp.Header.Get("Access-Control-Allow-Origin"), "Should have correct Allow-Origin header")
+	assert.Contains(t, resp.Header.Get("Access-Control-Allow-Methods"), "PUT", "Should allow PUT method")
+	assert.Contains(t, resp.Header.Get("Access-Control-Allow-Headers"), "Content-Type", "Should allow Content-Type header")
+	assert.Contains(t, resp.Header.Get("Access-Control-Allow-Headers"), "Authorization", "Should allow Authorization header")
+	assert.Equal(t, "3600", resp.Header.Get("Access-Control-Max-Age"), "Should have correct Max-Age header")
+	assert.Contains(t, resp.Header.Get("Access-Control-Expose-Headers"), "ETag", "Should expose ETag header")
+	assert.Equal(t, http.StatusOK, resp.StatusCode, "OPTIONS request should return 200")
+}
+
+// TestCORSActualRequest tests CORS behavior with actual requests
+func TestCORSActualRequest(t *testing.T) {
+	// Temporarily clear AWS environment variables to ensure truly anonymous requests
+	// This prevents AWS SDK from auto-signing requests in GitHub Actions
+	originalAccessKey := os.Getenv("AWS_ACCESS_KEY_ID")
+	originalSecretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
+	originalSessionToken := os.Getenv("AWS_SESSION_TOKEN")
+	originalProfile := os.Getenv("AWS_PROFILE")
+	originalRegion := os.Getenv("AWS_REGION")
+
+	os.Setenv("AWS_ACCESS_KEY_ID", "")
+	os.Setenv("AWS_SECRET_ACCESS_KEY", "")
+	os.Setenv("AWS_SESSION_TOKEN", "")
+	os.Setenv("AWS_PROFILE", "")
+	os.Setenv("AWS_REGION", "")
+
+	defer func() {
+		// Restore original environment variables
+		os.Setenv("AWS_ACCESS_KEY_ID", originalAccessKey)
+		os.Setenv("AWS_SECRET_ACCESS_KEY", originalSecretKey)
+		os.Setenv("AWS_SESSION_TOKEN", originalSessionToken)
+		os.Setenv("AWS_PROFILE", originalProfile)
+		os.Setenv("AWS_REGION", originalRegion)
+	}()
+
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	// Set up CORS configuration
+	corsConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"*"},
+				AllowedMethods: []string{"GET", "PUT"},
+				AllowedOrigins: []string{"https://example.com"},
+				ExposeHeaders:  []string{"ETag", "Content-Length"},
+				MaxAgeSeconds:  aws.Int32(3600),
+			},
+		},
+	}
+
+	_, err := client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: corsConfig,
+	})
+	require.NoError(t, err, "Should be able to put CORS configuration")
+
+	// Wait for CORS configuration to be fully processed
+	time.Sleep(100 * time.Millisecond)
+
+	// First, put an object using S3 client
+	objectKey := "test-cors-object"
+	_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   strings.NewReader("Test CORS content"),
+	})
+	require.NoError(t, err, "Should be able to put object")
+
+	// Test GET request with CORS headers using raw HTTP
+	// Create a completely isolated HTTP client to avoid AWS SDK auto-signing
+	transport := &http.Transport{
+		// Completely disable any proxy or middleware
+		Proxy: nil,
+	}
+
+	httpClient := &http.Client{
+		Timeout: 10 * time.Second,
+		// Use a completely clean transport to avoid any AWS SDK middleware
+		Transport: transport,
+	}
+
+	// Create URL manually to avoid any AWS SDK endpoint processing
+	// Use the same endpoint as the S3 client to ensure compatibility with GitHub Actions
+	config := getDefaultConfig()
+	endpoint := config.Endpoint
+	// Remove any protocol prefix and ensure it's http for anonymous requests
+	if strings.HasPrefix(endpoint, "https://") {
+		endpoint = strings.Replace(endpoint, "https://", "http://", 1)
+	}
+	if !strings.HasPrefix(endpoint, "http://") {
+		endpoint = "http://" + endpoint
+	}
+
+	requestURL := fmt.Sprintf("%s/%s/%s", endpoint, bucketName, objectKey)
+	req, err := http.NewRequest("GET", requestURL, nil)
+	require.NoError(t, err, "Should be able to create GET request")
+
+	// Add Origin header to simulate CORS request
+	req.Header.Set("Origin", "https://example.com")
+
+	// Explicitly ensure no AWS headers are present (defensive programming)
+	// Clear ALL potential AWS-related headers that might be auto-added
+	req.Header.Del("Authorization")
+	req.Header.Del("X-Amz-Content-Sha256")
+	req.Header.Del("X-Amz-Date")
+	req.Header.Del("Amz-Sdk-Invocation-Id")
+	req.Header.Del("Amz-Sdk-Request")
+	req.Header.Del("X-Amz-Security-Token")
+	req.Header.Del("X-Amz-Session-Token")
+	req.Header.Del("AWS-Session-Token")
+	req.Header.Del("X-Amz-Target")
+	req.Header.Del("X-Amz-User-Agent")
+
+	// Ensure User-Agent doesn't indicate AWS SDK
+	req.Header.Set("User-Agent", "anonymous-cors-test/1.0")
+
+	// Verify no AWS-related headers are present
+	for name := range req.Header {
+		headerLower := strings.ToLower(name)
+		if strings.Contains(headerLower, "aws") ||
+			strings.Contains(headerLower, "amz") ||
+			strings.Contains(headerLower, "authorization") {
+			t.Fatalf("Found AWS-related header in anonymous request: %s", name)
+		}
+	}
+
+	// Send the request
+	resp, err := httpClient.Do(req)
+	require.NoError(t, err, "Should be able to send GET request")
+	defer resp.Body.Close()
+
+	// Verify CORS headers are present
+	assert.Equal(t, "https://example.com", resp.Header.Get("Access-Control-Allow-Origin"), "Should have correct Allow-Origin header")
+	assert.Contains(t, resp.Header.Get("Access-Control-Expose-Headers"), "ETag", "Should expose ETag header")
+
+	// Anonymous requests should succeed when anonymous read permission is configured in IAM
+	// The server configuration allows anonymous users to have Read permissions
+	assert.Equal(t, http.StatusOK, resp.StatusCode, "Anonymous GET request should succeed when anonymous read is configured")
+}
+
+// TestCORSOriginMatching tests origin matching with different patterns
+func TestCORSOriginMatching(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	testCases := []struct {
+		name           string
+		allowedOrigins []string
+		requestOrigin  string
+		shouldAllow    bool
+	}{
+		{
+			name:           "exact match",
+			allowedOrigins: []string{"https://example.com"},
+			requestOrigin:  "https://example.com",
+			shouldAllow:    true,
+		},
+		{
+			name:           "wildcard match",
+			allowedOrigins: []string{"*"},
+			requestOrigin:  "https://example.com",
+			shouldAllow:    true,
+		},
+		{
+			name:           "subdomain wildcard match",
+			allowedOrigins: []string{"https://*.example.com"},
+			requestOrigin:  "https://api.example.com",
+			shouldAllow:    true,
+		},
+		{
+			name:           "no match",
+			allowedOrigins: []string{"https://example.com"},
+			requestOrigin:  "https://malicious.com",
+			shouldAllow:    false,
+		},
+		{
+			name:           "subdomain wildcard no match",
+			allowedOrigins: []string{"https://*.example.com"},
+			requestOrigin:  "https://example.com",
+			shouldAllow:    false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Set up CORS configuration for this test case
+			corsConfig := &types.CORSConfiguration{
+				CORSRules: []types.CORSRule{
+					{
+						AllowedHeaders: []string{"*"},
+						AllowedMethods: []string{"GET"},
+						AllowedOrigins: tc.allowedOrigins,
+						ExposeHeaders:  []string{"ETag"},
+						MaxAgeSeconds:  aws.Int32(3600),
+					},
+				},
+			}
+
+			_, err := client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+				Bucket:            aws.String(bucketName),
+				CORSConfiguration: corsConfig,
+			})
+			require.NoError(t, err, "Should be able to put CORS configuration")
+
+			// Wait for metadata subscription to update cache
+			time.Sleep(50 * time.Millisecond)
+
+			// Test preflight request
+			httpClient := &http.Client{Timeout: 10 * time.Second}
+
+			req, err := http.NewRequest("OPTIONS", fmt.Sprintf("%s/%s/test-object", getDefaultConfig().Endpoint, bucketName), nil)
+			require.NoError(t, err, "Should be able to create OPTIONS request")
+
+			req.Header.Set("Origin", tc.requestOrigin)
+			req.Header.Set("Access-Control-Request-Method", "GET")
+
+			resp, err := httpClient.Do(req)
+			require.NoError(t, err, "Should be able to send OPTIONS request")
+			defer resp.Body.Close()
+
+			if tc.shouldAllow {
+				assert.Equal(t, tc.requestOrigin, resp.Header.Get("Access-Control-Allow-Origin"), "Should have correct Allow-Origin header")
+				assert.Contains(t, resp.Header.Get("Access-Control-Allow-Methods"), "GET", "Should allow GET method")
+			} else {
+				assert.Empty(t, resp.Header.Get("Access-Control-Allow-Origin"), "Should not have Allow-Origin header for disallowed origin")
+			}
+		})
+	}
+}
+
+// TestCORSHeaderMatching tests header matching with different patterns
+func TestCORSHeaderMatching(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	testCases := []struct {
+		name            string
+		allowedHeaders  []string
+		requestHeaders  string
+		shouldAllow     bool
+		expectedHeaders string
+	}{
+		{
+			name:            "wildcard headers",
+			allowedHeaders:  []string{"*"},
+			requestHeaders:  "Content-Type, Authorization",
+			shouldAllow:     true,
+			expectedHeaders: "Content-Type, Authorization",
+		},
+		{
+			name:            "specific headers match",
+			allowedHeaders:  []string{"Content-Type", "Authorization"},
+			requestHeaders:  "Content-Type, Authorization",
+			shouldAllow:     true,
+			expectedHeaders: "Content-Type, Authorization",
+		},
+		{
+			name:            "partial header match",
+			allowedHeaders:  []string{"Content-Type"},
+			requestHeaders:  "Content-Type",
+			shouldAllow:     true,
+			expectedHeaders: "Content-Type",
+		},
+		{
+			name:            "case insensitive match",
+			allowedHeaders:  []string{"content-type"},
+			requestHeaders:  "Content-Type",
+			shouldAllow:     true,
+			expectedHeaders: "Content-Type",
+		},
+		{
+			name:           "disallowed header",
+			allowedHeaders: []string{"Content-Type"},
+			requestHeaders: "Authorization",
+			shouldAllow:    false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Set up CORS configuration for this test case
+			corsConfig := &types.CORSConfiguration{
+				CORSRules: []types.CORSRule{
+					{
+						AllowedHeaders: tc.allowedHeaders,
+						AllowedMethods: []string{"GET", "POST"},
+						AllowedOrigins: []string{"https://example.com"},
+						ExposeHeaders:  []string{"ETag"},
+						MaxAgeSeconds:  aws.Int32(3600),
+					},
+				},
+			}
+
+			_, err := client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+				Bucket:            aws.String(bucketName),
+				CORSConfiguration: corsConfig,
+			})
+			require.NoError(t, err, "Should be able to put CORS configuration")
+
+			// Wait for metadata subscription to update cache
+			time.Sleep(50 * time.Millisecond)
+
+			// Test preflight request
+			httpClient := &http.Client{Timeout: 10 * time.Second}
+
+			req, err := http.NewRequest("OPTIONS", fmt.Sprintf("%s/%s/test-object", getDefaultConfig().Endpoint, bucketName), nil)
+			require.NoError(t, err, "Should be able to create OPTIONS request")
+
+			req.Header.Set("Origin", "https://example.com")
+			req.Header.Set("Access-Control-Request-Method", "POST")
+			req.Header.Set("Access-Control-Request-Headers", tc.requestHeaders)
+
+			resp, err := httpClient.Do(req)
+			require.NoError(t, err, "Should be able to send OPTIONS request")
+			defer resp.Body.Close()
+
+			if tc.shouldAllow {
+				assert.Equal(t, "https://example.com", resp.Header.Get("Access-Control-Allow-Origin"), "Should have correct Allow-Origin header")
+				allowedHeaders := resp.Header.Get("Access-Control-Allow-Headers")
+				for _, header := range strings.Split(tc.expectedHeaders, ", ") {
+					assert.Contains(t, allowedHeaders, header, "Should allow header: %s", header)
+				}
+			} else {
+				// Even if headers are not allowed, the origin should still be in the response
+				// but the headers should not be echoed back
+				assert.Equal(t, "https://example.com", resp.Header.Get("Access-Control-Allow-Origin"), "Should have correct Allow-Origin header")
+				allowedHeaders := resp.Header.Get("Access-Control-Allow-Headers")
+				assert.NotContains(t, allowedHeaders, "Authorization", "Should not allow Authorization header")
+			}
+		})
+	}
+}
+
+// TestCORSWithoutConfiguration tests CORS behavior when no bucket-level configuration is set
+// With the fallback feature, buckets without explicit CORS config will use the global CORS settings
+func TestCORSWithoutConfiguration(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	// Test preflight request without bucket-level CORS configuration
+	// The global CORS fallback (default: "*") should be used
+	httpClient := &http.Client{Timeout: 10 * time.Second}
+
+	req, err := http.NewRequest("OPTIONS", fmt.Sprintf("%s/%s/test-object", getDefaultConfig().Endpoint, bucketName), nil)
+	require.NoError(t, err, "Should be able to create OPTIONS request")
+
+	req.Header.Set("Origin", "https://example.com")
+	req.Header.Set("Access-Control-Request-Method", "GET")
+	req.Header.Set("Access-Control-Request-Headers", "Content-Type")
+
+	resp, err := httpClient.Do(req)
+	require.NoError(t, err, "Should be able to send OPTIONS request")
+	defer resp.Body.Close()
+
+	// With fallback CORS (global default: "*"), CORS headers should be present
+	assert.Equal(t, "https://example.com", resp.Header.Get("Access-Control-Allow-Origin"), "Should have Allow-Origin header from global fallback")
+	assert.Contains(t, resp.Header.Get("Access-Control-Allow-Methods"), "GET", "Should have GET in Allow-Methods from global fallback")
+	assert.Contains(t, resp.Header.Get("Access-Control-Allow-Headers"), "Content-Type", "Should have requested headers in Allow-Headers from global fallback")
+}
+
+// TestCORSMethodMatching tests method matching
+func TestCORSMethodMatching(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	// Set up CORS configuration with limited methods
+	corsConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"*"},
+				AllowedMethods: []string{"GET", "POST"},
+				AllowedOrigins: []string{"https://example.com"},
+				ExposeHeaders:  []string{"ETag"},
+				MaxAgeSeconds:  aws.Int32(3600),
+			},
+		},
+	}
+
+	_, err := client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: corsConfig,
+	})
+	require.NoError(t, err, "Should be able to put CORS configuration")
+
+	// Wait for metadata subscription to update cache
+	time.Sleep(50 * time.Millisecond)
+
+	testCases := []struct {
+		method      string
+		shouldAllow bool
+	}{
+		{"GET", true},
+		{"POST", true},
+		{"PUT", false},
+		{"DELETE", false},
+		{"HEAD", false},
+	}
+
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("method_%s", tc.method), func(t *testing.T) {
+			httpClient := &http.Client{Timeout: 10 * time.Second}
+
+			req, err := http.NewRequest("OPTIONS", fmt.Sprintf("%s/%s/test-object", getDefaultConfig().Endpoint, bucketName), nil)
+			require.NoError(t, err, "Should be able to create OPTIONS request")
+
+			req.Header.Set("Origin", "https://example.com")
+			req.Header.Set("Access-Control-Request-Method", tc.method)
+
+			resp, err := httpClient.Do(req)
+			require.NoError(t, err, "Should be able to send OPTIONS request")
+			defer resp.Body.Close()
+
+			if tc.shouldAllow {
+				assert.Equal(t, "https://example.com", resp.Header.Get("Access-Control-Allow-Origin"), "Should have correct Allow-Origin header")
+				assert.Contains(t, resp.Header.Get("Access-Control-Allow-Methods"), tc.method, "Should allow method: %s", tc.method)
+			} else {
+				// Even if method is not allowed, the origin should still be in the response
+				// but the method should not be in the allowed methods
+				assert.Equal(t, "https://example.com", resp.Header.Get("Access-Control-Allow-Origin"), "Should have correct Allow-Origin header")
+				allowedMethods := resp.Header.Get("Access-Control-Allow-Methods")
+				assert.NotContains(t, allowedMethods, tc.method, "Should not allow method: %s", tc.method)
+			}
+		})
+	}
+}
+
+// TestCORSMultipleRulesMatching tests CORS with multiple rules
+func TestCORSMultipleRulesMatching(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	// Set up CORS configuration with multiple rules
+	corsConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"Content-Type"},
+				AllowedMethods: []string{"GET"},
+				AllowedOrigins: []string{"https://example.com"},
+				ExposeHeaders:  []string{"ETag"},
+				MaxAgeSeconds:  aws.Int32(3600),
+			},
+			{
+				AllowedHeaders: []string{"Authorization"},
+				AllowedMethods: []string{"POST", "PUT"},
+				AllowedOrigins: []string{"https://api.example.com"},
+				ExposeHeaders:  []string{"Content-Length"},
+				MaxAgeSeconds:  aws.Int32(7200),
+			},
+		},
+	}
+
+	_, err := client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: corsConfig,
+	})
+	require.NoError(t, err, "Should be able to put CORS configuration")
+
+	// Wait for metadata subscription to update cache
+	time.Sleep(50 * time.Millisecond)
+
+	// Test first rule
+	httpClient := &http.Client{Timeout: 10 * time.Second}
+
+	req, err := http.NewRequest("OPTIONS", fmt.Sprintf("%s/%s/test-object", getDefaultConfig().Endpoint, bucketName), nil)
+	require.NoError(t, err, "Should be able to create OPTIONS request")
+
+	req.Header.Set("Origin", "https://example.com")
+	req.Header.Set("Access-Control-Request-Method", "GET")
+	req.Header.Set("Access-Control-Request-Headers", "Content-Type")
+
+	resp, err := httpClient.Do(req)
+	require.NoError(t, err, "Should be able to send OPTIONS request")
+	defer resp.Body.Close()
+
+	assert.Equal(t, "https://example.com", resp.Header.Get("Access-Control-Allow-Origin"), "Should match first rule")
+	assert.Contains(t, resp.Header.Get("Access-Control-Allow-Methods"), "GET", "Should allow GET method")
+	assert.Contains(t, resp.Header.Get("Access-Control-Allow-Headers"), "Content-Type", "Should allow Content-Type header")
+	assert.Equal(t, "3600", resp.Header.Get("Access-Control-Max-Age"), "Should have first rule's max age")
+
+	// Test second rule
+	req2, err := http.NewRequest("OPTIONS", fmt.Sprintf("%s/%s/test-object", getDefaultConfig().Endpoint, bucketName), nil)
+	require.NoError(t, err, "Should be able to create OPTIONS request")
+
+	req2.Header.Set("Origin", "https://api.example.com")
+	req2.Header.Set("Access-Control-Request-Method", "POST")
+	req2.Header.Set("Access-Control-Request-Headers", "Authorization")
+
+	resp2, err := httpClient.Do(req2)
+	require.NoError(t, err, "Should be able to send OPTIONS request")
+	defer resp2.Body.Close()
+
+	assert.Equal(t, "https://api.example.com", resp2.Header.Get("Access-Control-Allow-Origin"), "Should match second rule")
+	assert.Contains(t, resp2.Header.Get("Access-Control-Allow-Methods"), "POST", "Should allow POST method")
+	assert.Contains(t, resp2.Header.Get("Access-Control-Allow-Headers"), "Authorization", "Should allow Authorization header")
+	assert.Equal(t, "7200", resp2.Header.Get("Access-Control-Max-Age"), "Should have second rule's max age")
+}
+
+// TestServiceLevelCORS tests that service-level endpoints (like /status) get proper CORS headers
+func TestServiceLevelCORS(t *testing.T) {
+	assert := assert.New(t)
+
+	endpoints := []string{
+		"/",
+		"/status",
+		"/healthz",
+	}
+
+	for _, endpoint := range endpoints {
+		t.Run(fmt.Sprintf("endpoint_%s", strings.ReplaceAll(endpoint, "/", "_")), func(t *testing.T) {
+			req, err := http.NewRequest("OPTIONS", fmt.Sprintf("%s%s", getDefaultConfig().Endpoint, endpoint), nil)
+			assert.NoError(err)
+
+			// Add Origin header to trigger CORS
+			req.Header.Set("Origin", "http://example.com")
+
+			client := &http.Client{}
+			resp, err := client.Do(req)
+			assert.NoError(err)
+			defer resp.Body.Close()
+
+			// Should return 200 OK
+			assert.Equal(http.StatusOK, resp.StatusCode)
+
+			// Should have CORS headers set
+			assert.Equal("*", resp.Header.Get("Access-Control-Allow-Origin"))
+			assert.Equal("*", resp.Header.Get("Access-Control-Expose-Headers"))
+			assert.Equal("*", resp.Header.Get("Access-Control-Allow-Methods"))
+			assert.Equal("*", resp.Header.Get("Access-Control-Allow-Headers"))
+		})
+	}
+}
+
+// TestServiceLevelCORSWithoutOrigin tests that service-level endpoints without Origin header don't get CORS headers
+func TestServiceLevelCORSWithoutOrigin(t *testing.T) {
+	assert := assert.New(t)
+
+	req, err := http.NewRequest("OPTIONS", fmt.Sprintf("%s/status", getDefaultConfig().Endpoint), nil)
+	assert.NoError(err)
+
+	// No Origin header
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	assert.NoError(err)
+	defer resp.Body.Close()
+
+	// Should return 200 OK
+	assert.Equal(http.StatusOK, resp.StatusCode)
+
+	// Should not have CORS headers set (or have empty values)
+	corsHeaders := []string{
+		"Access-Control-Allow-Origin",
+		"Access-Control-Expose-Headers",
+		"Access-Control-Allow-Methods",
+		"Access-Control-Allow-Headers",
+	}
+
+	for _, header := range corsHeaders {
+		value := resp.Header.Get(header)
+		// Headers should either be empty or not present
+		assert.True(value == "" || value == "*", "Header %s should be empty or wildcard, got: %s", header, value)
+	}
+}

test/s3/cors/s3_cors_test.go

@@ -0,0 +1,785 @@
+package cors
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/k0kubun/pp"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// S3TestConfig holds configuration for S3 tests
+type S3TestConfig struct {
+	Endpoint       string
+	MasterEndpoint string
+	AccessKey      string
+	SecretKey      string
+	Region         string
+	BucketPrefix   string
+	UseSSL         bool
+	SkipVerifySSL  bool
+}
+
+// allTestBucketPrefixes lists every prefix used to name buckets in this test suite.
+// cleanupAllTestBuckets uses it to find stale buckets from prior tests/runs.
+// Add the new prefix here whenever a test introduces one.
+var allTestBucketPrefixes = []string{
+	"test-cors-",
+}
+
+// testRunID uniquely identifies this `go test` invocation. It's embedded into
+// every bucket created by createTestBucket (after defaultConfig.BucketPrefix), so
+// the cleanup sweep can scope deletions to buckets that belong to this run only —
+// letting concurrent runs against the same endpoints coexist without tearing each
+// other's buckets down. Stale buckets left behind by a crashed prior run share the
+// family prefix but a different runID, so they are not swept here; `make clean`
+// (or wiping the data dir) is the right cleanup for that case.
+var testRunID = strconv.FormatInt(time.Now().UnixNano(), 36)
+
+// getDefaultConfig returns a fresh instance of the default test configuration
+// to avoid parallel test issues with global mutable state
+func getDefaultConfig() *S3TestConfig {
+	endpoint := os.Getenv("S3_ENDPOINT")
+	if endpoint == "" {
+		endpoint = "http://localhost:8333" // Default SeaweedFS S3 port
+	}
+	masterEndpoint := os.Getenv("MASTER_ENDPOINT")
+	if masterEndpoint == "" {
+		masterEndpoint = "http://127.0.0.1:9333" // Default SeaweedFS master HTTP port
+	}
+	return &S3TestConfig{
+		Endpoint:       endpoint,
+		MasterEndpoint: masterEndpoint,
+		AccessKey:      "some_access_key1",
+		SecretKey:      "some_secret_key1",
+		Region:         "us-east-1",
+		BucketPrefix:   "test-cors-",
+		UseSSL:         false,
+		SkipVerifySSL:  true,
+	}
+}
+
+// getS3Client creates an AWS S3 client for testing
+func getS3Client(t *testing.T) *s3.Client {
+	defaultConfig := getDefaultConfig()
+	cfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithRegion(defaultConfig.Region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+			defaultConfig.AccessKey,
+			defaultConfig.SecretKey,
+			"",
+		)),
+		config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
+			func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+				return aws.Endpoint{
+					URL:           defaultConfig.Endpoint,
+					SigningRegion: defaultConfig.Region,
+				}, nil
+			})),
+	)
+	require.NoError(t, err)
+
+	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+	})
+	return client
+}
+
+// createTestBucket creates a test bucket with a unique name
+func createTestBucket(t *testing.T, client *s3.Client) string {
+	defaultConfig := getDefaultConfig()
+	// Sweep stale buckets from prior tests in this run so each new bucket starts
+	// on a fresh slate. Without this, leaked collection volumes accumulate on a
+	// single `weed mini` data node and the suite eventually exhausts its volume
+	// slots.
+	cleanupAllTestBuckets(t, client)
+	bucketName := fmt.Sprintf("%sr%s-%d", defaultConfig.BucketPrefix, testRunID, time.Now().UnixNano())
+
+	_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+
+	// Wait for bucket metadata to be fully processed
+	time.Sleep(50 * time.Millisecond)
+
+	return bucketName
+}
+
+// cleanupTestBucket removes the test bucket and all its contents.
+// Always force-drops the underlying collection at the master afterwards: the S3
+// DeleteBucket can race with concurrent `volume_grow` requests (the warm-create
+// batch keeps registering volumes after the master's collection-delete sweep has
+// already snapshotted the layout), so 1-3 volumes per bucket can leak. Without
+// this, running enough tests on a single `weed mini` server exhausts the data
+// node's volume slots and every subsequent PutObject 500s with "Not enough data
+// nodes found".
+func cleanupTestBucket(t *testing.T, client *s3.Client, bucketName string) {
+	defer forceDeleteCollection(t, bucketName)
+	// First, delete all objects in the bucket
+	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+	})
+	if err == nil {
+		for _, obj := range listResp.Contents {
+			_, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    obj.Key,
+			})
+			if err != nil {
+				t.Logf("Warning: failed to delete object %s: %v", *obj.Key, err)
+			}
+		}
+	}
+
+	// Then delete the bucket
+	_, err = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		t.Logf("Warning: failed to delete bucket %s: %v", bucketName, err)
+	}
+}
+
+// forceDeleteCollection drops the SeaweedFS collection backing a test bucket via the master's
+// /col/delete admin endpoint. The S3 layer normally drops the collection on DeleteBucket, but
+// in-flight `volume_grow` requests can register volumes after the master's first sweep, leaking
+// them. Best-effort: a 400 from the master means the collection was already gone, which is the
+// success path and not an error.
+func forceDeleteCollection(t *testing.T, bucketName string) {
+	masterEndpoint := getDefaultConfig().MasterEndpoint
+	if masterEndpoint == "" {
+		return
+	}
+	endpoint := strings.TrimRight(masterEndpoint, "/") + "/col/delete?collection=" + url.QueryEscape(bucketName)
+	req, err := http.NewRequest(http.MethodGet, endpoint, nil)
+	if err != nil {
+		t.Logf("Note: building collection delete request for %s failed: %v", bucketName, err)
+		return
+	}
+	httpClient := &http.Client{Timeout: 5 * time.Second}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		t.Logf("Note: force-delete collection %s failed: %v", bucketName, err)
+		return
+	}
+	defer resp.Body.Close()
+	io.Copy(io.Discard, resp.Body)
+	switch resp.StatusCode {
+	case http.StatusNoContent:
+		t.Logf("Force-deleted collection %s", bucketName)
+	case http.StatusBadRequest:
+		// Collection already gone - normal path when DeleteBucket succeeded.
+	default:
+		t.Logf("Note: force-delete collection %s returned HTTP %d", bucketName, resp.StatusCode)
+	}
+}
+
+// cleanupAllTestBuckets cleans up any leftover test buckets owned by this run
+// (prefix + runID marker). Called from createTestBucket before each new bucket
+// creation so a single `weed mini` data node does not exhaust its volume slots
+// after many tests.
+func cleanupAllTestBuckets(t *testing.T, client *s3.Client) {
+	listResp, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{})
+	if err != nil {
+		t.Logf("Warning: failed to list buckets for cleanup: %v", err)
+		return
+	}
+
+	for _, bucket := range listResp.Buckets {
+		if bucket.Name == nil {
+			continue
+		}
+		for _, prefix := range allTestBucketPrefixes {
+			// Only sweep buckets owned by this run (prefix + runID marker).
+			// Buckets from concurrent runs carry a different runID and are skipped.
+			if strings.HasPrefix(*bucket.Name, prefix+"r"+testRunID+"-") {
+				t.Logf("Cleaning up leftover test bucket: %s", *bucket.Name)
+				cleanupTestBucket(t, client, *bucket.Name)
+				break
+			}
+		}
+	}
+}
+
+// TestCORSConfigurationManagement tests basic CORS configuration CRUD operations
+func TestCORSConfigurationManagement(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	// Test 1: Get CORS configuration when none exists (should return error)
+	_, err := client.GetBucketCors(context.TODO(), &s3.GetBucketCorsInput{
+		Bucket: aws.String(bucketName),
+	})
+	assert.Error(t, err, "Should get error when no CORS configuration exists")
+
+	// Test 2: Put CORS configuration
+	corsConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"*"},
+				AllowedMethods: []string{"GET", "POST", "PUT"},
+				AllowedOrigins: []string{"https://example.com"},
+				ExposeHeaders:  []string{"ETag"},
+				MaxAgeSeconds:  aws.Int32(3600),
+			},
+		},
+	}
+
+	_, err = client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: corsConfig,
+	})
+	require.NoError(t, err, "Should be able to put CORS configuration")
+
+	// Wait for metadata subscription to update cache
+	time.Sleep(50 * time.Millisecond)
+
+	// Test 3: Get CORS configuration
+	getResp, err := client.GetBucketCors(context.TODO(), &s3.GetBucketCorsInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err, "Should be able to get CORS configuration")
+	require.NotNil(t, getResp.CORSRules, "CORS configuration should not be nil")
+	require.Len(t, getResp.CORSRules, 1, "Should have one CORS rule")
+
+	rule := getResp.CORSRules[0]
+	assert.Equal(t, []string{"*"}, rule.AllowedHeaders, "Allowed headers should match")
+	assert.Equal(t, []string{"GET", "POST", "PUT"}, rule.AllowedMethods, "Allowed methods should match")
+	assert.Equal(t, []string{"https://example.com"}, rule.AllowedOrigins, "Allowed origins should match")
+	assert.Equal(t, []string{"ETag"}, rule.ExposeHeaders, "Expose headers should match")
+	assert.Equal(t, aws.Int32(3600), rule.MaxAgeSeconds, "Max age should match")
+
+	// Test 4: Update CORS configuration
+	updatedCorsConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"Content-Type"},
+				AllowedMethods: []string{"GET", "POST"},
+				AllowedOrigins: []string{"https://example.com", "https://another.com"},
+				ExposeHeaders:  []string{"ETag", "Content-Length"},
+				MaxAgeSeconds:  aws.Int32(7200),
+			},
+		},
+	}
+
+	_, err = client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: updatedCorsConfig,
+	})
+	require.NoError(t, err, "Should be able to update CORS configuration")
+
+	// Wait for CORS configuration update to be fully processed
+	time.Sleep(100 * time.Millisecond)
+
+	// Verify the update with retries for robustness
+	var updateSuccess bool
+	for i := 0; i < 3; i++ {
+		getResp, err = client.GetBucketCors(context.TODO(), &s3.GetBucketCorsInput{
+			Bucket: aws.String(bucketName),
+		})
+		if err != nil {
+			t.Logf("Attempt %d: Failed to get updated CORS config: %v", i+1, err)
+			time.Sleep(50 * time.Millisecond)
+			continue
+		}
+
+		if len(getResp.CORSRules) > 0 {
+			rule = getResp.CORSRules[0]
+			// Check if the update actually took effect
+			if len(rule.AllowedHeaders) > 0 && rule.AllowedHeaders[0] == "Content-Type" &&
+				len(rule.AllowedOrigins) > 1 {
+				updateSuccess = true
+				break
+			}
+		}
+		t.Logf("Attempt %d: CORS config not updated yet, retrying...", i+1)
+		time.Sleep(50 * time.Millisecond)
+	}
+
+	require.NoError(t, err, "Should be able to get updated CORS configuration")
+	require.True(t, updateSuccess, "CORS configuration should be updated after retries")
+	assert.Equal(t, []string{"Content-Type"}, rule.AllowedHeaders, "Updated allowed headers should match")
+	assert.Equal(t, []string{"https://example.com", "https://another.com"}, rule.AllowedOrigins, "Updated allowed origins should match")
+
+	// Test 5: Delete CORS configuration
+	_, err = client.DeleteBucketCors(context.TODO(), &s3.DeleteBucketCorsInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err, "Should be able to delete CORS configuration")
+
+	// Wait for deletion to be fully processed
+	time.Sleep(100 * time.Millisecond)
+
+	// Verify deletion - should get NoSuchCORSConfiguration error
+	_, err = client.GetBucketCors(context.TODO(), &s3.GetBucketCorsInput{
+		Bucket: aws.String(bucketName),
+	})
+
+	// Check that we get the expected error type
+	if err != nil {
+		// Log the error for debugging
+		t.Logf("Got expected error after CORS deletion: %v", err)
+		// Check if it's the correct error type (NoSuchCORSConfiguration)
+		errMsg := err.Error()
+		if !strings.Contains(errMsg, "NoSuchCORSConfiguration") && !strings.Contains(errMsg, "404") {
+			t.Errorf("Expected NoSuchCORSConfiguration error, got: %v", err)
+		}
+	} else {
+		// If no error, this might be a SeaweedFS implementation difference
+		// Some implementations might return empty config instead of error
+		t.Logf("CORS deletion test: No error returned - this may be implementation-specific behavior")
+	}
+}
+
+// TestCORSMultipleRules tests CORS configuration with multiple rules
+func TestCORSMultipleRules(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	// Create CORS configuration with multiple rules
+	corsConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"*"},
+				AllowedMethods: []string{"GET", "HEAD"},
+				AllowedOrigins: []string{"https://example.com"},
+				ExposeHeaders:  []string{"ETag"},
+				MaxAgeSeconds:  aws.Int32(3600),
+			},
+			{
+				AllowedHeaders: []string{"Content-Type", "Authorization"},
+				AllowedMethods: []string{"POST", "PUT", "DELETE"},
+				AllowedOrigins: []string{"https://app.example.com"},
+				ExposeHeaders:  []string{"ETag", "Content-Length"},
+				MaxAgeSeconds:  aws.Int32(7200),
+			},
+			{
+				AllowedHeaders: []string{"*"},
+				AllowedMethods: []string{"GET"},
+				AllowedOrigins: []string{"*"},
+				ExposeHeaders:  []string{"ETag"},
+				MaxAgeSeconds:  aws.Int32(1800),
+			},
+		},
+	}
+
+	_, err := client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: corsConfig,
+	})
+	require.NoError(t, err, "Should be able to put CORS configuration with multiple rules")
+
+	// Wait for CORS configuration to be fully processed
+	time.Sleep(100 * time.Millisecond)
+
+	// Get and verify the configuration with retries for robustness
+	var getResp *s3.GetBucketCorsOutput
+	var getErr error
+
+	// Retry getting CORS config up to 3 times to handle timing issues
+	for i := 0; i < 3; i++ {
+		getResp, getErr = client.GetBucketCors(context.TODO(), &s3.GetBucketCorsInput{
+			Bucket: aws.String(bucketName),
+		})
+		if getErr == nil {
+			break
+		}
+		t.Logf("Attempt %d: Failed to get multiple rules CORS config: %v", i+1, getErr)
+		time.Sleep(50 * time.Millisecond)
+	}
+
+	require.NoError(t, getErr, "Should be able to get CORS configuration after retries")
+	require.NotNil(t, getResp, "GetBucketCors response should not be nil")
+	require.Len(t, getResp.CORSRules, 3, "Should have three CORS rules")
+
+	// Verify first rule
+	rule1 := getResp.CORSRules[0]
+	assert.Equal(t, []string{"*"}, rule1.AllowedHeaders)
+	assert.Equal(t, []string{"GET", "HEAD"}, rule1.AllowedMethods)
+	assert.Equal(t, []string{"https://example.com"}, rule1.AllowedOrigins)
+
+	// Verify second rule
+	rule2 := getResp.CORSRules[1]
+	assert.Equal(t, []string{"Content-Type", "Authorization"}, rule2.AllowedHeaders)
+	assert.Equal(t, []string{"POST", "PUT", "DELETE"}, rule2.AllowedMethods)
+	assert.Equal(t, []string{"https://app.example.com"}, rule2.AllowedOrigins)
+
+	// Verify third rule
+	rule3 := getResp.CORSRules[2]
+	assert.Equal(t, []string{"*"}, rule3.AllowedHeaders)
+	assert.Equal(t, []string{"GET"}, rule3.AllowedMethods)
+	assert.Equal(t, []string{"*"}, rule3.AllowedOrigins)
+}
+
+// TestCORSValidation tests CORS configuration validation
+func TestCORSValidation(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	// Test invalid HTTP method
+	invalidMethodConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"*"},
+				AllowedMethods: []string{"INVALID_METHOD"},
+				AllowedOrigins: []string{"https://example.com"},
+			},
+		},
+	}
+
+	_, err := client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: invalidMethodConfig,
+	})
+	assert.Error(t, err, "Should get error for invalid HTTP method")
+
+	// Test empty origins
+	emptyOriginsConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"*"},
+				AllowedMethods: []string{"GET"},
+				AllowedOrigins: []string{},
+			},
+		},
+	}
+
+	_, err = client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: emptyOriginsConfig,
+	})
+	assert.Error(t, err, "Should get error for empty origins")
+
+	// Test negative MaxAge
+	negativeMaxAgeConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"*"},
+				AllowedMethods: []string{"GET"},
+				AllowedOrigins: []string{"https://example.com"},
+				MaxAgeSeconds:  aws.Int32(-1),
+			},
+		},
+	}
+
+	_, err = client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: negativeMaxAgeConfig,
+	})
+	assert.Error(t, err, "Should get error for negative MaxAge")
+}
+
+// TestCORSWithWildcards tests CORS configuration with wildcard patterns
+func TestCORSWithWildcards(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	// Create CORS configuration with wildcard patterns
+	corsConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"*"},
+				AllowedMethods: []string{"GET", "POST"},
+				AllowedOrigins: []string{"https://*.example.com"},
+				ExposeHeaders:  []string{"*"},
+				MaxAgeSeconds:  aws.Int32(3600),
+			},
+		},
+	}
+
+	_, err := client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: corsConfig,
+	})
+	require.NoError(t, err, "Should be able to put CORS configuration with wildcards")
+
+	// Wait for CORS configuration to be fully processed and available
+	time.Sleep(100 * time.Millisecond)
+
+	// Get and verify the configuration with retries for robustness
+	var getResp *s3.GetBucketCorsOutput
+	var getErr error
+
+	// Retry getting CORS config up to 3 times to handle timing issues
+	for i := 0; i < 3; i++ {
+		getResp, getErr = client.GetBucketCors(context.TODO(), &s3.GetBucketCorsInput{
+			Bucket: aws.String(bucketName),
+		})
+		if getErr == nil {
+			break
+		}
+		t.Logf("Attempt %d: Failed to get CORS config: %v", i+1, getErr)
+		time.Sleep(50 * time.Millisecond)
+	}
+
+	require.NoError(t, getErr, "Should be able to get CORS configuration after retries")
+	require.NotNil(t, getResp, "GetBucketCors response should not be nil")
+	require.Len(t, getResp.CORSRules, 1, "Should have one CORS rule")
+
+	rule := getResp.CORSRules[0]
+	require.NotNil(t, rule, "CORS rule should not be nil")
+	assert.Equal(t, []string{"*"}, rule.AllowedHeaders, "Wildcard headers should be preserved")
+	assert.Equal(t, []string{"https://*.example.com"}, rule.AllowedOrigins, "Wildcard origins should be preserved")
+	assert.Equal(t, []string{"*"}, rule.ExposeHeaders, "Wildcard expose headers should be preserved")
+}
+
+// TestCORSRuleLimit tests the maximum number of CORS rules
+func TestCORSRuleLimit(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	// Create CORS configuration with maximum allowed rules (100)
+	rules := make([]types.CORSRule, 100)
+	for i := 0; i < 100; i++ {
+		rules[i] = types.CORSRule{
+			AllowedHeaders: []string{"*"},
+			AllowedMethods: []string{"GET"},
+			AllowedOrigins: []string{fmt.Sprintf("https://example%d.com", i)},
+			MaxAgeSeconds:  aws.Int32(3600),
+		}
+	}
+
+	corsConfig := &types.CORSConfiguration{
+		CORSRules: rules,
+	}
+
+	_, err := client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: corsConfig,
+	})
+	assert.NoError(t, err, "Should be able to put CORS configuration with 100 rules")
+
+	// Try to add one more rule (should fail)
+	rules = append(rules, types.CORSRule{
+		AllowedHeaders: []string{"*"},
+		AllowedMethods: []string{"GET"},
+		AllowedOrigins: []string{"https://example101.com"},
+		MaxAgeSeconds:  aws.Int32(3600),
+	})
+
+	corsConfig.CORSRules = rules
+
+	_, err = client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: corsConfig,
+	})
+	assert.Error(t, err, "Should get error when exceeding maximum number of rules")
+}
+
+// TestCORSNonExistentBucket tests CORS operations on non-existent bucket
+func TestCORSNonExistentBucket(t *testing.T) {
+	client := getS3Client(t)
+	nonExistentBucket := "non-existent-bucket-cors-test"
+
+	// Test Get CORS on non-existent bucket
+	_, err := client.GetBucketCors(context.TODO(), &s3.GetBucketCorsInput{
+		Bucket: aws.String(nonExistentBucket),
+	})
+	assert.Error(t, err, "Should get error for non-existent bucket")
+
+	// Test Put CORS on non-existent bucket
+	corsConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"*"},
+				AllowedMethods: []string{"GET"},
+				AllowedOrigins: []string{"https://example.com"},
+			},
+		},
+	}
+
+	_, err = client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(nonExistentBucket),
+		CORSConfiguration: corsConfig,
+	})
+	assert.Error(t, err, "Should get error for non-existent bucket")
+
+	// Test Delete CORS on non-existent bucket
+	_, err = client.DeleteBucketCors(context.TODO(), &s3.DeleteBucketCorsInput{
+		Bucket: aws.String(nonExistentBucket),
+	})
+	assert.Error(t, err, "Should get error for non-existent bucket")
+}
+
+// TestCORSObjectOperations tests CORS behavior with object operations
+func TestCORSObjectOperations(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	// Set up CORS configuration
+	corsConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"*"},
+				AllowedMethods: []string{"GET", "POST", "PUT", "DELETE"},
+				AllowedOrigins: []string{"https://example.com"},
+				ExposeHeaders:  []string{"ETag", "Content-Length"},
+				MaxAgeSeconds:  aws.Int32(3600),
+			},
+		},
+	}
+
+	_, err := client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: corsConfig,
+	})
+	assert.NoError(t, err, "Should be able to put CORS configuration")
+
+	// Test putting an object (this should work normally)
+	objectKey := "test-object.txt"
+	objectContent := "Hello, CORS World!"
+
+	_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   strings.NewReader(objectContent),
+	})
+	assert.NoError(t, err, "Should be able to put object in CORS-enabled bucket")
+
+	// Test getting the object
+	getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	assert.NoError(t, err, "Should be able to get object from CORS-enabled bucket")
+	assert.NotNil(t, getResp.Body, "Object body should not be nil")
+
+	// Test deleting the object
+	_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	assert.NoError(t, err, "Should be able to delete object from CORS-enabled bucket")
+}
+
+// TestCORSCaching tests CORS configuration caching behavior
+func TestCORSCaching(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	// Set up initial CORS configuration
+	corsConfig1 := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"*"},
+				AllowedMethods: []string{"GET"},
+				AllowedOrigins: []string{"https://example.com"},
+				MaxAgeSeconds:  aws.Int32(3600),
+			},
+		},
+	}
+
+	_, err := client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: corsConfig1,
+	})
+	require.NoError(t, err, "Should be able to put initial CORS configuration")
+
+	// Get the configuration
+	getResp1, err := client.GetBucketCors(context.TODO(), &s3.GetBucketCorsInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err, "Should be able to get initial CORS configuration")
+	require.Len(t, getResp1.CORSRules, 1, "Should have one CORS rule")
+
+	// Update the configuration
+	corsConfig2 := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"Content-Type"},
+				AllowedMethods: []string{"GET", "POST"},
+				AllowedOrigins: []string{"https://example.com", "https://another.com"},
+				MaxAgeSeconds:  aws.Int32(7200),
+			},
+		},
+	}
+
+	_, err = client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: corsConfig2,
+	})
+	require.NoError(t, err, "Should be able to update CORS configuration")
+
+	// Get the updated configuration (should reflect the changes)
+	getResp2, err := client.GetBucketCors(context.TODO(), &s3.GetBucketCorsInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err, "Should be able to get updated CORS configuration")
+	require.Len(t, getResp2.CORSRules, 1, "Should have one CORS rule")
+
+	rule := getResp2.CORSRules[0]
+	assert.Equal(t, []string{"Content-Type"}, rule.AllowedHeaders, "Should have updated headers")
+	assert.Equal(t, []string{"GET", "POST"}, rule.AllowedMethods, "Should have updated methods")
+	assert.Equal(t, []string{"https://example.com", "https://another.com"}, rule.AllowedOrigins, "Should have updated origins")
+	assert.Equal(t, aws.Int32(7200), rule.MaxAgeSeconds, "Should have updated max age")
+}
+
+// TestCORSErrorHandling tests various error conditions
+func TestCORSErrorHandling(t *testing.T) {
+	client := getS3Client(t)
+	bucketName := createTestBucket(t, client)
+	defer cleanupTestBucket(t, client, bucketName)
+
+	// Test empty CORS configuration
+	emptyCorsConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{},
+	}
+
+	_, err := client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: emptyCorsConfig,
+	})
+	assert.Error(t, err, "Should get error for empty CORS configuration")
+
+	// Test nil CORS configuration
+	_, err = client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: nil,
+	})
+	assert.Error(t, err, "Should get error for nil CORS configuration")
+
+	// Test CORS rule with empty methods
+	emptyMethodsConfig := &types.CORSConfiguration{
+		CORSRules: []types.CORSRule{
+			{
+				AllowedHeaders: []string{"*"},
+				AllowedMethods: []string{},
+				AllowedOrigins: []string{"https://example.com"},
+			},
+		},
+	}
+
+	_, err = client.PutBucketCors(context.TODO(), &s3.PutBucketCorsInput{
+		Bucket:            aws.String(bucketName),
+		CORSConfiguration: emptyMethodsConfig,
+	})
+	assert.Error(t, err, "Should get error for empty methods")
+}
+
+// Debugging helper to pretty print responses
+func debugResponse(t *testing.T, title string, response interface{}) {
+	t.Logf("=== %s ===", title)
+	pp.Println(response)
+}

test/s3/cors/s3_test_config.json

@@ -0,0 +1,27 @@
+{
+    "identities": [
+        {
+            "name": "anonymous",
+            "actions": [
+                "Read",
+                "List"
+            ]
+        },
+        {
+            "name": "admin",
+            "credentials": [
+                {
+                    "accessKey": "some_access_key1",
+                    "secretKey": "some_secret_key1"
+                }
+            ],
+            "actions": [
+                "Admin",
+                "Read",
+                "List",
+                "Tagging",
+                "Write"
+            ]
+        }
+    ]
+}

test/s3/delete/Makefile

@@ -0,0 +1,98 @@
+# S3 delete regression test harness
+
+.PHONY: all test help build-weed check-deps start-server stop-server test-delete test-with-server clean logs
+
+WEED_BINARY := ../../../weed/weed_binary
+S3_PORT := 8333
+TEST_TIMEOUT := 10m
+SERVER_DIR := ./test-volume-data
+S3_CONFIG := ./test_s3.json
+
+.DEFAULT_GOAL := help
+
+all: test
+
+test: test-with-server
+
+help:
+	@echo "S3 delete regression test harness"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  build-weed      - Build the SeaweedFS binary"
+	@echo "  start-server    - Start SeaweedFS with S3 enabled"
+	@echo "  stop-server     - Stop the test server"
+	@echo "  test-delete     - Run delete regression tests"
+	@echo "  test-with-server - Start server, run tests, stop server"
+	@echo "  logs            - Show server logs"
+	@echo "  clean           - Remove test artifacts"
+
+build-weed:
+	@echo "Building SeaweedFS binary..."
+	@cd ../../../weed && go build -o weed_binary .
+	@chmod +x $(WEED_BINARY)
+
+check-deps: build-weed
+	@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1)
+	@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1)
+	@test -f $(S3_CONFIG) || (echo "S3 config not found at $(S3_CONFIG)" && exit 1)
+
+start-server: check-deps
+	@echo "Starting SeaweedFS server for delete regression tests..."
+	@rm -f weed-server.pid
+	@mkdir -p $(SERVER_DIR)
+	@$(WEED_BINARY) mini \
+		-dir=$(SERVER_DIR) \
+		-s3 \
+		-s3.port=$(S3_PORT) \
+		-s3.config=$(S3_CONFIG) \
+		-master.peers=none \
+		> weed-test.log 2>&1 & echo $$! > weed-server.pid
+	@for i in $$(seq 1 60); do \
+		if curl -sS -o /dev/null http://127.0.0.1:$(S3_PORT)/ >/dev/null 2>&1; then \
+			echo "SeaweedFS S3 server is ready on port $(S3_PORT)"; \
+			exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "SeaweedFS S3 server failed to start"; \
+	echo "=== Server logs ==="; \
+	test -f weed-test.log && cat weed-test.log || true; \
+	exit 1
+
+stop-server:
+	@echo "Stopping SeaweedFS server..."
+	@if [ -f weed-server.pid ]; then \
+		SERVER_PID=$$(cat weed-server.pid); \
+		kill -TERM $$SERVER_PID 2>/dev/null || true; \
+		for i in $$(seq 1 20); do \
+			if ! kill -0 $$SERVER_PID >/dev/null 2>&1; then \
+				break; \
+			fi; \
+			sleep 0.1; \
+		done; \
+		if kill -0 $$SERVER_PID >/dev/null 2>&1; then \
+			kill -KILL $$SERVER_PID 2>/dev/null || true; \
+		fi; \
+		rm -f weed-server.pid; \
+	fi
+
+test-delete: check-deps
+	@echo "Running delete regression tests..."
+	@go test -v -timeout=$(TEST_TIMEOUT) .
+
+test-with-server: start-server
+	@trap "$(MAKE) stop-server" EXIT; \
+	$(MAKE) test-delete || (echo "=== Last 100 lines of server logs ===" && tail -100 weed-test.log && exit 1)
+	@$(MAKE) stop-server
+
+logs:
+	@if test -f weed-test.log; then \
+		tail -f weed-test.log; \
+	else \
+		echo "No server log file found."; \
+	fi
+
+clean:
+	@$(MAKE) stop-server
+	@rm -f weed-test.log weed-server.pid
+	@rm -rf $(SERVER_DIR)

test/s3/delete/s3_conditional_delete_test.go

@@ -0,0 +1,135 @@
+package delete
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConditionalDeleteIfMatchOnLatestVersion(t *testing.T) {
+	client := getTestClient(t)
+	bucket := createTestBucket(t, client)
+	defer cleanupBucket(t, client, bucket)
+
+	key := "conditional-delete.txt"
+	putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+		Body:   bytes.NewReader([]byte("versioned body")),
+	})
+	require.NoError(t, err)
+	require.NotNil(t, putResp.ETag)
+
+	_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+		Bucket:  aws.String(bucket),
+		Key:     aws.String(key),
+		IfMatch: aws.String(`"not-the-current-etag"`),
+	})
+	require.Error(t, err, "DeleteObject should reject a mismatched If-Match header")
+
+	var apiErr smithy.APIError
+	if assert.True(t, errors.As(err, &apiErr), "Expected smithy API error for conditional delete") {
+		assert.Equal(t, "PreconditionFailed", apiErr.ErrorCode())
+	}
+
+	_, err = client.HeadObject(context.TODO(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	require.NoError(t, err, "Object should remain current after a failed conditional delete")
+
+	deleteResp, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+		Bucket:  aws.String(bucket),
+		Key:     aws.String(key),
+		IfMatch: putResp.ETag,
+	})
+	require.NoError(t, err)
+	require.NotNil(t, deleteResp.DeleteMarker)
+	assert.True(t, *deleteResp.DeleteMarker, "Successful conditional delete on a versioned bucket should create a delete marker")
+	require.NotNil(t, deleteResp.VersionId)
+
+	_, err = client.HeadObject(context.TODO(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	require.Error(t, err, "Delete marker should hide the current object after a successful conditional delete")
+}
+
+func TestConditionalMultiDeletePerObjectETag(t *testing.T) {
+	client := getTestClient(t)
+	bucket := createTestBucket(t, client)
+	defer cleanupBucket(t, client, bucket)
+
+	okKey := "delete-ok.txt"
+	failKey := "delete-fail.txt"
+
+	okPutResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(okKey),
+		Body:   bytes.NewReader([]byte("delete me")),
+	})
+	require.NoError(t, err)
+	require.NotNil(t, okPutResp.ETag)
+
+	_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(failKey),
+		Body:   bytes.NewReader([]byte("keep me")),
+	})
+	require.NoError(t, err)
+
+	deleteResp, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
+		Bucket: aws.String(bucket),
+		Delete: &types.Delete{
+			Objects: []types.ObjectIdentifier{
+				{
+					Key:  aws.String(okKey),
+					ETag: okPutResp.ETag,
+				},
+				{
+					Key:  aws.String(failKey),
+					ETag: aws.String(`"mismatched-etag"`),
+				},
+			},
+		},
+	})
+	require.NoError(t, err)
+	require.Len(t, deleteResp.Deleted, 1, "One object should satisfy its ETag precondition")
+	require.Len(t, deleteResp.Errors, 1, "One object should report a precondition failure")
+	deletedKeys := make([]string, 0, len(deleteResp.Deleted))
+	for _, deleted := range deleteResp.Deleted {
+		deletedKeys = append(deletedKeys, aws.ToString(deleted.Key))
+	}
+	assert.Contains(t, deletedKeys, okKey)
+
+	var matchedError *types.Error
+	for i := range deleteResp.Errors {
+		if aws.ToString(deleteResp.Errors[i].Key) == failKey {
+			matchedError = &deleteResp.Errors[i]
+			break
+		}
+	}
+	if assert.NotNil(t, matchedError, "Expected error entry for failed key") {
+		assert.Equal(t, "PreconditionFailed", aws.ToString(matchedError.Code))
+	}
+
+	_, err = client.HeadObject(context.TODO(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(okKey),
+	})
+	require.Error(t, err, "Successfully deleted key should no longer be current")
+
+	_, err = client.HeadObject(context.TODO(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(failKey),
+	})
+	require.NoError(t, err, "Object with mismatched ETag should remain untouched")
+}

test/s3/delete/s3_multi_delete_versioning_test.go

@@ -0,0 +1,164 @@
+package delete
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testEndpoint  = "http://localhost:8333"
+	testAccessKey = "admin"
+	testSecretKey = "admin"
+	testRegion    = "us-east-1"
+)
+
+func getTestClient(t *testing.T) *s3.Client {
+	cfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithRegion(testRegion),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+			testAccessKey,
+			testSecretKey,
+			"",
+		)),
+	)
+	require.NoError(t, err)
+
+	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.BaseEndpoint = aws.String(testEndpoint)
+		o.UsePathStyle = true
+	})
+
+	return client
+}
+
+func createTestBucket(t *testing.T, client *s3.Client) string {
+	bucketName := fmt.Sprintf("test-multi-delete-%d", time.Now().UnixNano())
+	_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+
+	_, err = client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
+		Bucket: aws.String(bucketName),
+		VersioningConfiguration: &types.VersioningConfiguration{
+			Status: types.BucketVersioningStatusEnabled,
+		},
+	})
+	require.NoError(t, err)
+
+	return bucketName
+}
+
+func cleanupBucket(t *testing.T, client *s3.Client, bucket string) {
+	listResp, _ := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucket),
+	})
+
+	if listResp != nil {
+		var objectsToDelete []types.ObjectIdentifier
+
+		for _, version := range listResp.Versions {
+			objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{
+				Key:       version.Key,
+				VersionId: version.VersionId,
+			})
+		}
+
+		for _, marker := range listResp.DeleteMarkers {
+			objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{
+				Key:       marker.Key,
+				VersionId: marker.VersionId,
+			})
+		}
+
+		if len(objectsToDelete) > 0 {
+			_, _ = client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
+				Bucket: aws.String(bucket),
+				Delete: &types.Delete{
+					Objects: objectsToDelete,
+					Quiet:   aws.Bool(false),
+				},
+			})
+		}
+	}
+
+	_, _ = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
+		Bucket: aws.String(bucket),
+	})
+}
+
+func TestVersioningMultiObjectDelete(t *testing.T) {
+	client := getTestClient(t)
+	bucket := createTestBucket(t, client)
+	defer cleanupBucket(t, client, bucket)
+
+	key := "key"
+	numVersions := 2
+	var versionIds []string
+
+	for i := 0; i < numVersions; i++ {
+		content := fmt.Sprintf("content-%d", i)
+		putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String(key),
+			Body:   bytes.NewReader([]byte(content)),
+		})
+		require.NoError(t, err)
+		require.NotNil(t, putResp.VersionId)
+		versionIds = append(versionIds, *putResp.VersionId)
+	}
+
+	assert.Len(t, versionIds, 2)
+
+	var objectsToDelete []types.ObjectIdentifier
+	for _, vid := range versionIds {
+		objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{
+			Key:       aws.String(key),
+			VersionId: aws.String(vid),
+		})
+	}
+
+	deleteResp, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
+		Bucket: aws.String(bucket),
+		Delete: &types.Delete{
+			Objects: objectsToDelete,
+		},
+	})
+	require.NoError(t, err)
+	t.Logf("Delete response: Deleted=%d, Errors=%d", len(deleteResp.Deleted), len(deleteResp.Errors))
+
+	listResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucket),
+	})
+	require.NoError(t, err)
+
+	if listResp.Versions != nil && len(listResp.Versions) > 0 {
+		t.Errorf("FAIL: Expected no versions, but found %d versions:", len(listResp.Versions))
+		for _, v := range listResp.Versions {
+			t.Logf("  - Key=%s, VersionId=%s, IsLatest=%v", *v.Key, *v.VersionId, v.IsLatest)
+		}
+	} else {
+		t.Logf("PASS: Versions correctly deleted")
+	}
+	assert.Nil(t, listResp.Versions, "Expected no versions after deletion")
+
+	deleteResp2, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
+		Bucket: aws.String(bucket),
+		Delete: &types.Delete{
+			Objects: objectsToDelete,
+		},
+	})
+	require.NoError(t, err)
+	assert.Empty(t, deleteResp2.Errors, "Idempotent delete should not return errors")
+}

test/s3/delete/test_s3.json

@@ -0,0 +1,20 @@
+{
+  "identities": [
+    {
+      "name": "admin",
+      "credentials": [
+        {
+          "accessKey": "admin",
+          "secretKey": "admin"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    }
+  ]
+}

test/s3/distributed_lock/distributed_lock_cluster_test.go

@@ -0,0 +1,632 @@
+package distributed_lock
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/seaweedfs/seaweedfs/test/testutil"
+	"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+const (
+	distributedLockTestRegion    = "us-east-1"
+	distributedLockTestAccessKey = "some_access_key1"
+	distributedLockTestSecretKey = "some_secret_key1"
+	distributedLockTestGroup     = "distributed-lock-it"
+)
+
+type distributedLockCluster struct {
+	t         testing.TB
+	baseDir   string
+	configDir string
+	logsDir   string
+	keepLogs  bool
+
+	weedBinary string
+	filerGroup string
+	s3Config   string
+
+	masterPort     int
+	masterGrpcPort int
+	volumePort     int
+	volumeGrpcPort int
+	filerPorts     []int
+	filerGrpcPorts []int
+	s3Ports        []int
+	s3GrpcPorts    []int
+
+	masterCmd *exec.Cmd
+	volumeCmd *exec.Cmd
+	filerCmds []*exec.Cmd
+	s3Cmds    []*exec.Cmd
+	logFiles  []*os.File
+
+	cleanupOnce sync.Once
+}
+
+type s3IdentityConfig struct {
+	Identities []s3Identity `json:"identities"`
+}
+
+type s3Identity struct {
+	Name        string         `json:"name"`
+	Credentials []s3Credential `json:"credentials,omitempty"`
+	Actions     []string       `json:"actions"`
+}
+
+type s3Credential struct {
+	AccessKey string `json:"accessKey"`
+	SecretKey string `json:"secretKey"`
+}
+
+func startDistributedLockCluster(t *testing.T) *distributedLockCluster {
+	t.Helper()
+
+	weedBinary, err := framework.FindOrBuildWeedBinary()
+	require.NoError(t, err, "resolve weed binary")
+
+	baseDir, err := os.MkdirTemp("", "seaweedfs_s3_distributed_lock_")
+	require.NoError(t, err, "create temp directory")
+
+	cluster := &distributedLockCluster{
+		t:          t,
+		baseDir:    baseDir,
+		configDir:  filepath.Join(baseDir, "config"),
+		logsDir:    filepath.Join(baseDir, "logs"),
+		keepLogs:   os.Getenv("S3_DISTRIBUTED_LOCK_KEEP_LOGS") == "1",
+		weedBinary: weedBinary,
+		filerGroup: distributedLockTestGroup,
+		filerCmds:  make([]*exec.Cmd, 0, 2),
+		s3Cmds:     make([]*exec.Cmd, 0, 2),
+	}
+	t.Cleanup(cluster.Stop)
+
+	dirs := []string{
+		cluster.configDir,
+		cluster.logsDir,
+		filepath.Join(baseDir, "master"),
+		filepath.Join(baseDir, "volume"),
+		filepath.Join(baseDir, "filer0"),
+		filepath.Join(baseDir, "filer1"),
+	}
+	for _, dir := range dirs {
+		require.NoError(t, os.MkdirAll(dir, 0o755), "create %s", dir)
+	}
+
+	ports, err := testutil.AllocatePorts(12)
+	require.NoError(t, err, "allocate ports")
+	cluster.masterPort = ports[0]
+	cluster.masterGrpcPort = ports[1]
+	cluster.volumePort = ports[2]
+	cluster.volumeGrpcPort = ports[3]
+	cluster.filerPorts = []int{ports[4], ports[6]}
+	cluster.filerGrpcPorts = []int{ports[5], ports[7]}
+	cluster.s3Ports = []int{ports[8], ports[10]}
+	cluster.s3GrpcPorts = []int{ports[9], ports[11]}
+
+	require.NoError(t, cluster.writeSecurityConfig(), "write security config")
+	require.NoError(t, cluster.writeS3Config(), "write s3 config")
+
+	require.NoError(t, cluster.startMaster(), "start master")
+	require.NoError(t, cluster.waitForHTTP("http://"+cluster.masterHTTPAddress()+"/dir/status", 30*time.Second), "wait for master\n%s", cluster.tailLog("master.log"))
+
+	require.NoError(t, cluster.startVolume(), "start volume")
+	require.NoError(t, cluster.waitForHTTP("http://"+cluster.volumeHTTPAddress()+"/status", 30*time.Second), "wait for volume\n%s", cluster.tailLog("volume.log"))
+	require.NoError(t, cluster.waitForTCP(cluster.volumeGRPCAddress(), 30*time.Second), "wait for volume grpc\n%s", cluster.tailLog("volume.log"))
+
+	for i := 0; i < 2; i++ {
+		require.NoError(t, cluster.startFiler(i), "start filer %d", i)
+		require.NoError(t, cluster.waitForTCP(cluster.filerGRPCAddress(i), 30*time.Second), "wait for filer %d grpc\n%s", i, cluster.tailLog(fmt.Sprintf("filer%d.log", i)))
+	}
+	require.NoError(t, cluster.waitForFilerCount(2, 30*time.Second), "wait for filer group registration")
+	require.NoError(t, cluster.waitForLockRingConverged(30*time.Second), "wait for lock ring convergence")
+
+	for i := 0; i < 2; i++ {
+		require.NoError(t, cluster.startS3(i), "start s3 %d", i)
+		client := cluster.newS3Client(t, cluster.s3Endpoint(i))
+		require.NoError(t, cluster.waitForS3Ready(client, 30*time.Second), "wait for s3 %d\n%s", i, cluster.tailLog(fmt.Sprintf("s3-%d.log", i)))
+	}
+
+	return cluster
+}
+
+func (c *distributedLockCluster) Stop() {
+	if c == nil {
+		return
+	}
+	c.cleanupOnce.Do(func() {
+		for i := len(c.s3Cmds) - 1; i >= 0; i-- {
+			stopProcess(c.s3Cmds[i])
+		}
+		for i := len(c.filerCmds) - 1; i >= 0; i-- {
+			stopProcess(c.filerCmds[i])
+		}
+		stopProcess(c.volumeCmd)
+		stopProcess(c.masterCmd)
+
+		for _, f := range c.logFiles {
+			_ = f.Close()
+		}
+
+		if !c.keepLogs && !c.t.Failed() {
+			_ = os.RemoveAll(c.baseDir)
+		} else if c.baseDir != "" {
+			c.t.Logf("distributed lock integration logs kept at %s", c.baseDir)
+		}
+	})
+}
+
+func (c *distributedLockCluster) masterHTTPAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
+}
+
+func (c *distributedLockCluster) masterGRPCAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterGrpcPort))
+}
+
+func (c *distributedLockCluster) volumeHTTPAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort))
+}
+
+func (c *distributedLockCluster) volumeGRPCAddress() string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort))
+}
+
+func (c *distributedLockCluster) filerServerAddress(index int) pb.ServerAddress {
+	return pb.NewServerAddress("127.0.0.1", c.filerPorts[index], c.filerGrpcPorts[index])
+}
+
+func (c *distributedLockCluster) filerGRPCAddress(index int) string {
+	return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.filerGrpcPorts[index]))
+}
+
+func (c *distributedLockCluster) s3Endpoint(index int) string {
+	return fmt.Sprintf("http://127.0.0.1:%d", c.s3Ports[index])
+}
+
+func (c *distributedLockCluster) startMaster() error {
+	logFile, err := c.openLog("master.log")
+	if err != nil {
+		return err
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"master",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port=" + strconv.Itoa(c.masterPort),
+		"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
+		"-mdir=" + filepath.Join(c.baseDir, "master"),
+		"-peers=none",
+		"-volumeSizeLimitMB=32",
+		"-defaultReplication=000",
+	}
+
+	c.masterCmd = exec.Command(c.weedBinary, args...)
+	c.masterCmd.Dir = c.baseDir
+	c.masterCmd.Stdout = logFile
+	c.masterCmd.Stderr = logFile
+	return c.masterCmd.Start()
+}
+
+func (c *distributedLockCluster) startVolume() error {
+	logFile, err := c.openLog("volume.log")
+	if err != nil {
+		return err
+	}
+
+	masterAddress := string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"volume",
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port=" + strconv.Itoa(c.volumePort),
+		"-port.grpc=" + strconv.Itoa(c.volumeGrpcPort),
+		"-dir=" + filepath.Join(c.baseDir, "volume"),
+		"-max=16",
+		"-master=" + masterAddress,
+		"-readMode=proxy",
+	}
+
+	c.volumeCmd = exec.Command(c.weedBinary, args...)
+	c.volumeCmd.Dir = c.baseDir
+	c.volumeCmd.Stdout = logFile
+	c.volumeCmd.Stderr = logFile
+	return c.volumeCmd.Start()
+}
+
+func (c *distributedLockCluster) startFiler(index int) error {
+	logFile, err := c.openLog(fmt.Sprintf("filer%d.log", index))
+	if err != nil {
+		return err
+	}
+
+	masterAddress := string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"filer",
+		"-master=" + masterAddress,
+		"-filerGroup=" + c.filerGroup,
+		"-ip=127.0.0.1",
+		"-ip.bind=127.0.0.1",
+		"-port=" + strconv.Itoa(c.filerPorts[index]),
+		"-port.grpc=" + strconv.Itoa(c.filerGrpcPorts[index]),
+		"-defaultStoreDir=" + filepath.Join(c.baseDir, fmt.Sprintf("filer%d", index)),
+	}
+
+	cmd := exec.Command(c.weedBinary, args...)
+	cmd.Dir = c.baseDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	c.filerCmds = append(c.filerCmds, cmd)
+	return nil
+}
+
+func (c *distributedLockCluster) startS3(index int) error {
+	logFile, err := c.openLog(fmt.Sprintf("s3-%d.log", index))
+	if err != nil {
+		return err
+	}
+
+	filers := []string{string(c.filerServerAddress(0)), string(c.filerServerAddress(1))}
+	if index%2 == 1 {
+		filers[0], filers[1] = filers[1], filers[0]
+	}
+
+	args := []string{
+		"-config_dir=" + c.configDir,
+		"s3",
+		"-ip.bind=127.0.0.1",
+		"-port=" + strconv.Itoa(c.s3Ports[index]),
+		"-port.grpc=" + strconv.Itoa(c.s3GrpcPorts[index]),
+		"-port.iceberg=0",
+		"-port.lance=0",
+		"-filer=" + strings.Join(filers, ","),
+		"-config=" + c.s3Config,
+		"-iam.readOnly=false",
+	}
+
+	cmd := exec.Command(c.weedBinary, args...)
+	cmd.Dir = c.baseDir
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	c.s3Cmds = append(c.s3Cmds, cmd)
+	return nil
+}
+
+func (c *distributedLockCluster) writeSecurityConfig() error {
+	return os.WriteFile(filepath.Join(c.configDir, "security.toml"), []byte("# generated for distributed lock integration tests\n"), 0o644)
+}
+
+func (c *distributedLockCluster) writeS3Config() error {
+	configPath := filepath.Join(c.configDir, "s3.json")
+	payload := s3IdentityConfig{
+		Identities: []s3Identity{
+			{
+				Name: "distributed-lock-admin",
+				Credentials: []s3Credential{
+					{
+						AccessKey: distributedLockTestAccessKey,
+						SecretKey: distributedLockTestSecretKey,
+					},
+				},
+				Actions: []string{"Admin", "Read", "List", "Tagging", "Write"},
+			},
+		},
+	}
+	data, err := json.MarshalIndent(payload, "", "  ")
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(configPath, data, 0o644); err != nil {
+		return err
+	}
+	c.s3Config = configPath
+	return nil
+}
+
+func (c *distributedLockCluster) newS3Client(t testing.TB, endpoint string) *s3.Client {
+	t.Helper()
+
+	cfg, err := config.LoadDefaultConfig(context.Background(),
+		config.WithRegion(distributedLockTestRegion),
+		config.WithRetryMaxAttempts(1),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+			distributedLockTestAccessKey,
+			distributedLockTestSecretKey,
+			"",
+		)),
+	)
+	require.NoError(t, err, "load aws config")
+
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.BaseEndpoint = aws.String(endpoint)
+		o.UsePathStyle = true
+	})
+}
+
+func (c *distributedLockCluster) waitForS3Ready(client *s3.Client, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		_, err := client.ListBuckets(ctx, &s3.ListBucketsInput{})
+		cancel()
+		if err == nil {
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("timed out waiting for s3 readiness")
+}
+
+func (c *distributedLockCluster) waitForFilerCount(expected int, timeout time.Duration) error {
+	conn, err := grpc.NewClient(c.masterGRPCAddress(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	client := master_pb.NewSeaweedClient(conn)
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
+			ClientType: "filer",
+			FilerGroup: c.filerGroup,
+		})
+		cancel()
+		if err == nil && len(resp.ClusterNodes) >= expected {
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("timed out waiting for %d filers in group %q", expected, c.filerGroup)
+}
+
+// waitForLockRingConverged verifies that both filers have a consistent view of the
+// lock ring by acquiring the same lock through each filer and checking mutual exclusion.
+// This guards against the window between master seeing both filers and the filers
+// actually receiving the LockRingUpdate broadcast (delayed by the stabilization timer).
+//
+// A single arbitrary key could false-pass: if the key's real primary is filer0 and
+// filer0 has a stale ring (only sees itself), it still grants correctly because it IS
+// the primary. To catch the stale ring we must also test a key whose real primary is
+// filer1. So we generate one test key per primary filer using the same consistent-hash
+// ring as production, and require mutual exclusion for all of them.
+func (c *distributedLockCluster) waitForLockRingConverged(timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+
+	owners := make([]pb.ServerAddress, 0, len(c.filerPorts))
+	for i := range c.filerPorts {
+		owners = append(owners, c.filerServerAddress(i))
+	}
+	ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
+	ring.SetServers(owners)
+
+	attempt := 0
+	for time.Now().Before(deadline) {
+		// Generate one unique test key per primary filer
+		testKeys := c.convergenceKeysPerPrimary(ring, owners, attempt)
+		attempt++
+
+		allConverged := true
+		for _, key := range testKeys {
+			converged, _ := c.checkLockMutualExclusion(key)
+			if !converged {
+				allConverged = false
+				break
+			}
+		}
+		if allConverged {
+			return nil
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	return fmt.Errorf("lock ring did not converge: both filers independently grant the same lock")
+}
+
+// convergenceKeysPerPrimary returns one lock key per distinct primary filer,
+// using the same consistent-hash ring as production routing.
+func (c *distributedLockCluster) convergenceKeysPerPrimary(ring *lock_manager.HashRing, owners []pb.ServerAddress, attempt int) []string {
+	keysByOwner := make(map[pb.ServerAddress]string, len(owners))
+	for i := 0; len(keysByOwner) < len(owners) && i < 1024; i++ {
+		candidate := fmt.Sprintf("convergence-%d-%d", attempt, i)
+		primary := ring.GetPrimary(candidate)
+		if _, exists := keysByOwner[primary]; !exists {
+			keysByOwner[primary] = candidate
+		}
+	}
+	keys := make([]string, 0, len(keysByOwner))
+	for _, k := range keysByOwner {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+// checkLockMutualExclusion acquires a lock via filer0, then tries the same lock via filer1.
+// Returns true if mutual exclusion holds (second attempt is denied).
+func (c *distributedLockCluster) checkLockMutualExclusion(testKey string) (bool, error) {
+	conn0, err := grpc.NewClient(c.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return false, err
+	}
+	defer conn0.Close()
+
+	conn1, err := grpc.NewClient(c.filerGRPCAddress(1), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return false, err
+	}
+	defer conn1.Close()
+
+	client0 := filer_pb.NewSeaweedFilerClient(conn0)
+	client1 := filer_pb.NewSeaweedFilerClient(conn1)
+
+	// Acquire lock via filer0
+	ctx0, cancel0 := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel0()
+	resp0, err := client0.DistributedLock(ctx0, &filer_pb.LockRequest{
+		Name:          testKey,
+		SecondsToLock: 5,
+		Owner:         "convergence-filer0",
+	})
+	if err != nil || resp0.RenewToken == "" {
+		return false, fmt.Errorf("filer0 lock failed: err=%v resp=%v", err, resp0)
+	}
+	defer func() {
+		// Always release the lock we acquired
+		client0.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
+			Name:       testKey,
+			RenewToken: resp0.RenewToken,
+		})
+	}()
+
+	// Try the same lock via filer1 - should be denied
+	ctx1, cancel1 := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel1()
+	resp1, err := client1.DistributedLock(ctx1, &filer_pb.LockRequest{
+		Name:          testKey,
+		SecondsToLock: 5,
+		Owner:         "convergence-filer1",
+	})
+	if err != nil {
+		return false, err
+	}
+	if resp1.RenewToken != "" {
+		// Both filers granted the lock - ring not converged. Release the second lock.
+		client1.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
+			Name:       testKey,
+			RenewToken: resp1.RenewToken,
+		})
+		return false, nil
+	}
+	// Verify the denial is specifically because the lock is already held,
+	// not due to a transient error that might give a false positive.
+	if !strings.Contains(resp1.Error, "lock already owned") {
+		return false, nil
+	}
+	return true, nil
+}
+
+func (c *distributedLockCluster) waitForHTTP(url string, timeout time.Duration) error {
+	client := &net.Dialer{Timeout: time.Second}
+	httpClient := &httpClientWithDialer{dialer: client}
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if err := httpClient.Get(url); err == nil {
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("timed out waiting for http %s", url)
+}
+
+func (c *distributedLockCluster) waitForTCP(addr string, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", addr, time.Second)
+		if err == nil {
+			_ = conn.Close()
+			return nil
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return fmt.Errorf("timed out waiting for tcp %s", addr)
+}
+
+func (c *distributedLockCluster) openLog(name string) (*os.File, error) {
+	f, err := os.Create(filepath.Join(c.logsDir, name))
+	if err != nil {
+		return nil, err
+	}
+	c.logFiles = append(c.logFiles, f)
+	return f, nil
+}
+
+func (c *distributedLockCluster) tailLog(name string) string {
+	f, err := os.Open(filepath.Join(c.logsDir, name))
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	lines := make([]string, 0, 40)
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+		if len(lines) > 40 {
+			lines = lines[1:]
+		}
+	}
+	return strings.Join(lines, "\n")
+}
+
+func stopProcess(cmd *exec.Cmd) {
+	if cmd == nil || cmd.Process == nil {
+		return
+	}
+
+	_ = cmd.Process.Signal(os.Interrupt)
+	done := make(chan error, 1)
+	go func() {
+		done <- cmd.Wait()
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(10 * time.Second):
+		_ = cmd.Process.Kill()
+		<-done
+	}
+}
+
+type httpClientWithDialer struct {
+	dialer *net.Dialer
+}
+
+func (h *httpClientWithDialer) Get(url string) error {
+	client := &http.Client{
+		Timeout: time.Second,
+		Transport: &http.Transport{
+			DialContext: h.dialer.DialContext,
+		},
+	}
+	resp, err := client.Get(url)
+	if err != nil {
+		return err
+	}
+	_ = resp.Body.Close()
+	return nil
+}

test/s3/distributed_lock/distributed_lock_test.go

@@ -0,0 +1,170 @@
+package distributed_lock
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/smithy-go"
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConditionalPutIfNoneMatchDistributedLockAcrossS3Gateways(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping distributed lock integration test in short mode")
+	}
+
+	cluster := startDistributedLockCluster(t)
+	clientA := cluster.newS3Client(t, cluster.s3Endpoint(0))
+	clientB := cluster.newS3Client(t, cluster.s3Endpoint(1))
+
+	bucket := fmt.Sprintf("distributed-lock-%d", time.Now().UnixNano())
+	_, err := clientA.CreateBucket(context.Background(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucket),
+	})
+	require.NoError(t, err)
+
+	require.Eventually(t, func() bool {
+		_, err := clientB.HeadBucket(context.Background(), &s3.HeadBucketInput{
+			Bucket: aws.String(bucket),
+		})
+		return err == nil
+	}, 30*time.Second, 200*time.Millisecond, "bucket should replicate to the second filer-backed gateway")
+
+	keysByOwner := cluster.findLockOwnerKeys(bucket, "conditional-put")
+	require.Len(t, keysByOwner, len(cluster.filerPorts), "should exercise both filer lock owners")
+
+	for owner, key := range keysByOwner {
+		owner := owner
+		key := key
+		t.Run(lockOwnerLabel(owner), func(t *testing.T) {
+			runConditionalPutRace(t, []s3RaceClient{
+				{name: "s3-a", client: clientA},
+				{name: "s3-b", client: clientB},
+			}, bucket, key)
+		})
+	}
+}
+
+type s3RaceClient struct {
+	name   string
+	client *s3.Client
+}
+
+type putAttemptResult struct {
+	clientName string
+	body       string
+	err        error
+}
+
+func runConditionalPutRace(t *testing.T, clients []s3RaceClient, bucket, key string) {
+	t.Helper()
+
+	start := make(chan struct{})
+	results := make(chan putAttemptResult, len(clients))
+	var wg sync.WaitGroup
+
+	for _, client := range clients {
+		wg.Add(1)
+		body := fmt.Sprintf("%s-race", client.name)
+		go func(client s3RaceClient, body string) {
+			defer wg.Done()
+			<-start
+
+			ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+			defer cancel()
+
+			_, err := client.client.PutObject(ctx, &s3.PutObjectInput{
+				Bucket:      aws.String(bucket),
+				Key:         aws.String(key),
+				IfNoneMatch: aws.String("*"),
+				Body:        bytes.NewReader([]byte(body)),
+			})
+			results <- putAttemptResult{
+				clientName: client.name,
+				body:       body,
+				err:        err,
+			}
+		}(client, body)
+	}
+
+	close(start)
+	wg.Wait()
+	close(results)
+
+	successes := 0
+	preconditionFailures := 0
+	winnerBody := ""
+	unexpectedErrors := make([]string, 0)
+
+	for result := range results {
+		if result.err == nil {
+			successes++
+			winnerBody = result.body
+			continue
+		}
+		if isPreconditionFailed(result.err) {
+			preconditionFailures++
+			continue
+		}
+		unexpectedErrors = append(unexpectedErrors, fmt.Sprintf("%s: %v", result.clientName, result.err))
+	}
+
+	require.Empty(t, unexpectedErrors, "unexpected race errors")
+	require.Equal(t, 1, successes, "exactly one write should win")
+	require.Equal(t, len(clients)-1, preconditionFailures, "all losing writes should fail with 412")
+
+	object, err := clients[0].client.GetObject(context.Background(), &s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	require.NoError(t, err)
+	defer object.Body.Close()
+
+	data, err := io.ReadAll(object.Body)
+	require.NoError(t, err)
+	assert.Equal(t, winnerBody, string(data), "stored object body should match the successful request")
+}
+
+func isPreconditionFailed(err error) bool {
+	var apiErr smithy.APIError
+	return errors.As(err, &apiErr) && apiErr.ErrorCode() == "PreconditionFailed"
+}
+
+func (c *distributedLockCluster) findLockOwnerKeys(bucket, prefix string) map[pb.ServerAddress]string {
+	owners := make([]pb.ServerAddress, 0, len(c.filerPorts))
+	for i := range c.filerPorts {
+		owners = append(owners, c.filerServerAddress(i))
+	}
+
+	ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
+	ring.SetServers(owners)
+
+	keysByOwner := make(map[pb.ServerAddress]string, len(owners))
+	for i := 0; i < 1024 && len(keysByOwner) < len(owners); i++ {
+		key := fmt.Sprintf("%s-%03d.txt", prefix, i)
+		lockKey := fmt.Sprintf("s3.object.write:/buckets/%s/%s", bucket, s3_constants.NormalizeObjectKey(key))
+		lockOwner := ring.GetPrimary(lockKey)
+		if _, exists := keysByOwner[lockOwner]; !exists {
+			keysByOwner[lockOwner] = key
+		}
+	}
+	return keysByOwner
+}
+
+func lockOwnerLabel(owner pb.ServerAddress) string {
+	replacer := strings.NewReplacer(":", "_", ".", "_")
+	return "owner_" + replacer.Replace(string(owner))
+}

test/s3/etag/Makefile

@@ -0,0 +1,50 @@
+# ETag Format Integration Tests
+#
+# These tests verify S3 ETag format compatibility, particularly for large files
+# that trigger SeaweedFS auto-chunking. This addresses GitHub Issue #7768.
+#
+# Prerequisites:
+#   - SeaweedFS running with S3 API enabled on port 8333
+#   - Go 1.21+
+#
+# Usage:
+#   make test          - Run all tests
+#   make test-large    - Run only large file tests
+#   make test-verbose  - Run with verbose output
+#   make clean         - Clean test artifacts
+
+.PHONY: all test test-large test-verbose test-quick clean help
+
+# Default S3 endpoint
+S3_ENDPOINT ?= http://127.0.0.1:8333
+
+all: test
+
+test:
+	@echo "Running ETag format tests against $(S3_ENDPOINT)..."
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m ./...
+
+test-large:
+	@echo "Running large file ETag tests..."
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m -run "LargeFile|ExtraLarge" ./...
+
+test-verbose:
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m -count=1 ./...
+
+test-quick:
+	@echo "Running quick ETag tests (small files only)..."
+	S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 1m -run "SmallFile|Consistency" ./...
+
+clean:
+	go clean -testcache
+
+help:
+	@echo "ETag Format Integration Tests"
+	@echo "Targets:"
+	@echo "  test          Run all ETag format tests"
+	@echo "  test-large    Run only large file tests (>8MB)"
+	@echo "  test-quick    Run quick tests (small files only)"
+	@echo "  test-verbose  Run with verbose output"
+	@echo "  clean         Clean test cache"
+	@echo "Environment Variables:"
+	@echo "  S3_ENDPOINT   S3 endpoint URL (default: http://127.0.0.1:8333)"

test/s3/etag/README.md

@@ -0,0 +1,93 @@
+# S3 ETag Format Integration Tests
+
+This test suite verifies that SeaweedFS returns correct ETag formats for S3 operations, ensuring compatibility with AWS S3 SDKs.
+
+## Background
+
+**GitHub Issue #7768**: AWS S3 SDK for Java v2 was failing with `Invalid base 16 character: '-'` when performing `PutObject` on large files.
+
+### Root Cause
+
+SeaweedFS internally auto-chunks large files (>8MB) for efficient storage. Previously, when a regular `PutObject` request resulted in multiple internal chunks, SeaweedFS returned a composite ETag format (`<md5>-<count>`) instead of a pure MD5 hash.
+
+### AWS S3 Specification
+
+| Operation | ETag Format | Example |
+|-----------|-------------|---------|
+| PutObject (any size) | Pure MD5 hex (32 chars) | `d41d8cd98f00b204e9800998ecf8427e` |
+| CompleteMultipartUpload | Composite (`<md5>-<partcount>`) | `d41d8cd98f00b204e9800998ecf8427e-3` |
+
+AWS S3 SDK v2 for Java validates `PutObject` ETags as hexadecimal, which fails when the ETag contains a hyphen.
+
+## Test Coverage
+
+| Test | File Size | Purpose |
+|------|-----------|---------|
+| `TestPutObjectETagFormat_SmallFile` | 1KB | Verify single-chunk uploads return pure MD5 |
+| `TestPutObjectETagFormat_LargeFile` | 10MB | **Critical**: Verify auto-chunked uploads return pure MD5 |
+| `TestPutObjectETagFormat_ExtraLargeFile` | 25MB | Verify multi-chunk auto-chunked uploads return pure MD5 |
+| `TestMultipartUploadETagFormat` | 15MB | Verify multipart uploads correctly return composite ETag |
+| `TestPutObjectETagConsistency` | Various | Verify ETag consistency across PUT/HEAD/GET |
+| `TestETagHexValidation` | 10MB | Simulate AWS SDK v2 hex validation |
+| `TestMultipleLargeFileUploads` | 10MB x5 | Stress test multiple large uploads |
+
+## Prerequisites
+
+1. SeaweedFS running with S3 API enabled:
+   ```bash
+   weed server -s3
+   ```
+
+2. Go 1.21 or later
+
+3. AWS SDK v2 for Go (installed via go modules)
+
+## Running Tests
+
+```bash
+# Run all tests
+make test
+
+# Run only large file tests (the critical ones for issue #7768)
+make test-large
+
+# Run quick tests (small files only)
+make test-quick
+
+# Run with verbose output
+make test-verbose
+```
+
+## Configuration
+
+By default, tests connect to `http://127.0.0.1:8333`. To use a different endpoint:
+
+```bash
+S3_ENDPOINT=http://localhost:8333 make test
+```
+
+Or modify `defaultConfig` in `s3_etag_test.go`.
+
+## SDK Compatibility
+
+These tests use **AWS SDK v2 for Go**, which has the same ETag validation behavior as AWS SDK v2 for Java. The tests include:
+
+- ETag format validation (pure MD5 vs composite)
+- Hex decoding validation (simulates `Base16Codec.decode`)
+- Content integrity verification
+
+## Validated SDK Versions
+
+| SDK | Version | Status |
+|-----|---------|--------|
+| AWS SDK v2 for Go | 1.20+ | ✅ Tested |
+| AWS SDK v2 for Java | 2.20+ | ✅ Compatible (issue #7768 fixed) |
+| AWS SDK v1 for Go | 1.x | ✅ Compatible (less strict validation) |
+| AWS SDK v1 for Java | 1.x | ✅ Compatible (less strict validation) |
+
+## Related
+
+- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
+- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
+
+

test/s3/etag/s3_etag_test.go

@@ -0,0 +1,644 @@
+// Package etag_test provides integration tests for S3 ETag format validation.
+//
+// These tests verify that SeaweedFS returns correct ETag formats for different
+// upload scenarios, ensuring compatibility with AWS S3 SDKs that validate ETags.
+//
+// Background (GitHub Issue #7768):
+// AWS S3 SDK for Java v2 validates ETags as hexadecimal MD5 hashes for PutObject
+// responses. SeaweedFS was incorrectly returning composite ETags ("<md5>-<count>")
+// for regular PutObject when files were internally auto-chunked (>8MB), causing
+// the SDK to fail with "Invalid base 16 character: '-'".
+//
+// Per AWS S3 specification:
+// - Regular PutObject: ETag is always a pure MD5 hex string (32 chars)
+// - Multipart Upload (CompleteMultipartUpload): ETag is "<md5>-<partcount>"
+//
+// These tests ensure this behavior is maintained.
+package etag_test
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io"
+	mathrand "math/rand"
+	"os"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// S3TestConfig holds configuration for S3 tests
+type S3TestConfig struct {
+	Endpoint     string
+	AccessKey    string
+	SecretKey    string
+	Region       string
+	BucketPrefix string
+}
+
+// Default test configuration
+var defaultConfig = &S3TestConfig{
+	Endpoint:     "http://127.0.0.1:8333",
+	AccessKey:    "some_access_key1",
+	SecretKey:    "some_secret_key1",
+	Region:       "us-east-1",
+	BucketPrefix: "test-etag-",
+}
+
+// Constants for auto-chunking thresholds (must match s3api_object_handlers_put.go)
+const (
+	// SeaweedFS auto-chunks files larger than 8MB
+	autoChunkSize = 8 * 1024 * 1024
+
+	// Test sizes
+	smallFileSize  = 1 * 1024         // 1KB - single chunk
+	mediumFileSize = 256 * 1024       // 256KB - single chunk (at threshold)
+	largeFileSize  = 10 * 1024 * 1024 // 10MB - triggers auto-chunking (2 chunks)
+	xlFileSize     = 25 * 1024 * 1024 // 25MB - triggers auto-chunking (4 chunks)
+	multipartSize  = 5 * 1024 * 1024  // 5MB per part for multipart uploads
+)
+
+// ETag format patterns
+var (
+	// Pure MD5 ETag: 32 hex characters (with or without quotes)
+	pureMD5Pattern = regexp.MustCompile(`^"?[a-f0-9]{32}"?$`)
+
+	// Composite ETag for multipart: 32 hex chars, hyphen, part count (with or without quotes)
+	compositePattern = regexp.MustCompile(`^"?[a-f0-9]{32}-\d+"?$`)
+)
+
+func init() {
+	mathrand.Seed(time.Now().UnixNano())
+}
+
+// getS3Client creates an AWS S3 v2 client for testing
+func getS3Client(t *testing.T) *s3.Client {
+	endpoint := os.Getenv("S3_ENDPOINT")
+	if endpoint == "" {
+		endpoint = defaultConfig.Endpoint
+	}
+	t.Logf("Using S3 endpoint: %s", endpoint)
+
+	cfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithRegion(defaultConfig.Region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+			defaultConfig.AccessKey,
+			defaultConfig.SecretKey,
+			"",
+		)),
+		config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
+			func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+				return aws.Endpoint{
+					URL:               endpoint,
+					SigningRegion:     defaultConfig.Region,
+					HostnameImmutable: true,
+				}, nil
+			})),
+	)
+	require.NoError(t, err)
+
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+	})
+}
+
+// getNewBucketName generates a unique bucket name
+func getNewBucketName() string {
+	timestamp := time.Now().UnixNano()
+	randomSuffix := mathrand.Intn(100000)
+	return fmt.Sprintf("%s%d-%d", defaultConfig.BucketPrefix, timestamp, randomSuffix)
+}
+
+// generateRandomData generates random test data of specified size
+func generateRandomData(size int) []byte {
+	data := make([]byte, size)
+	if _, err := rand.Read(data); err != nil {
+		panic(fmt.Sprintf("failed to generate random test data: %v", err))
+	}
+	return data
+}
+
+// calculateMD5 calculates the MD5 hash of data and returns hex string
+func calculateMD5(data []byte) string {
+	hash := md5.Sum(data)
+	return hex.EncodeToString(hash[:])
+}
+
+// cleanETag removes quotes from ETag
+func cleanETag(etag string) string {
+	return strings.Trim(etag, `"`)
+}
+
+// isPureMD5ETag checks if ETag is a pure MD5 hex string (no composite format)
+func isPureMD5ETag(etag string) bool {
+	return pureMD5Pattern.MatchString(etag)
+}
+
+// isCompositeETag checks if ETag is in composite format (md5-count)
+func isCompositeETag(etag string) bool {
+	return compositePattern.MatchString(etag)
+}
+
+// createTestBucket creates a new bucket for testing
+func createTestBucket(ctx context.Context, client *s3.Client, bucketName string) error {
+	_, err := client.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	return err
+}
+
+// cleanupTestBucket deletes all objects and the bucket
+func cleanupTestBucket(ctx context.Context, client *s3.Client, bucketName string) {
+	// Delete all objects
+	paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+	})
+
+	for paginator.HasMorePages() {
+		page, err := paginator.NextPage(ctx)
+		if err != nil {
+			break
+		}
+		for _, obj := range page.Contents {
+			client.DeleteObject(ctx, &s3.DeleteObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    obj.Key,
+			})
+		}
+	}
+
+	// Abort any in-progress multipart uploads
+	mpPaginator := s3.NewListMultipartUploadsPaginator(client, &s3.ListMultipartUploadsInput{
+		Bucket: aws.String(bucketName),
+	})
+	for mpPaginator.HasMorePages() {
+		page, err := mpPaginator.NextPage(ctx)
+		if err != nil {
+			break
+		}
+		for _, upload := range page.Uploads {
+			client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
+				Bucket:   aws.String(bucketName),
+				Key:      upload.Key,
+				UploadId: upload.UploadId,
+			})
+		}
+	}
+
+	// Delete bucket
+	client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+}
+
+// TestPutObjectETagFormat_SmallFile verifies ETag format for small files (single chunk)
+func TestPutObjectETagFormat_SmallFile(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testData := generateRandomData(smallFileSize)
+	expectedMD5 := calculateMD5(testData)
+	objectKey := "small-file.bin"
+
+	// Upload small file
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload small file")
+
+	// Verify ETag format
+	etag := aws.ToString(putResp.ETag)
+	t.Logf("Small file (%d bytes) ETag: %s", smallFileSize, etag)
+
+	assert.True(t, isPureMD5ETag(etag),
+		"Small file ETag should be pure MD5, got: %s", etag)
+	assert.False(t, isCompositeETag(etag),
+		"Small file ETag should NOT be composite format, got: %s", etag)
+	assert.Equal(t, expectedMD5, cleanETag(etag),
+		"ETag should match calculated MD5")
+}
+
+// TestPutObjectETagFormat_LargeFile verifies ETag format for large files that trigger auto-chunking
+// This is the critical test for GitHub Issue #7768
+func TestPutObjectETagFormat_LargeFile(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testData := generateRandomData(largeFileSize)
+	expectedMD5 := calculateMD5(testData)
+	objectKey := "large-file.bin"
+
+	t.Logf("Uploading large file (%d bytes, > %d byte auto-chunk threshold)...",
+		largeFileSize, autoChunkSize)
+
+	// Upload large file (triggers auto-chunking internally)
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload large file")
+
+	// Verify ETag format - MUST be pure MD5, NOT composite
+	etag := aws.ToString(putResp.ETag)
+	t.Logf("Large file (%d bytes, ~%d internal chunks) ETag: %s",
+		largeFileSize, (largeFileSize/autoChunkSize)+1, etag)
+
+	assert.True(t, isPureMD5ETag(etag),
+		"Large file PutObject ETag MUST be pure MD5 (not composite), got: %s", etag)
+	assert.False(t, isCompositeETag(etag),
+		"Large file PutObject ETag should NOT contain '-' (composite format), got: %s", etag)
+	assert.False(t, strings.Contains(cleanETag(etag), "-"),
+		"ETag should not contain hyphen for regular PutObject, got: %s", etag)
+	assert.Equal(t, expectedMD5, cleanETag(etag),
+		"ETag should match calculated MD5 of entire content")
+
+	// Verify we can read back the object correctly
+	getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to get large file")
+	defer getResp.Body.Close()
+
+	downloadedData, err := io.ReadAll(getResp.Body)
+	require.NoError(t, err, "Failed to read large file content")
+	assert.Equal(t, testData, downloadedData, "Downloaded content should match uploaded content")
+}
+
+// TestPutObjectETagFormat_ExtraLargeFile tests even larger files with multiple internal chunks
+func TestPutObjectETagFormat_ExtraLargeFile(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testData := generateRandomData(xlFileSize)
+	expectedMD5 := calculateMD5(testData)
+	objectKey := "xl-file.bin"
+
+	expectedChunks := (xlFileSize / autoChunkSize) + 1
+	t.Logf("Uploading XL file (%d bytes, expected ~%d internal chunks)...",
+		xlFileSize, expectedChunks)
+
+	// Upload extra large file
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err, "Failed to upload XL file")
+
+	// Verify ETag format
+	etag := aws.ToString(putResp.ETag)
+	t.Logf("XL file (%d bytes) ETag: %s", xlFileSize, etag)
+
+	assert.True(t, isPureMD5ETag(etag),
+		"XL file PutObject ETag MUST be pure MD5, got: %s", etag)
+	assert.False(t, isCompositeETag(etag),
+		"XL file PutObject ETag should NOT be composite, got: %s", etag)
+	assert.Equal(t, expectedMD5, cleanETag(etag),
+		"ETag should match calculated MD5")
+}
+
+// TestMultipartUploadETagFormat verifies that ONLY multipart uploads get composite ETags
+func TestMultipartUploadETagFormat(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	// Create test data for multipart upload (15MB = 3 parts of 5MB each)
+	totalSize := 15 * 1024 * 1024
+	testData := generateRandomData(totalSize)
+	objectKey := "multipart-file.bin"
+
+	expectedPartCount := (totalSize + multipartSize - 1) / multipartSize // ceiling division
+	t.Logf("Performing multipart upload (%d bytes, %d parts)...",
+		totalSize, expectedPartCount)
+
+	// Initiate multipart upload
+	createResp, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err, "Failed to create multipart upload")
+
+	uploadId := createResp.UploadId
+	var completedParts []types.CompletedPart
+	partNumber := int32(1)
+
+	// Upload parts
+	for offset := 0; offset < totalSize; offset += multipartSize {
+		end := offset + multipartSize
+		if end > totalSize {
+			end = totalSize
+		}
+		partData := testData[offset:end]
+
+		uploadResp, err := client.UploadPart(ctx, &s3.UploadPartInput{
+			Bucket:     aws.String(bucketName),
+			Key:        aws.String(objectKey),
+			UploadId:   uploadId,
+			PartNumber: aws.Int32(partNumber),
+			Body:       bytes.NewReader(partData),
+		})
+		require.NoError(t, err, "Failed to upload part %d", partNumber)
+
+		completedParts = append(completedParts, types.CompletedPart{
+			ETag:       uploadResp.ETag,
+			PartNumber: aws.Int32(partNumber),
+		})
+		partNumber++
+	}
+
+	// Complete multipart upload
+	completeResp, err := client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: uploadId,
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: completedParts,
+		},
+	})
+	require.NoError(t, err, "Failed to complete multipart upload")
+
+	// Verify ETag format - SHOULD be composite for multipart
+	etag := aws.ToString(completeResp.ETag)
+	t.Logf("Multipart upload ETag: %s", etag)
+
+	assert.True(t, isCompositeETag(etag),
+		"Multipart upload ETag SHOULD be composite format (md5-count), got: %s", etag)
+	assert.True(t, strings.Contains(cleanETag(etag), "-"),
+		"Multipart ETag should contain hyphen, got: %s", etag)
+
+	// Verify the part count in the ETag matches
+	parts := strings.Split(cleanETag(etag), "-")
+	require.Len(t, parts, 2, "Composite ETag should have format 'hash-count'")
+	assert.Equal(t, fmt.Sprintf("%d", len(completedParts)), parts[1],
+		"Part count in ETag should match number of parts uploaded")
+}
+
+func TestMultipartUploadETagVerification(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	// Create test data for multipart upload (11MB = 2 parts: 5MB + 6MB)
+	// Using parts of different sizes to ensure correct calculation
+	part1Size := 5 * 1024 * 1024
+	part2Size := 6 * 1024 * 1024
+	data1 := generateRandomData(part1Size)
+	data2 := generateRandomData(part2Size)
+
+	objectKey := "verify-etag-multipart.bin"
+
+	// Pre-calculate expected ETag
+	md1 := md5.Sum(data1)
+	md2 := md5.Sum(data2)
+	concatenatedMD5s := append(md1[:], md2[:]...)
+	finalMD5 := md5.Sum(concatenatedMD5s)
+	expectedETagValue := fmt.Sprintf("%x-2", finalMD5)
+
+	t.Logf("Expected multipart ETag: %s", expectedETagValue)
+
+	// 1. CreateMultipartUpload
+	createResp, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	uploadId := createResp.UploadId
+
+	// 2. UploadPart 1
+	putPart1, err := client.UploadPart(ctx, &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   uploadId,
+		PartNumber: aws.Int32(1),
+		Body:       bytes.NewReader(data1),
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "\""+calculateMD5(data1)+"\"", aws.ToString(putPart1.ETag))
+
+	// 3. UploadPart 2
+	putPart2, err := client.UploadPart(ctx, &s3.UploadPartInput{
+		Bucket:     aws.String(bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   uploadId,
+		PartNumber: aws.Int32(2),
+		Body:       bytes.NewReader(data2),
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "\""+calculateMD5(data2)+"\"", aws.ToString(putPart2.ETag))
+
+	// 4. CompleteMultipartUpload
+	completeResp, err := client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: uploadId,
+		MultipartUpload: &types.CompletedMultipartUpload{
+			Parts: []types.CompletedPart{
+				{ETag: putPart1.ETag, PartNumber: aws.Int32(1)},
+				{ETag: putPart2.ETag, PartNumber: aws.Int32(2)},
+			},
+		},
+	})
+	require.NoError(t, err)
+
+	completeETag := cleanETag(aws.ToString(completeResp.ETag))
+	t.Logf("CompleteMultipartUpload ETag: %s", completeETag)
+	assert.Equal(t, expectedETagValue, completeETag, "CompleteMultipartUpload ETag mismatch")
+
+	// 5. HeadObject
+	headResp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	headETag := cleanETag(aws.ToString(headResp.ETag))
+	assert.Equal(t, expectedETagValue, headETag, "HeadObject ETag mismatch")
+
+	// 6. GetObject
+	getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+	})
+	require.NoError(t, err)
+	getETag := cleanETag(aws.ToString(getResp.ETag))
+	assert.Equal(t, expectedETagValue, getETag, "GetObject ETag mismatch")
+	getResp.Body.Close()
+}
+
+// TestPutObjectETagConsistency verifies ETag consistency between PUT and GET
+func TestPutObjectETagConsistency(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	testCases := []struct {
+		name string
+		size int
+	}{
+		{"tiny", 100},
+		{"small", smallFileSize},
+		{"medium", mediumFileSize},
+		{"large", largeFileSize},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			testData := generateRandomData(tc.size)
+			objectKey := fmt.Sprintf("consistency-test-%s.bin", tc.name)
+
+			// PUT object
+			putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+				Body:   bytes.NewReader(testData),
+			})
+			require.NoError(t, err)
+			putETag := aws.ToString(putResp.ETag)
+
+			// HEAD object
+			headResp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+			})
+			require.NoError(t, err)
+			headETag := aws.ToString(headResp.ETag)
+
+			// GET object
+			getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String(objectKey),
+			})
+			require.NoError(t, err)
+			getETag := aws.ToString(getResp.ETag)
+			getResp.Body.Close()
+
+			// All ETags should match
+			t.Logf("%s (%d bytes): PUT=%s, HEAD=%s, GET=%s",
+				tc.name, tc.size, putETag, headETag, getETag)
+
+			assert.Equal(t, putETag, headETag,
+				"PUT and HEAD ETags should match")
+			assert.Equal(t, putETag, getETag,
+				"PUT and GET ETags should match")
+
+			// All should be pure MD5 (not composite) for regular PutObject
+			assert.True(t, isPureMD5ETag(putETag),
+				"PutObject ETag should be pure MD5, got: %s", putETag)
+		})
+	}
+}
+
+// TestETagHexValidation simulates the AWS SDK v2 validation that caused issue #7768
+func TestETagHexValidation(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	// Test with a file large enough to trigger auto-chunking
+	testData := generateRandomData(largeFileSize)
+	objectKey := "hex-validation-test.bin"
+
+	putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(testData),
+	})
+	require.NoError(t, err)
+
+	etag := cleanETag(aws.ToString(putResp.ETag))
+
+	// Simulate AWS SDK v2's hex validation (Base16Codec.decode)
+	// This is what fails in issue #7768 when ETag contains '-'
+	t.Logf("Validating ETag as hex: %s", etag)
+
+	_, err = hex.DecodeString(etag)
+	assert.NoError(t, err,
+		"ETag should be valid hexadecimal (AWS SDK v2 validation). "+
+			"Got ETag: %s. If this fails with 'invalid byte', the ETag contains non-hex chars like '-'",
+		etag)
+}
+
+// TestMultipleLargeFileUploads verifies ETag format across multiple large uploads
+func TestMultipleLargeFileUploads(t *testing.T) {
+	ctx := context.Background()
+	client := getS3Client(t)
+
+	bucketName := getNewBucketName()
+	err := createTestBucket(ctx, client, bucketName)
+	require.NoError(t, err, "Failed to create test bucket")
+	defer cleanupTestBucket(ctx, client, bucketName)
+
+	numFiles := 5
+	for i := 0; i < numFiles; i++ {
+		testData := generateRandomData(largeFileSize)
+		expectedMD5 := calculateMD5(testData)
+		objectKey := fmt.Sprintf("large-file-%d.bin", i)
+
+		putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(objectKey),
+			Body:   bytes.NewReader(testData),
+		})
+		require.NoError(t, err, "Failed to upload file %d", i)
+
+		etag := aws.ToString(putResp.ETag)
+		t.Logf("File %d ETag: %s (expected MD5: %s)", i, etag, expectedMD5)
+
+		assert.True(t, isPureMD5ETag(etag),
+			"File %d ETag should be pure MD5, got: %s", i, etag)
+		assert.Equal(t, expectedMD5, cleanETag(etag),
+			"File %d ETag should match MD5", i)
+
+		// Validate as hex (AWS SDK v2 check)
+		_, err = hex.DecodeString(cleanETag(etag))
+		assert.NoError(t, err, "File %d ETag should be valid hex", i)
+	}
+}

test/s3/etag/test_config.json

@@ -0,0 +1,20 @@
+{
+  "endpoint": "http://127.0.0.1:8333",
+  "access_key": "some_access_key1",
+  "secret_key": "some_secret_key1",
+  "region": "us-east-1",
+  "bucket_prefix": "test-etag-",
+  "notes": {
+    "description": "S3 ETag format integration tests",
+    "issue": "https://github.com/seaweedfs/seaweedfs/issues/7768",
+    "auto_chunk_size": "8MB - files larger than this trigger auto-chunking",
+    "test_sizes": {
+      "small": "1KB - single chunk",
+      "medium": "256KB - at inline threshold", 
+      "large": "10MB - 2 internal chunks",
+      "xl": "25MB - 4 internal chunks"
+    }
+  }
+}
+
+

test/s3/filer_group/Makefile

@@ -0,0 +1,157 @@
+# S3 Filer Group Test Makefile
+# This Makefile provides targets for running S3 filer group integration tests
+# These tests verify that S3 bucket operations work correctly when a filer group is configured.
+
+.PHONY: help build-weed check-deps start-server stop-server test test-with-server clean logs health-check
+
+# Configuration
+WEED_BINARY := ../../../weed/weed_binary
+S3_PORT := 8333
+MASTER_PORT := 9333
+MASTER_GRPC_PORT := 19333
+VOLUME_PORT := 8080
+FILER_PORT := 8888
+TEST_TIMEOUT := 10m
+TEST_PATTERN := Test
+
+# Filer group configuration
+FILER_GROUP := testgroup
+SERVER_DIR := ./test-volume-data
+S3_CONFIG := ../../../docker/compose/s3.json
+
+# Default target
+help:
+	@echo "S3 Filer Group Test Makefile"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  help                    - Show this help message"
+	@echo "  build-weed             - Build the SeaweedFS binary"
+	@echo "  check-deps             - Check dependencies and build binary if needed"
+	@echo "  start-server           - Start SeaweedFS server with filer group"
+	@echo "  stop-server            - Stop SeaweedFS server"
+	@echo "  test                   - Run filer group tests (server must be running)"
+	@echo "  test-with-server       - Start server, run tests, stop server"
+	@echo "  logs                   - Show server logs"
+	@echo "  clean                  - Clean up test artifacts and stop server"
+	@echo "  health-check           - Check if server is accessible"
+	@echo ""
+	@echo "Configuration:"
+	@echo "  S3_PORT=${S3_PORT}"
+	@echo "  MASTER_GRPC_PORT=${MASTER_GRPC_PORT}"
+	@echo "  FILER_GROUP=${FILER_GROUP}"
+	@echo "  TEST_TIMEOUT=${TEST_TIMEOUT}"
+
+# Build the SeaweedFS binary
+build-weed:
+	@echo "Building SeaweedFS binary..."
+	@cd ../../../weed && go build -o weed_binary .
+	@chmod +x $(WEED_BINARY)
+	@echo "OK SeaweedFS binary built at $(WEED_BINARY)"
+
+check-deps: build-weed
+	@echo "Checking dependencies..."
+	@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1)
+	@echo "Go version: $$(go version)"
+	@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1)
+	@go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1)
+	@go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1)
+	@echo "OK All dependencies are available"
+
+# Start SeaweedFS server with filer group configured
+start-server: check-deps
+	@echo "Starting SeaweedFS server with filer group: $(FILER_GROUP)..."
+	@rm -f weed-server.pid
+	@mkdir -p $(SERVER_DIR)
+	@if netstat -tlnp 2>/dev/null | grep $(S3_PORT) >/dev/null; then \
+		echo "WARNING: Port $(S3_PORT) is already in use"; \
+		exit 1; \
+	fi
+	@echo "Launching SeaweedFS server with filer group $(FILER_GROUP)..."
+	@export AWS_ACCESS_KEY_ID=some_access_key1 && \
+	export AWS_SECRET_ACCESS_KEY=some_secret_key1 && \
+	$(WEED_BINARY) mini \
+		-debug \
+		-dir=$(SERVER_DIR) \
+		-s3.port=$(S3_PORT) \
+		-s3.config=$(S3_CONFIG) \
+		-filer.filerGroup=$(FILER_GROUP) \
+		> weed-server.log 2>&1 & \
+		echo $$! > weed-server.pid
+	
+	@echo "Waiting for S3 server to be ready..."
+	@for i in $$(seq 1 30); do \
+		if echo | nc -z localhost $(S3_PORT); then \
+			echo "S3 server is ready!"; \
+			exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "❌ Server failed to start within 30 seconds"; \
+	if [ -f weed-server.log ]; then \
+		cat weed-server.log; \
+	fi; \
+	exit 1
+
+# Stop SeaweedFS server
+stop-server:
+	@echo "Stopping SeaweedFS server..."
+	@if [ -f weed-server.pid ]; then \
+		SERVER_PID=$$(cat weed-server.pid); \
+		echo "Killing server PID $$SERVER_PID"; \
+		if ps -p $$SERVER_PID >/dev/null 2>&1; then \
+			kill -TERM $$SERVER_PID 2>/dev/null || true; \
+			sleep 2; \
+			if ps -p $$SERVER_PID >/dev/null 2>&1; then \
+				echo "Process still running, sending KILL signal..."; \
+				kill -KILL $$SERVER_PID 2>/dev/null || true; \
+				sleep 1; \
+			fi; \
+		else \
+			echo "Process $$SERVER_PID not found (already stopped)"; \
+		fi; \
+		rm -f weed-server.pid; \
+	else \
+		echo "No PID file found"; \
+	fi
+	@echo "✅ SeaweedFS server stopped"
+
+# Show server logs
+logs:
+	@if test -f weed-server.log; then \
+		echo "=== SeaweedFS Server Logs ==="; \
+		tail -f weed-server.log; \
+	else \
+		echo "No log file found. Server may not be running."; \
+	fi
+
+# Run filer group tests (assumes server is already running)
+test: check-deps
+	@echo "Running filer group S3 tests..."
+	@FILER_GROUP=$(FILER_GROUP) S3_ENDPOINT=http://localhost:$(S3_PORT) MASTER_ADDRESS=localhost:$(MASTER_GRPC_PORT) \
+		go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" .
+	@echo "✅ Filer group tests completed"
+
+# Run tests with automatic server management
+test-with-server: start-server
+	@echo "Server started successfully, running filer group tests..."
+	@echo "Test pattern: $(TEST_PATTERN)"
+	@echo "Test timeout: $(TEST_TIMEOUT)"
+	@trap "$(MAKE) stop-server" EXIT; \
+	$(MAKE) test || (echo "❌ Tests failed, showing server logs:" && echo "=== Last 50 lines of server logs ===" && tail -50 weed-server.log && echo "=== End of server logs ===" && exit 1)
+	@$(MAKE) stop-server
+	@echo "✅ Tests completed and server stopped"
+
+# Clean up test artifacts
+clean:
+	@echo "Cleaning up test artifacts..."
+	@$(MAKE) stop-server
+	@rm -f weed-server.log weed-test*.log weed-server.pid
+	@rm -rf test-volume-data/
+	@go clean -testcache
+	@echo "✅ Cleanup completed"
+
+# Quick health check
+health-check:
+	@echo "Running health check..."
+	@curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1 && echo "✅ S3 API is accessible" || echo "❌ S3 API is not accessible"
+	@curl -s http://localhost:9325/metrics >/dev/null 2>&1 && echo "✅ Metrics endpoint is accessible" || echo "❌ Metrics endpoint is not accessible"

test/s3/filer_group/README.md

@@ -0,0 +1,99 @@
+# Filer Group S3 Tests
+
+These tests verify that S3 bucket operations work correctly when a filer group is configured.
+
+## Background
+
+When SeaweedFS is configured with a filer group (via `-filer.group` option), collections are named with the filer group prefix:
+
+```text
+Collection name = {filerGroup}_{bucketName}
+```
+
+For example, with filer group `mygroup` and bucket `mybucket`, the collection will be named `mygroup_mybucket`.
+
+## Issue Being Tested
+
+This test suite was created to verify the fix for a bug where:
+- The admin UI was using just the bucket name when deleting collections
+- This caused collection deletion to fail when a filer group was configured
+- After bucket deletion via admin UI, the collection data would be orphaned
+
+## Running the Tests
+
+### Prerequisites
+
+1. SeaweedFS servers must be running with a filer group configured
+2. The S3 gateway must be accessible
+3. Master server must be accessible for collection verification
+
+### Quick Start
+
+```bash
+# Set environment variables
+export FILER_GROUP=testgroup
+export S3_ENDPOINT=http://localhost:8333
+export MASTER_ADDRESS=localhost:9333
+
+# Run tests
+go test -v ./...
+```
+
+### Using the Makefile
+
+```bash
+# Start test servers with filer group configured
+make start-servers FILER_GROUP=testgroup
+
+# Run tests
+make test
+
+# Stop servers
+make stop-servers
+
+# Or run full test cycle
+make full-test
+```
+
+### Configuration
+
+Tests can be configured via:
+
+1. Environment variables:
+   - `FILER_GROUP`: The filer group name (required for tests to run)
+   - `S3_ENDPOINT`: S3 API endpoint (default: `http://localhost:8333`)
+   - `MASTER_ADDRESS`: Master server address (default: localhost:9333)
+
+2. `test_config.json` file
+
+## Test Cases
+
+### TestFilerGroupCollectionNaming
+Verifies that when a bucket is created and objects are uploaded:
+1. The collection is created with the correct filer group prefix
+2. Bucket deletion removes the correctly-named collection
+
+### TestBucketDeletionWithFilerGroup
+Specifically tests that bucket deletion via S3 API correctly deletes
+the collection when filer group is configured.
+
+### TestMultipleBucketsWithFilerGroup
+Tests creating and deleting multiple buckets to ensure the filer group
+prefix is correctly applied and removed for all buckets.
+
+## Expected Behavior
+
+With filer group `testgroup`:
+
+| Bucket Name | Expected Collection Name |
+|-------------|-------------------------|
+| mybucket    | testgroup_mybucket      |
+| test-123    | testgroup_test-123      |
+
+Without filer group:
+
+| Bucket Name | Expected Collection Name |
+|-------------|-------------------------|
+| mybucket    | mybucket                |
+| test-123    | test-123                |
+

test/s3/filer_group/s3_filer_group_test.go

@@ -0,0 +1,332 @@
+package filer_group
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+// TestConfig holds configuration for filer group S3 tests
+type TestConfig struct {
+	S3Endpoint    string `json:"s3_endpoint"`
+	MasterAddress string `json:"master_address"`
+	AccessKey     string `json:"access_key"`
+	SecretKey     string `json:"secret_key"`
+	Region        string `json:"region"`
+	FilerGroup    string `json:"filer_group"`
+}
+
+var testConfig = &TestConfig{
+	S3Endpoint:    "http://localhost:8333",
+	MasterAddress: "localhost:19333", // gRPC port = 10000 + master HTTP port (9333)
+	AccessKey:     "some_access_key1",
+	SecretKey:     "some_secret_key1",
+	Region:        "us-east-1",
+	FilerGroup:    "testgroup", // Expected filer group for these tests
+}
+
+func init() {
+	// Load config from file if exists
+	if data, err := os.ReadFile("test_config.json"); err == nil {
+		if err := json.Unmarshal(data, testConfig); err != nil {
+			// Log but don't fail - env vars can still override
+			fmt.Fprintf(os.Stderr, "Warning: failed to parse test_config.json: %v\n", err)
+		}
+	}
+
+	// Override from environment variables
+	if v := os.Getenv("S3_ENDPOINT"); v != "" {
+		testConfig.S3Endpoint = v
+	}
+	if v := os.Getenv("MASTER_ADDRESS"); v != "" {
+		testConfig.MasterAddress = v
+	}
+	if v := os.Getenv("FILER_GROUP"); v != "" {
+		testConfig.FilerGroup = v
+	}
+}
+
+func getS3Client(t *testing.T) *s3.Client {
+	cfg, err := config.LoadDefaultConfig(context.TODO(),
+		config.WithRegion(testConfig.Region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+			testConfig.AccessKey,
+			testConfig.SecretKey,
+			"",
+		)),
+	)
+	require.NoError(t, err)
+
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.BaseEndpoint = aws.String(testConfig.S3Endpoint)
+		o.UsePathStyle = true
+	})
+}
+
+func getMasterClient(t *testing.T) master_pb.SeaweedClient {
+	conn, err := grpc.NewClient(testConfig.MasterAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	require.NoError(t, err)
+	t.Cleanup(func() { conn.Close() })
+	return master_pb.NewSeaweedClient(conn)
+}
+
+func getNewBucketName() string {
+	return fmt.Sprintf("filergroup-test-%d", time.Now().UnixNano())
+}
+
+// getExpectedCollectionName returns the expected collection name for a bucket
+// When filer group is configured, collections are named: {filerGroup}_{bucketName}
+func getExpectedCollectionName(bucketName string) string {
+	if testConfig.FilerGroup != "" {
+		return fmt.Sprintf("%s_%s", testConfig.FilerGroup, bucketName)
+	}
+	return bucketName
+}
+
+// listAllCollections returns a list of all collection names from the master
+func listAllCollections(t *testing.T, masterClient master_pb.SeaweedClient) []string {
+	collectionResp, err := masterClient.CollectionList(context.Background(), &master_pb.CollectionListRequest{
+		IncludeNormalVolumes: true,
+		IncludeEcVolumes:     true,
+	})
+	if err != nil {
+		t.Logf("Warning: failed to list collections: %v", err)
+		return nil
+	}
+	var names []string
+	for _, c := range collectionResp.Collections {
+		names = append(names, c.Name)
+	}
+	return names
+}
+
+// collectionExists checks if a collection exists in the master
+func collectionExists(t *testing.T, masterClient master_pb.SeaweedClient, collectionName string) bool {
+	for _, name := range listAllCollections(t, masterClient) {
+		if name == collectionName {
+			return true
+		}
+	}
+	return false
+}
+
+// waitForCollectionExists waits for a collection to exist using polling
+func waitForCollectionExists(t *testing.T, masterClient master_pb.SeaweedClient, collectionName string) {
+	var lastCollections []string
+	success := assert.Eventually(t, func() bool {
+		lastCollections = listAllCollections(t, masterClient)
+		for _, name := range lastCollections {
+			if name == collectionName {
+				return true
+			}
+		}
+		return false
+	}, 10*time.Second, 200*time.Millisecond)
+	if !success {
+		t.Fatalf("collection %s should be created; existing collections: %v", collectionName, lastCollections)
+	}
+}
+
+// waitForCollectionDeleted waits for a collection to be deleted using polling
+func waitForCollectionDeleted(t *testing.T, masterClient master_pb.SeaweedClient, collectionName string) {
+	require.Eventually(t, func() bool {
+		return !collectionExists(t, masterClient, collectionName)
+	}, 10*time.Second, 200*time.Millisecond, "collection %s should be deleted", collectionName)
+}
+
+// TestFilerGroupCollectionNaming verifies that when a filer group is configured,
+// collections are created with the correct prefix ({filerGroup}_{bucketName})
+func TestFilerGroupCollectionNaming(t *testing.T) {
+	if testConfig.FilerGroup == "" {
+		t.Skip("Skipping test: FILER_GROUP not configured. Set FILER_GROUP environment variable or configure in test_config.json")
+	}
+
+	s3Client := getS3Client(t)
+	masterClient := getMasterClient(t)
+	bucketName := getNewBucketName()
+	expectedCollection := getExpectedCollectionName(bucketName)
+
+	t.Logf("Testing with filer group: %s", testConfig.FilerGroup)
+	t.Logf("Bucket name: %s", bucketName)
+	t.Logf("Expected collection name: %s", expectedCollection)
+
+	// Create bucket
+	_, err := s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err, "CreateBucket should succeed")
+
+	// Upload an object to trigger collection creation
+	_, err = s3Client.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("test-object"),
+		Body:   strings.NewReader("test content"),
+	})
+	require.NoError(t, err, "PutObject should succeed")
+
+	// Wait for collection to be visible using polling
+	waitForCollectionExists(t, masterClient, expectedCollection)
+
+	// Verify collection exists with correct name
+	require.True(t, collectionExists(t, masterClient, expectedCollection),
+		"Collection %s should exist (filer group prefix applied)", expectedCollection)
+
+	// Cleanup: delete object and bucket
+	_, err = s3Client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("test-object"),
+	})
+	require.NoError(t, err, "DeleteObject should succeed")
+
+	_, err = s3Client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err, "DeleteBucket should succeed")
+
+	// Wait for collection to be deleted using polling
+	waitForCollectionDeleted(t, masterClient, expectedCollection)
+
+	t.Log("SUCCESS: Collection naming with filer group works correctly")
+}
+
+// TestBucketDeletionWithFilerGroup verifies that bucket deletion correctly
+// deletes the collection when filer group is configured
+func TestBucketDeletionWithFilerGroup(t *testing.T) {
+	if testConfig.FilerGroup == "" {
+		t.Skip("Skipping test: FILER_GROUP not configured")
+	}
+
+	s3Client := getS3Client(t)
+	masterClient := getMasterClient(t)
+	bucketName := getNewBucketName()
+	expectedCollection := getExpectedCollectionName(bucketName)
+
+	// Create bucket and add an object
+	_, err := s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+
+	_, err = s3Client.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("test-object"),
+		Body:   strings.NewReader("test content"),
+	})
+	require.NoError(t, err)
+
+	// Wait for collection to be created using polling
+	waitForCollectionExists(t, masterClient, expectedCollection)
+
+	// Verify collection exists before deletion
+	require.True(t, collectionExists(t, masterClient, expectedCollection),
+		"Collection should exist before bucket deletion")
+
+	// Delete object first
+	_, err = s3Client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("test-object"),
+	})
+	require.NoError(t, err)
+
+	// Delete bucket
+	_, err = s3Client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err, "DeleteBucket should succeed")
+
+	// Wait for collection to be deleted using polling
+	waitForCollectionDeleted(t, masterClient, expectedCollection)
+
+	// Verify collection was deleted
+	require.False(t, collectionExists(t, masterClient, expectedCollection),
+		"Collection %s should be deleted after bucket deletion", expectedCollection)
+
+	t.Log("SUCCESS: Bucket deletion with filer group correctly deletes collection")
+}
+
+// TestMultipleBucketsWithFilerGroup tests creating and deleting multiple buckets
+func TestMultipleBucketsWithFilerGroup(t *testing.T) {
+	if testConfig.FilerGroup == "" {
+		t.Skip("Skipping test: FILER_GROUP not configured")
+	}
+
+	s3Client := getS3Client(t)
+	masterClient := getMasterClient(t)
+
+	buckets := make([]string, 3)
+	for i := 0; i < 3; i++ {
+		buckets[i] = getNewBucketName()
+	}
+
+	// Create all buckets and add objects
+	for _, bucket := range buckets {
+		_, err := s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{
+			Bucket: aws.String(bucket),
+		})
+		require.NoError(t, err)
+
+		_, err = s3Client.PutObject(context.Background(), &s3.PutObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String("test-object"),
+			Body:   strings.NewReader("test content"),
+		})
+		require.NoError(t, err)
+	}
+
+	// Wait for all collections to be created using polling
+	for _, bucket := range buckets {
+		expectedCollection := getExpectedCollectionName(bucket)
+		waitForCollectionExists(t, masterClient, expectedCollection)
+	}
+
+	// Verify all collections exist with correct naming
+	for _, bucket := range buckets {
+		expectedCollection := getExpectedCollectionName(bucket)
+		require.True(t, collectionExists(t, masterClient, expectedCollection),
+			"Collection %s should exist for bucket %s", expectedCollection, bucket)
+	}
+
+	// Delete all buckets
+	for _, bucket := range buckets {
+		_, err := s3Client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String("test-object"),
+		})
+		require.NoError(t, err)
+
+		_, err = s3Client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
+			Bucket: aws.String(bucket),
+		})
+		require.NoError(t, err)
+	}
+
+	// Wait for all collections to be deleted using polling
+	for _, bucket := range buckets {
+		expectedCollection := getExpectedCollectionName(bucket)
+		waitForCollectionDeleted(t, masterClient, expectedCollection)
+	}
+
+	// Verify all collections are deleted
+	for _, bucket := range buckets {
+		expectedCollection := getExpectedCollectionName(bucket)
+		require.False(t, collectionExists(t, masterClient, expectedCollection),
+			"Collection %s should be deleted for bucket %s", expectedCollection, bucket)
+	}
+
+	t.Log("SUCCESS: Multiple bucket operations with filer group work correctly")
+}

test/s3/filer_group/test_config.json

@@ -0,0 +1,7 @@
+{
+  "s3_endpoint": "http://localhost:8333",
+  "access_key": "some_access_key1",
+  "secret_key": "some_secret_key1",
+  "region": "us-east-1",
+  "filer_group": "testgroup"
+}

test/s3/iam/Dockerfile.s3

@@ -0,0 +1,33 @@
+# Multi-stage build for SeaweedFS S3 with IAM
+FROM golang:1.26-alpine AS builder
+
+# Install build dependencies
+RUN apk add --no-cache git make curl wget
+
+# Set working directory
+WORKDIR /app
+
+# Copy source code
+COPY . .
+
+# Build SeaweedFS with IAM integration
+RUN cd weed && go build -o /usr/local/bin/weed
+
+# Final runtime image
+FROM alpine:latest
+
+# Install runtime dependencies
+RUN apk add --no-cache ca-certificates wget curl
+
+# Copy weed binary
+COPY --from=builder /usr/local/bin/weed /usr/local/bin/weed
+
+# Create directories
+RUN mkdir -p /etc/seaweedfs /data
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+  CMD wget --quiet --tries=1 --spider http://localhost:8333/ || exit 1
+
+# Set entrypoint
+ENTRYPOINT ["/usr/local/bin/weed"]

test/s3/iam/Makefile

@@ -0,0 +1,311 @@
+# SeaweedFS S3 IAM Integration Tests Makefile
+
+.PHONY: all test clean setup start-services stop-services wait-for-services help
+
+# Default target
+all: test
+
+# Test configuration
+WEED_BINARY ?= $(shell go env GOPATH)/bin/weed
+LOG_LEVEL ?= 2
+S3_PORT ?= 8333
+FILER_PORT ?= 8888
+MASTER_PORT ?= 9333
+VOLUME_PORT ?= 8081
+TEST_TIMEOUT ?= 30m
+
+# Service PIDs
+MASTER_PID_FILE = /tmp/weed-master.pid
+VOLUME_PID_FILE = /tmp/weed-volume.pid  
+FILER_PID_FILE = /tmp/weed-filer.pid
+S3_PID_FILE = /tmp/weed-s3.pid
+MINI_PID_FILE = /tmp/weed-mini.pid
+
+help: ## Show this help message
+	@echo "SeaweedFS S3 IAM Integration Tests"
+	@echo ""
+	@echo "Usage:"
+	@echo "  make [target]"
+	@echo ""
+	@echo "Standard Targets:"
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  %-25s %s\n", $$1, $$2}' $(MAKEFILE_LIST) | head -20
+	@echo ""
+	@echo "New Test Targets (Previously Skipped):"
+	@echo "  test-distributed         Run distributed IAM tests"
+	@echo "  test-performance         Run performance tests"
+	@echo "  test-stress              Run stress tests"
+	@echo "  test-versioning-stress   Run S3 versioning stress tests"
+	@echo "  test-keycloak-full       Run complete Keycloak integration tests"
+	@echo "  test-all-previously-skipped  Run all previously skipped tests"
+	@echo "  setup-all-tests          Setup environment for all tests"
+	@echo ""
+	@echo "Docker Compose Targets:"
+	@echo "  docker-test              Run tests with Docker Compose including Keycloak"
+	@echo "  docker-up                Start all services with Docker Compose"
+	@echo "  docker-down              Stop all Docker Compose services"
+	@echo "  docker-logs              Show logs from all services"
+
+test: clean setup start-services run-tests stop-services ## Run complete IAM integration test suite
+
+test-quick: run-tests ## Run tests assuming services are already running
+
+run-tests: ## Execute the Go tests
+	@echo "Running S3 IAM Integration Tests..."
+	go test -v -timeout $(TEST_TIMEOUT) ./...
+
+setup: ## Setup test environment
+	@echo "Setting up test environment..."
+	@mkdir -p test-volume-data/filerldb2
+	@mkdir -p test-volume-data/m9333
+	@if [ ! -f iam_config.json ]; then \
+		echo "Creating iam_config.json from iam_config.local.json..."; \
+		cp iam_config.local.json iam_config.json; \
+	fi
+
+start-services: ## Start SeaweedFS services for testing
+	@echo "Starting SeaweedFS services using weed mini..."
+	@mkdir -p test-volume-data
+	@$(WEED_BINARY) mini \
+		-dir=test-volume-data \
+		-s3.port=$(S3_PORT) \
+		-s3.config=test_config.json \
+		-s3.iam.config=$(CURDIR)/iam_config.json \
+		-s3.iam.readOnly=false \
+		> weed-mini.log 2>&1 & \
+		echo $$! > $(MINI_PID_FILE)
+	
+	@echo "Waiting for services to be ready..."
+	@$(MAKE) wait-for-services
+	@echo "OK All services started and ready"
+
+wait-for-services: ## Wait for all services to be ready
+	@echo "Waiting for services to be ready..."
+	@echo "Checking S3 API server..."
+	@for i in $$(seq 1 30); do \
+		if curl -s http://localhost:$(S3_PORT) > /dev/null 2>&1; then \
+			echo "OK S3 API server is ready"; \
+			exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+		echo "ERROR S3 API failed to start"; \
+	exit 1
+	
+	@echo "Pre-allocating volumes for concurrent operations..."
+	@curl -s "http://localhost:$(MASTER_PORT)/vol/grow?collection=default&count=10&replication=000" > /dev/null || echo "WARNING Volume pre-allocation failed, but continuing..."
+	@sleep 3
+	@echo "OK All services are ready"
+
+stop-services: ## Stop all SeaweedFS services
+	@echo "Stopping SeaweedFS services..."
+	@if [ -f $(MINI_PID_FILE) ]; then \
+		echo "Stopping weed mini..."; \
+		kill $$(cat $(MINI_PID_FILE)) 2>/dev/null || true; \
+		rm -f $(MINI_PID_FILE); \
+	fi
+	@if [ -f $(S3_PID_FILE) ]; then \
+		echo "Stopping S3 API server..."; \
+		kill $$(cat $(S3_PID_FILE)) 2>/dev/null || true; \
+		rm -f $(S3_PID_FILE); \
+	fi
+	@if [ -f $(FILER_PID_FILE) ]; then \
+		echo "Stopping filer server..."; \
+		kill $$(cat $(FILER_PID_FILE)) 2>/dev/null || true; \
+		rm -f $(FILER_PID_FILE); \
+	fi
+	@if [ -f $(VOLUME_PID_FILE) ]; then \
+		echo "Stopping volume server..."; \
+		kill $$(cat $(VOLUME_PID_FILE)) 2>/dev/null || true; \
+		rm -f $(VOLUME_PID_FILE); \
+	fi
+	@if [ -f $(MASTER_PID_FILE) ]; then \
+		echo "Stopping master server..."; \
+		kill $$(cat $(MASTER_PID_FILE)) 2>/dev/null || true; \
+		rm -f $(MASTER_PID_FILE); \
+	fi
+	@echo "OK All services stopped"
+
+clean: stop-services ## Clean up test environment
+	@echo "Cleaning up test environment..."
+	@rm -rf test-volume-data
+	@rm -f weed-*.log
+	@rm -f *.test
+	@rm -f iam_config.json
+	@rm -f .test_env
+	@docker rm -f keycloak-iam-test >/dev/null 2>&1 || true
+	@docker rm -f openldap-iam-test >/dev/null 2>&1 || true
+	@echo "Cleanup complete"
+
+logs: ## Show service logs
+	@echo "Service Logs:"
+	@echo "=== Master Log ==="
+	@tail -20 weed-master.log 2>/dev/null || echo "No master log"
+	@echo ""
+	@echo "=== Volume Log ==="
+	@tail -20 weed-volume.log 2>/dev/null || echo "No volume log"
+	@echo ""
+	@echo "=== Filer Log ==="
+	@tail -20 weed-filer.log 2>/dev/null || echo "No filer log"
+	@echo ""
+	@echo "=== S3 API Log ==="
+	@tail -20 weed-s3.log 2>/dev/null || echo "No S3 log"
+
+status: ## Check service status
+	@echo "Service Status:"
+	@echo -n "Master: "; curl -s http://localhost:$(MASTER_PORT)/cluster/status > /dev/null 2>&1 && echo "✅ Running" || echo "❌ Not running"
+	@echo -n "Filer: "; curl -s http://localhost:$(FILER_PORT)/status > /dev/null 2>&1 && echo "✅ Running" || echo "❌ Not running"
+	@echo -n "S3 API: "; curl -s http://localhost:$(S3_PORT) > /dev/null 2>&1 && echo "✅ Running" || echo "❌ Not running"
+
+debug: start-services wait-for-services ## Start services and keep them running for debugging
+	@echo "🐛 Services started in debug mode. Press Ctrl+C to stop..."
+	@trap 'make stop-services' INT; \
+	while true; do \
+		sleep 1; \
+	done
+
+# Test specific scenarios
+test-auth: ## Test only authentication scenarios
+	go test -v -run TestS3IAMAuthentication ./...
+
+test-policy: ## Test only policy enforcement
+	go test -v -run TestS3IAMPolicyEnforcement ./...
+
+test-expiration: ## Test only session expiration
+	go test -v -run TestS3IAMSessionExpiration ./...
+
+test-multipart: ## Test only multipart upload IAM integration
+	go test -v -run TestS3IAMMultipartUploadPolicyEnforcement ./...
+
+test-bucket-policy: ## Test only bucket policy integration
+	go test -v -run TestS3IAMBucketPolicyIntegration ./...
+
+test-context: ## Test only contextual policy enforcement  
+	go test -v -run TestS3IAMContextualPolicyEnforcement ./...
+
+test-presigned: ## Test only presigned URL integration
+	go test -v -run TestS3IAMPresignedURLIntegration ./...
+
+test-group: ## Run IAM group management tests
+	go test -v -run "TestIAMGroup" ./...
+
+test-sts: ## Run all STS tests
+	go test -v -run "TestSTS" ./...
+
+test-sts-assume-role: ## Run AssumeRole STS tests
+	go test -v -run "TestSTSAssumeRole" ./...
+
+test-sts-ldap: ## Run LDAP STS tests
+	go test -v -run "TestSTSLDAP" ./...
+
+test-sts-suite: start-services ## Run all STS tests with full environment setup/teardown
+	@echo "Running STS test suite..."
+	-go test -v -run "TestSTS" ./...
+	@$(MAKE) stop-services
+
+# Performance testing
+benchmark: setup start-services wait-for-services ## Run performance benchmarks
+	@echo "🏁 Running IAM performance benchmarks..."
+	go test -bench=. -benchmem -timeout $(TEST_TIMEOUT) ./...
+	@make stop-services
+
+# Continuous integration
+ci: ## Run tests suitable for CI environment
+	@echo "🔄 Running CI tests..."
+	@export CGO_ENABLED=0; make test
+
+# Development helpers
+watch: ## Watch for file changes and re-run tests
+	@echo "👀 Watching for changes..."
+	@command -v entr >/dev/null 2>&1 || (echo "entr is required for watch mode. Install with: brew install entr" && exit 1)
+	@find . -name "*.go" | entr -r make test-quick
+
+install-deps: ## Install test dependencies
+	@echo "📦 Installing test dependencies..."
+	go mod tidy
+	go get -u github.com/stretchr/testify
+	go get -u github.com/aws/aws-sdk-go
+	go get -u github.com/golang-jwt/jwt/v5
+
+# Docker support
+docker-test-legacy: ## Run tests in Docker container (legacy)
+	@echo "🐳 Running tests in Docker..."
+	docker build -f Dockerfile.test -t seaweedfs-s3-iam-test .
+	docker run --rm -v $(PWD)/../../../:/app seaweedfs-s3-iam-test
+
+# Docker Compose support with Keycloak
+docker-up: ## Start all services with Docker Compose (including Keycloak)
+	@echo "🐳 Starting services with Docker Compose including Keycloak..."
+	@docker compose up -d
+	@echo "⏳ Waiting for services to be healthy..."
+	@timeout 120 bash -c 'until curl -s http://localhost:8080/health/ready > /dev/null 2>&1; do sleep 2; done' || (echo "❌ Keycloak failed to become ready" && exit 1)
+	@timeout 60 bash -c 'until curl -s http://localhost:8333 > /dev/null 2>&1; do sleep 2; done' || (echo "❌ S3 API failed to become ready" && exit 1)
+	@timeout 60 bash -c 'until curl -s http://localhost:8888 > /dev/null 2>&1; do sleep 2; done' || (echo "❌ Filer failed to become ready" && exit 1)
+	@timeout 60 bash -c 'until curl -s http://localhost:9333 > /dev/null 2>&1; do sleep 2; done' || (echo "❌ Master failed to become ready" && exit 1)
+	@echo "✅ All services are healthy and ready"
+
+docker-down: ## Stop all Docker Compose services
+	@echo "🐳 Stopping Docker Compose services..."
+	@docker compose down -v
+	@echo "✅ All services stopped"
+
+docker-logs: ## Show logs from all services
+	@docker compose logs -f
+
+docker-test: docker-up ## Run tests with Docker Compose including Keycloak
+	@echo "🧪 Running Keycloak integration tests..."
+	@export KEYCLOAK_URL="http://localhost:8080" && \
+	 export S3_ENDPOINT="http://localhost:8333" && \
+	 go test -v -timeout $(TEST_TIMEOUT) -run "TestKeycloak" ./...
+	@echo "🐳 Stopping services after tests..."
+	@make docker-down
+
+docker-build: ## Build custom SeaweedFS image for Docker tests
+	@echo "🏗️ Building custom SeaweedFS image..."
+	@docker build -f Dockerfile.s3 -t seaweedfs-iam:latest ../../..
+	@echo "✅ Image built successfully"
+
+# All PHONY targets
+.PHONY: test test-quick run-tests setup start-services stop-services wait-for-services clean logs status debug
+.PHONY: test-auth test-policy test-expiration test-multipart test-bucket-policy test-context test-presigned test-group test-sts test-sts-assume-role test-sts-ldap  
+.PHONY: benchmark ci watch install-deps docker-test docker-up docker-down docker-logs docker-build
+.PHONY: test-distributed test-performance test-stress test-versioning-stress test-keycloak-full test-all-previously-skipped setup-all-tests help-advanced
+
+
+
+# New test targets for previously skipped tests
+
+test-distributed: ## Run distributed IAM tests
+	@echo "🌐 Running distributed IAM tests..."
+	@export ENABLE_DISTRIBUTED_TESTS=true && go test -v -timeout $(TEST_TIMEOUT) -run "TestS3IAMDistributedTests" ./...
+
+test-performance: ## Run performance tests
+	@echo "🏁 Running performance tests..."
+	@export ENABLE_PERFORMANCE_TESTS=true && go test -v -timeout $(TEST_TIMEOUT) -run "TestS3IAMPerformanceTests" ./...
+
+test-stress: ## Run stress tests
+	@echo "💪 Running stress tests..."
+	@export ENABLE_STRESS_TESTS=true && ./run_stress_tests.sh
+
+test-versioning-stress: ## Run S3 versioning stress tests
+	@echo "📚 Running versioning stress tests..."
+	@cd ../versioning && ./enable_stress_tests.sh
+
+test-keycloak-full: docker-up ## Run complete Keycloak integration tests
+	@echo "🔐 Running complete Keycloak integration tests..."
+	@export KEYCLOAK_URL="http://localhost:8080" && \
+	 export S3_ENDPOINT="http://localhost:8333" && \
+	 go test -v -timeout $(TEST_TIMEOUT) -run "TestKeycloak" ./...
+	@make docker-down
+
+test-all-previously-skipped: ## Run all previously skipped tests
+	@echo "🎯 Running all previously skipped tests..."
+	@./run_all_tests.sh
+
+.PHONY: cleanup
+cleanup: clean
+
+setup-all-tests: ## Setup environment for all tests (including Keycloak)
+	@echo "🚀 Setting up complete test environment..."
+	@./setup_all_tests.sh
+
+

test/s3/iam/Makefile.docker

@@ -0,0 +1,166 @@
+# Makefile for SeaweedFS S3 IAM Integration Tests with Docker Compose
+.PHONY: help docker-build docker-up docker-down docker-logs docker-test docker-clean docker-status docker-keycloak-setup
+
+# Default target
+.DEFAULT_GOAL := help
+
+# Docker Compose configuration
+COMPOSE_FILE := docker-compose.yml
+PROJECT_NAME := seaweedfs-iam-test
+
+help: ## Show this help message
+	@echo "SeaweedFS S3 IAM Integration Tests - Docker Compose"
+	@echo ""
+	@echo "Available commands:"
+	@echo ""
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+	@echo ""
+	@echo "Environment:"
+	@echo "  COMPOSE_FILE: $(COMPOSE_FILE)"
+	@echo "  PROJECT_NAME: $(PROJECT_NAME)"
+
+docker-build: ## Build local SeaweedFS image for testing
+	@echo "🔨 Building local SeaweedFS image..."
+	@echo "Creating build directory..."
+	@cd ../../.. && mkdir -p .docker-build
+	@echo "Building weed binary..."
+	@cd ../../.. && cd weed && go build -o ../.docker-build/weed
+	@echo "Copying required files to build directory..."
+	@cd ../../.. && cp docker/filer.toml .docker-build/ && cp docker/entrypoint.sh .docker-build/
+	@echo "Building Docker image..."
+	@cd ../../.. && docker build -f docker/Dockerfile.local -t local/seaweedfs:latest .docker-build/
+	@echo "Cleaning up build directory..."
+	@cd ../../.. && rm -rf .docker-build
+	@echo "✅ Built local/seaweedfs:latest"
+
+docker-up: ## Start all services with Docker Compose
+	@echo "🚀 Starting SeaweedFS S3 IAM integration environment..."
+	@docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) up -d
+	@echo ""
+	@echo "✅ Environment started! Services will be available at:"
+	@echo "  🔐 Keycloak:     http://localhost:8080 (admin/admin)"
+	@echo "  🗄️  S3 API:       http://localhost:8333"
+	@echo "  📁 Filer:        http://localhost:8888"
+	@echo "  🎯 Master:       http://localhost:9333"
+	@echo ""
+	@echo "⏳ Waiting for all services to be healthy..."
+	@docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) ps
+
+docker-down: ## Stop and remove all containers
+	@echo "🛑 Stopping SeaweedFS S3 IAM integration environment..."
+	@docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) down -v
+	@echo "✅ Environment stopped and cleaned up"
+
+docker-restart: docker-down docker-up ## Restart the entire environment
+
+docker-logs: ## Show logs from all services
+	@docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) logs -f
+
+docker-logs-s3: ## Show logs from S3 service only
+	@docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) logs -f weed-s3
+
+docker-logs-keycloak: ## Show logs from Keycloak service only  
+	@docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) logs -f keycloak
+
+docker-status: ## Check status of all services
+	@echo "📊 Service Status:"
+	@docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) ps
+	@echo ""
+	@echo "🏥 Health Checks:"
+	@docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | grep $(PROJECT_NAME) || true
+
+docker-test: docker-wait-healthy ## Run integration tests against Docker environment
+	@echo "🧪 Running SeaweedFS S3 IAM integration tests..."
+	@echo ""
+	@KEYCLOAK_URL=http://localhost:8080 go test -v -timeout 10m ./...
+
+docker-test-single: ## Run a single test (use TEST_NAME=TestName)
+	@if [ -z "$(TEST_NAME)" ]; then \
+		echo "❌ Please specify TEST_NAME, e.g., make docker-test-single TEST_NAME=TestKeycloakAuthentication"; \
+		exit 1; \
+	fi
+	@echo "🧪 Running single test: $(TEST_NAME)"
+	@KEYCLOAK_URL=http://localhost:8080 go test -v -run "$(TEST_NAME)" -timeout 5m ./...
+
+docker-keycloak-setup: ## Manually run Keycloak setup (usually automatic)
+	@echo "🔧 Running Keycloak setup manually..."
+	@docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) run --rm keycloak-setup
+
+docker-clean: ## Clean up everything (containers, volumes, images)
+	@echo "🧹 Cleaning up Docker environment..."
+	@docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) down -v --remove-orphans
+	@docker system prune -f
+	@echo "✅ Cleanup complete"
+
+docker-shell-s3: ## Get shell access to S3 container
+	@docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) exec weed-s3 sh
+
+docker-shell-keycloak: ## Get shell access to Keycloak container  
+	@docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) exec keycloak bash
+
+docker-debug: ## Show debug information
+	@echo "🔍 Docker Environment Debug Information"
+	@echo ""
+	@echo "📋 Docker Compose Config:"
+	@docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) config
+	@echo ""
+	@echo "📊 Container Status:"
+	@docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) ps
+	@echo ""
+	@echo "🌐 Network Information:"
+	@docker network ls | grep $(PROJECT_NAME) || echo "No networks found"
+	@echo ""
+	@echo "💾 Volume Information:"  
+	@docker volume ls | grep $(PROJECT_NAME) || echo "No volumes found"
+
+# Quick test targets
+docker-test-auth: ## Quick test of authentication only
+	@KEYCLOAK_URL=http://localhost:8080 go test -v -run "TestKeycloakAuthentication" -timeout 2m ./...
+
+docker-test-roles: ## Quick test of role mapping only  
+	@KEYCLOAK_URL=http://localhost:8080 go test -v -run "TestKeycloakRoleMapping" -timeout 2m ./...
+
+docker-test-s3ops: ## Quick test of S3 operations only
+	@KEYCLOAK_URL=http://localhost:8080 go test -v -run "TestKeycloakS3Operations" -timeout 2m ./...
+
+# Development workflow
+docker-dev: docker-down docker-up docker-test ## Complete dev workflow: down -> up -> test
+
+# Show service URLs for easy access
+docker-urls: ## Display all service URLs
+	@echo "🌐 Service URLs:"
+	@echo ""  
+	@echo "  🔐 Keycloak Admin:   http://localhost:8080 (admin/admin)"
+	@echo "  🔐 Keycloak Realm:   http://localhost:8080/realms/seaweedfs-test"
+	@echo "  📁 S3 API:          http://localhost:8333"
+	@echo "  📂 Filer UI:        http://localhost:8888"  
+	@echo "  🎯 Master UI:       http://localhost:9333"
+	@echo "  💾 Volume Server:   http://localhost:8080"
+	@echo ""
+	@echo "  📖 Test Users:"
+	@echo "    • admin-user      (password: adminuser123)     - s3-admin role"
+	@echo "    • read-user       (password: readuser123)      - s3-read-only role"  
+	@echo "    • write-user      (password: writeuser123)     - s3-read-write role"
+	@echo "    • write-only-user (password: writeonlyuser123) - s3-write-only role"
+
+# Wait targets for CI/CD
+docker-wait-healthy: ## Wait for all services to be healthy
+	@echo "⏳ Waiting for all services to be healthy..."
+	@timeout 300 bash -c ' \
+		required_services="keycloak weed-master weed-volume weed-filer weed-s3"; \
+		while true; do \
+			all_healthy=true; \
+			for service in $$required_services; do \
+				if ! docker-compose -p $(PROJECT_NAME) -f $(COMPOSE_FILE) ps $$service | grep -q "healthy"; then \
+					echo "Waiting for $$service to be healthy..."; \
+					all_healthy=false; \
+					break; \
+				fi; \
+			done; \
+			if [ "$$all_healthy" = "true" ]; then \
+				break; \
+			fi; \
+			sleep 5; \
+		done \
+	'
+	@echo "✅ All required services are healthy"

test/s3/iam/README-Docker.md

@@ -0,0 +1,241 @@
+# SeaweedFS S3 IAM Integration with Docker Compose
+
+This directory contains a complete Docker Compose setup for testing SeaweedFS S3 IAM integration with Keycloak OIDC authentication.
+
+## 🚀 Quick Start
+
+1. **Build local SeaweedFS image:**
+   ```bash
+   make -f Makefile.docker docker-build
+   ```
+
+2. **Start the environment:**
+   ```bash
+   make -f Makefile.docker docker-up
+   ```
+
+3. **Run the tests:**
+   ```bash
+   make -f Makefile.docker docker-test
+   ```
+
+4. **Stop the environment:**
+   ```bash
+   make -f Makefile.docker docker-down
+   ```
+
+## 📋 What's Included
+
+The Docker Compose setup includes:
+
+- **🔐 Keycloak** - Identity provider with OIDC support
+- **🎯 SeaweedFS Master** - Metadata management
+- **💾 SeaweedFS Volume** - Data storage
+- **📁 SeaweedFS Filer** - File system interface
+- **📊 SeaweedFS S3** - S3-compatible API with IAM integration
+- **🔧 Keycloak Setup** - Automated realm and user configuration
+
+## 🌐 Service URLs
+
+After starting with `docker-up`, services are available at:
+
+| Service | URL | Credentials |
+|---------|-----|-------------|
+| 🔐 Keycloak Admin | http://localhost:8080 | admin/admin |
+| 📊 S3 API | http://localhost:8333 | JWT tokens |
+| 📁 Filer | http://localhost:8888 | - |
+| 🎯 Master | http://localhost:9333 | - |
+
+## 👥 Test Users
+
+The setup automatically creates test users in Keycloak:
+
+| Username | Password | Role | Permissions |
+|----------|----------|------|-------------|
+| admin-user | adminuser123 | s3-admin | Full S3 access |
+| read-user | readuser123 | s3-read-only | Read-only access |
+| write-user | writeuser123 | s3-read-write | Read and write |
+| write-only-user | writeonlyuser123 | s3-write-only | Write only |
+
+## 🧪 Running Tests
+
+### All Tests
+```bash
+make -f Makefile.docker docker-test
+```
+
+### Specific Test Categories
+```bash
+# Authentication tests only
+make -f Makefile.docker docker-test-auth
+
+# Role mapping tests only  
+make -f Makefile.docker docker-test-roles
+
+# S3 operations tests only
+make -f Makefile.docker docker-test-s3ops
+```
+
+### Single Test
+```bash
+make -f Makefile.docker docker-test-single TEST_NAME=TestKeycloakAuthentication
+```
+
+## 🔧 Development Workflow
+
+### Complete workflow (recommended)
+```bash
+# Build, start, test, and clean up
+make -f Makefile.docker docker-build
+make -f Makefile.docker docker-dev
+```
+This runs: build → down → up → test
+
+### Using Published Images (Alternative)
+If you want to use published Docker Hub images instead of building locally:
+```bash
+export SEAWEEDFS_IMAGE=chrislusf/seaweedfs:latest
+make -f Makefile.docker docker-up
+```
+
+### Manual steps
+```bash
+# Build image (required first time, or after code changes)
+make -f Makefile.docker docker-build
+
+# Start services
+make -f Makefile.docker docker-up
+
+# Watch logs
+make -f Makefile.docker docker-logs
+
+# Check status
+make -f Makefile.docker docker-status
+
+# Run tests
+make -f Makefile.docker docker-test
+
+# Stop services  
+make -f Makefile.docker docker-down
+```
+
+## 🔍 Debugging
+
+### View logs
+```bash
+# All services
+make -f Makefile.docker docker-logs
+
+# S3 service only (includes role mapping debug)
+make -f Makefile.docker docker-logs-s3  
+
+# Keycloak only
+make -f Makefile.docker docker-logs-keycloak
+```
+
+### Get shell access
+```bash
+# S3 container
+make -f Makefile.docker docker-shell-s3
+
+# Keycloak container
+make -f Makefile.docker docker-shell-keycloak
+```
+
+## 📁 File Structure
+
+```
+seaweedfs/test/s3/iam/
+├── docker-compose.yml          # Main Docker Compose configuration
+├── Makefile.docker             # Docker-specific Makefile
+├── setup_keycloak_docker.sh    # Keycloak setup for containers
+├── README-Docker.md            # This file
+├── iam_config.json            # IAM configuration (auto-generated)
+├── test_config.json           # S3 service configuration
+└── *_test.go                  # Go integration tests
+```
+
+## 🔄 Configuration
+
+### IAM Configuration
+The `setup_keycloak_docker.sh` script automatically generates `iam_config.json` with:
+
+- **OIDC Provider**: Keycloak configuration with proper container networking
+- **Role Mapping**: Maps Keycloak roles to SeaweedFS IAM roles
+- **Policies**: Defines S3 permissions for each role
+- **Trust Relationships**: Allows Keycloak users to assume SeaweedFS roles
+
+### Role Mapping Rules
+```json
+{
+  "claim": "roles",
+  "value": "s3-admin", 
+  "role": "arn:aws:iam::role/KeycloakAdminRole"
+}
+```
+
+## 🐛 Troubleshooting
+
+### Services not starting
+```bash
+# Check service status
+make -f Makefile.docker docker-status
+
+# View logs for specific service
+docker-compose -p seaweedfs-iam-test logs <service-name>
+```
+
+### Keycloak setup issues
+```bash  
+# Re-run Keycloak setup manually
+make -f Makefile.docker docker-keycloak-setup
+
+# Check Keycloak logs
+make -f Makefile.docker docker-logs-keycloak
+```
+
+### Role mapping not working
+```bash
+# Check S3 logs for role mapping debug messages
+make -f Makefile.docker docker-logs-s3 | grep -i "role\|claim\|mapping"
+```
+
+### Port conflicts
+If ports are already in use, modify `docker-compose.yml`:
+```yaml
+ports:
+  - "8081:8080"  # Change external port
+```
+
+## 🧹 Cleanup
+
+```bash
+# Stop containers and remove volumes
+make -f Makefile.docker docker-down
+
+# Complete cleanup (containers, volumes, images)
+make -f Makefile.docker docker-clean
+```
+
+## 🎯 Key Features
+
+- **Local Code Testing**: Uses locally built SeaweedFS images to test current code
+- **Isolated Environment**: No conflicts with local services
+- **Consistent Networking**: Services communicate via Docker network
+- **Automated Setup**: Keycloak realm and users created automatically
+- **Debug Logging**: Verbose logging enabled for troubleshooting  
+- **Health Checks**: Proper service dependency management
+- **Volume Persistence**: Data persists between restarts (until docker-down)
+
+## 🚦 CI/CD Integration
+
+For automated testing:
+
+```bash
+# Build image, run tests with proper cleanup
+make -f Makefile.docker docker-build
+make -f Makefile.docker docker-up
+make -f Makefile.docker docker-wait-healthy  
+make -f Makefile.docker docker-test
+make -f Makefile.docker docker-down
+```

test/s3/iam/README.md

@@ -0,0 +1,506 @@
+# SeaweedFS S3 IAM Integration Tests
+
+This directory contains comprehensive integration tests for the SeaweedFS S3 API with Advanced IAM (Identity and Access Management) system integration.
+
+## Overview
+
+**Important**: The STS service uses a **stateless JWT design** where all session information is embedded directly in the JWT token. No external session storage is required.
+
+The S3 IAM integration tests validate the complete end-to-end functionality of:
+
+- **JWT Authentication**: OIDC token-based authentication with S3 API
+- **Policy Enforcement**: Fine-grained access control for S3 operations
+- **Stateless Session Management**: JWT-based session token validation and expiration (no external storage)
+- **Role-Based Access Control (RBAC)**: IAM roles with different permission levels
+- **Bucket Policies**: Resource-based access control integration
+- **Multipart Upload IAM**: Policy enforcement for multipart operations
+- **Contextual Policies**: IP-based, time-based, and conditional access control
+- **Presigned URLs**: IAM-integrated temporary access URL generation
+
+## Test Architecture
+
+### Components Tested
+
+1. **S3 API Gateway** - SeaweedFS S3-compatible API server with IAM integration
+2. **IAM Manager** - Core IAM orchestration and policy evaluation
+3. **STS Service** - Security Token Service for temporary credentials
+4. **Policy Engine** - AWS IAM-compatible policy evaluation
+5. **Identity Providers** - OIDC and LDAP authentication providers
+6. **Policy Store** - Persistent policy storage using SeaweedFS filer
+
+### Test Framework
+
+- **S3IAMTestFramework**: Comprehensive test utilities and setup
+- **Mock OIDC Provider**: In-memory OIDC server with JWT signing
+- **Service Management**: Automatic SeaweedFS service lifecycle management
+- **Resource Cleanup**: Automatic cleanup of buckets and test data
+
+## Test Scenarios
+
+### 1. Authentication Tests (`TestS3IAMAuthentication`)
+
+- ✅ **Valid JWT Token**: Successful authentication with proper OIDC tokens
+- ✅ **Invalid JWT Token**: Rejection of malformed or invalid tokens  
+- ✅ **Expired JWT Token**: Proper handling of expired authentication tokens
+
+### 2. Policy Enforcement Tests (`TestS3IAMPolicyEnforcement`)
+
+- ✅ **Read-Only Policy**: Users can only read objects and list buckets
+- ✅ **Write-Only Policy**: Users can only create/delete objects but not read
+- ✅ **Admin Policy**: Full access to all S3 operations including bucket management
+
+### 3. Session Expiration Tests (`TestS3IAMSessionExpiration`)
+
+- ✅ **Short-Lived Sessions**: Creation and validation of time-limited sessions
+- ✅ **Manual Expiration**: Testing session expiration enforcement
+- ✅ **Expired Session Rejection**: Proper access denial for expired sessions
+
+### 4. Multipart Upload Tests (`TestS3IAMMultipartUploadPolicyEnforcement`)
+
+- ✅ **Admin Multipart Access**: Full multipart upload capabilities
+- ✅ **Read-Only Denial**: Rejection of multipart operations for read-only users
+- ✅ **Complete Upload Flow**: Initiate → Upload Parts → Complete workflow
+
+### 5. Bucket Policy Tests (`TestS3IAMBucketPolicyIntegration`)
+
+- ✅ **Public Read Policy**: Bucket-level policies allowing public access
+- ✅ **Explicit Deny Policy**: Bucket policies that override IAM permissions
+- ✅ **Policy CRUD Operations**: Get/Put/Delete bucket policy operations
+
+### 6. Contextual Policy Tests (`TestS3IAMContextualPolicyEnforcement`)
+
+- 🔧 **IP-Based Restrictions**: Source IP validation in policy conditions
+- 🔧 **Time-Based Restrictions**: Temporal access control policies
+- 🔧 **User-Agent Restrictions**: Request context-based policy evaluation
+
+### 7. Presigned URL Tests (`TestS3IAMPresignedURLIntegration`)
+
+- ✅ **URL Generation**: IAM-validated presigned URL creation
+- ✅ **Permission Validation**: Ensuring users have required permissions
+- 🔧 **HTTP Request Testing**: Direct HTTP calls to presigned URLs
+
+## Quick Start
+
+### Prerequisites
+
+1. **Go 1.26+** with modules enabled
+2. **SeaweedFS Binary** (`weed`) built with IAM support
+3. **Test Dependencies**:
+   ```bash
+   go get github.com/stretchr/testify
+   go get github.com/aws/aws-sdk-go
+   go get github.com/golang-jwt/jwt/v5
+   ```
+
+### Running Tests
+
+#### Complete Test Suite
+```bash
+# Run all tests with service management
+make test
+
+# Quick test run (assumes services running)
+make test-quick
+```
+
+#### Specific Test Categories
+```bash
+# Test only authentication
+make test-auth
+
+# Test only policy enforcement  
+make test-policy
+
+# Test only session expiration
+make test-expiration
+
+# Test only multipart uploads
+make test-multipart
+
+# Test only bucket policies
+make test-bucket-policy
+```
+
+#### Development & Debugging
+```bash
+# Start services and keep running
+make debug
+
+# Show service logs
+make logs
+
+# Check service status
+make status
+
+# Watch for changes and re-run tests
+make watch
+```
+
+### Manual Service Management
+
+If you prefer to manage services manually:
+
+```bash
+# Start services
+make start-services
+
+# Wait for services to be ready
+make wait-for-services
+
+# Run tests
+make run-tests
+
+# Stop services
+make stop-services
+```
+
+## Configuration
+
+### Test Configuration (`test_config.json`)
+
+The test configuration defines:
+
+- **Identity Providers**: OIDC and LDAP configurations
+- **IAM Roles**: Role definitions with trust policies
+- **IAM Policies**: Permission policies for different access levels
+- **Policy Stores**: Persistent storage configurations for IAM policies and roles
+
+### Service Ports
+
+| Service | Port | Purpose |
+|---------|------|---------|
+| Master | 9333 | Cluster coordination |
+| Volume | 8080 | Object storage |
+| Filer | 8888 | Metadata & IAM storage |
+| S3 API | 8333 | S3-compatible API with IAM |
+
+### Environment Variables
+
+```bash
+# SeaweedFS binary location
+export WEED_BINARY=../../../weed
+
+# Service ports (optional)
+export S3_PORT=8333
+export FILER_PORT=8888  
+export MASTER_PORT=9333
+export VOLUME_PORT=8080
+
+# Test timeout
+export TEST_TIMEOUT=30m
+
+# Log level (0-4)
+export LOG_LEVEL=2
+```
+
+## Test Data & Cleanup
+
+### Automatic Cleanup
+
+The test framework automatically:
+- 🗑️ **Deletes test buckets** created during tests
+- 🗑️ **Removes test objects** and multipart uploads
+- 🗑️ **Cleans up IAM sessions** and temporary tokens
+- 🗑️ **Stops services** after test completion
+
+### Manual Cleanup
+
+```bash
+# Clean everything
+make clean
+
+# Clean while keeping services running
+rm -rf test-volume-data/
+```
+
+## Extending Tests
+
+### Adding New Test Scenarios
+
+1. **Create Test Function**:
+   ```go
+   func TestS3IAMNewFeature(t *testing.T) {
+       framework := NewS3IAMTestFramework(t)
+       defer framework.Cleanup()
+       
+       // Test implementation
+   }
+   ```
+
+2. **Use Test Framework**:
+   ```go
+   // Create authenticated S3 client
+   s3Client, err := framework.CreateS3ClientWithJWT("user", "TestRole")
+   require.NoError(t, err)
+   
+   // Test S3 operations
+   err = framework.CreateBucket(s3Client, "test-bucket")
+   require.NoError(t, err)
+   ```
+
+3. **Add to Makefile**:
+   ```makefile
+   test-new-feature: ## Test new feature
+   	go test -v -run TestS3IAMNewFeature ./...
+   ```
+
+### Creating Custom Policies
+
+Add policies to `test_config.json`:
+
+```json
+{
+  "policies": {
+    "CustomPolicy": {
+      "Version": "2012-10-17",
+      "Statement": [
+        {
+          "Effect": "Allow",
+          "Action": ["s3:GetObject"],
+          "Resource": ["arn:aws:s3:::specific-bucket/*"],
+          "Condition": {
+            "StringEquals": {
+              "s3:prefix": ["allowed-prefix/"]
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+```
+
+### Adding Identity Providers
+
+1. **Mock Provider Setup**:
+   ```go
+   // In test framework
+   func (f *S3IAMTestFramework) setupCustomProvider() {
+       provider := custom.NewCustomProvider("test-custom")
+       // Configure and register
+   }
+   ```
+
+2. **Configuration**:
+   ```json
+   {
+     "providers": {
+       "custom": {
+         "test-custom": {
+           "endpoint": "http://localhost:8080",
+           "clientId": "custom-client"
+         }
+       }
+     }
+   }
+   ```
+
+## Troubleshooting
+
+### Common Issues
+
+#### 1. Services Not Starting
+```bash
+# Check if ports are available
+netstat -an | grep -E "(8333|8888|9333|8080)"
+
+# Check service logs
+make logs
+
+# Try different ports
+export S3_PORT=18333
+make start-services
+```
+
+#### 2. JWT Token Issues
+```bash
+# Verify OIDC mock server
+curl http://localhost:8080/.well-known/openid_configuration
+
+# Check JWT token format in logs
+make logs | grep -i jwt
+```
+
+#### 3. Permission Denied Errors
+```bash
+# Verify IAM configuration
+cat test_config.json | jq '.policies'
+
+# Check policy evaluation in logs  
+export LOG_LEVEL=4
+make start-services
+```
+
+#### 4. Test Timeouts
+```bash
+# Increase timeout
+export TEST_TIMEOUT=60m
+make test
+
+# Run individual tests
+make test-auth
+```
+
+### Debug Mode
+
+Start services in debug mode to inspect manually:
+
+```bash
+# Start and keep running
+make debug
+
+# In another terminal, run specific operations
+aws s3 ls --endpoint-url http://localhost:8333
+
+# Stop when done (Ctrl+C in debug terminal)
+```
+
+### Log Analysis
+
+```bash
+# Service-specific logs
+tail -f weed-s3.log       # S3 API server
+tail -f weed-filer.log    # Filer (IAM storage)  
+tail -f weed-master.log   # Master server
+tail -f weed-volume.log   # Volume server
+
+# Filter for IAM-related logs
+make logs | grep -i iam
+make logs | grep -i jwt
+make logs | grep -i policy
+```
+
+## Performance Testing
+
+### Benchmarks
+
+```bash
+# Run performance benchmarks
+make benchmark
+
+# Profile memory usage  
+go test -bench=. -memprofile=mem.prof
+go tool pprof mem.prof
+```
+
+### Load Testing
+
+For load testing with IAM:
+
+1. **Create Multiple Clients**:
+   ```go
+   // Generate multiple JWT tokens
+   tokens := framework.GenerateMultipleJWTTokens(100)
+   
+   // Create concurrent clients
+   var wg sync.WaitGroup
+   for _, token := range tokens {
+       wg.Add(1)
+       go func(token string) {
+           defer wg.Done()
+           // Perform S3 operations
+       }(token)
+   }
+   wg.Wait()
+   ```
+
+2. **Measure Performance**:
+   ```bash
+   # Run with verbose output
+   go test -v -bench=BenchmarkS3IAMOperations
+   ```
+
+## CI/CD Integration
+
+### GitHub Actions
+
+```yaml
+name: S3 IAM Integration Tests
+on: [push, pull_request]
+
+jobs:
+  s3-iam-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: 'go.mod'
+      
+      - name: Build SeaweedFS
+        run: go build -o weed ./main.go
+      
+      - name: Run S3 IAM Tests  
+        run: |
+          cd test/s3/iam
+          make ci
+```
+
+### Jenkins Pipeline
+
+```groovy
+pipeline {
+    agent any
+    stages {
+        stage('Build') {
+            steps {
+                sh 'go build -o weed ./main.go'
+            }
+        }
+        stage('S3 IAM Tests') {
+            steps {
+                dir('test/s3/iam') {
+                    sh 'make ci'
+                }
+            }
+            post {
+                always {
+                    dir('test/s3/iam') {
+                        sh 'make clean'
+                    }
+                }
+            }
+        }
+    }
+}
+```
+
+## Contributing
+
+### Adding New Tests
+
+1. **Follow Test Patterns**:
+   - Use `S3IAMTestFramework` for setup
+   - Include cleanup with `defer framework.Cleanup()`
+   - Use descriptive test names and subtests
+   - Assert both success and failure cases
+
+2. **Update Documentation**:
+   - Add test descriptions to this README
+   - Include Makefile targets for new test categories
+   - Document any new configuration options
+
+3. **Ensure Test Reliability**:
+   - Tests should be deterministic and repeatable
+   - Include proper error handling and assertions
+   - Use appropriate timeouts for async operations
+
+### Code Style
+
+- Follow standard Go testing conventions
+- Use `require.NoError()` for critical assertions
+- Use `assert.Equal()` for value comparisons  
+- Include descriptive error messages in assertions
+
+## Support
+
+For issues with S3 IAM integration tests:
+
+1. **Check Logs**: Use `make logs` to inspect service logs
+2. **Verify Configuration**: Ensure `test_config.json` is correct
+3. **Test Services**: Run `make status` to check service health
+4. **Clean Environment**: Try `make clean && make test`
+
+## License
+
+This test suite is part of the SeaweedFS project and follows the same licensing terms.

test/s3/iam/STS_DISTRIBUTED.md

@@ -0,0 +1,559 @@
+# Distributed STS Service for SeaweedFS S3 Gateway
+
+This document explains how to configure and deploy the STS (Security Token Service) for distributed SeaweedFS S3 Gateway deployments with consistent identity provider configurations.
+
+## Problem Solved
+
+Previously, identity providers had to be **manually registered** on each S3 gateway instance, leading to:
+
+- ❌ **Inconsistent authentication**: Different instances might have different providers
+- ❌ **Manual synchronization**: No guarantee all instances have same provider configs
+- ❌ **Authentication failures**: Users getting different responses from different instances
+- ❌ **Operational complexity**: Difficult to manage provider configurations at scale
+
+## Solution: Configuration-Driven Providers
+
+The STS service now supports **automatic provider loading** from configuration files, ensuring:
+
+- ✅ **Consistent providers**: All instances load identical providers from config
+- ✅ **Automatic synchronization**: Configuration-driven, no manual registration needed
+- ✅ **Reliable authentication**: Same behavior from all instances
+- ✅ **Easy management**: Update config file, restart services
+
+## Configuration Schema
+
+### Basic STS Configuration
+
+```json
+{
+  "sts": {
+    "tokenDuration": "1h",
+    "maxSessionLength": "12h", 
+    "issuer": "seaweedfs-sts",
+    "signingKey": "base64-encoded-signing-key-32-chars-min"
+  }
+}
+```
+
+**Note**: The STS service uses a **stateless JWT design** where all session information is embedded directly in the JWT token. No external session storage is required.
+
+### Configuration-Driven Providers
+
+```json
+{
+  "sts": {
+    "tokenDuration": "1h",
+    "maxSessionLength": "12h",
+    "issuer": "seaweedfs-sts",
+    "signingKey": "base64-encoded-signing-key",
+    "providers": [
+      {
+        "name": "keycloak-oidc",
+        "type": "oidc", 
+        "enabled": true,
+        "config": {
+          "issuer": "https://keycloak.company.com/realms/seaweedfs",
+          "clientId": "seaweedfs-s3",
+          "clientSecret": "super-secret-key",
+          "jwksUri": "https://keycloak.company.com/realms/seaweedfs/protocol/openid-connect/certs",
+          "scopes": ["openid", "profile", "email", "roles"],
+          "claimsMapping": {
+            "usernameClaim": "preferred_username",
+            "groupsClaim": "roles"
+          }
+        }
+      },
+      {
+        "name": "backup-oidc",
+        "type": "oidc",
+        "enabled": false,
+        "config": {
+          "issuer": "https://backup-oidc.company.com",
+          "clientId": "seaweedfs-backup"
+        }
+      },
+      {
+        "name": "dev-mock-provider",
+        "type": "mock",
+        "enabled": true,
+        "config": {
+          "issuer": "http://localhost:9999",
+          "clientId": "mock-client"
+        }
+      }
+    ]
+  }
+}
+```
+
+## Supported Provider Types
+
+### 1. OIDC Provider (`"type": "oidc"`)
+
+For production authentication with OpenID Connect providers like Keycloak, Auth0, Google, etc.
+
+**Required Configuration:**
+- `issuer`: OIDC issuer URL
+- `clientId`: OAuth2 client ID
+
+**Optional Configuration:**
+- `clientSecret`: OAuth2 client secret (for confidential clients)
+- `jwksUri`: JSON Web Key Set URI (auto-discovered if not provided)
+- `userInfoUri`: UserInfo endpoint URI (auto-discovered if not provided)
+- `scopes`: OAuth2 scopes to request (default: `["openid"]`)
+- `claimsMapping`: Map OIDC claims to identity attributes
+
+**Example:**
+```json
+{
+  "name": "corporate-keycloak",
+  "type": "oidc",
+  "enabled": true,
+  "config": {
+    "issuer": "https://sso.company.com/realms/production",
+    "clientId": "seaweedfs-prod",
+    "clientSecret": "confidential-secret", 
+    "scopes": ["openid", "profile", "email", "groups"],
+    "claimsMapping": {
+      "usernameClaim": "preferred_username",
+      "groupsClaim": "groups",
+      "emailClaim": "email"
+    }
+  }
+}
+```
+
+### 2. Mock Provider (`"type": "mock"`)
+
+For development, testing, and staging environments.
+
+**Configuration:**
+- `issuer`: Mock issuer URL (default: `http://localhost:9999`)
+- `clientId`: Mock client ID
+
+**Example:**
+```json
+{
+  "name": "dev-mock",
+  "type": "mock", 
+  "enabled": true,
+  "config": {
+    "issuer": "http://dev-mock:9999",
+    "clientId": "dev-client"
+  }
+}
+```
+
+**Built-in Test Tokens:**
+- `valid_test_token`: Returns test user with developer groups
+- `valid-oidc-token`: Compatible with integration tests
+- `expired_token`: Returns token expired error
+- `invalid_token`: Returns invalid token error
+
+### 3. Future Provider Types
+
+The factory pattern supports easy addition of new provider types:
+
+- `"type": "ldap"`: LDAP/Active Directory authentication
+- `"type": "saml"`: SAML 2.0 authentication  
+- `"type": "oauth2"`: Generic OAuth2 providers
+- `"type": "custom"`: Custom authentication backends
+
+## Deployment Patterns
+
+### Single Instance (Development)
+
+```bash
+# Standard deployment with config-driven providers
+weed s3 -filer=localhost:8888 -port=8333 -iam.config=/path/to/sts_config.json
+```
+
+### Multiple Instances (Production)
+
+```bash
+# Instance 1 
+weed s3 -filer=prod-filer:8888 -port=8333 -iam.config=/shared/sts_distributed.json
+
+# Instance 2
+weed s3 -filer=prod-filer:8888 -port=8334 -iam.config=/shared/sts_distributed.json
+
+# Instance N
+weed s3 -filer=prod-filer:8888 -port=833N -iam.config=/shared/sts_distributed.json
+```
+
+**Critical Requirements for Distributed Deployment:**
+
+1. **Identical Configuration Files**: All instances must use the exact same configuration file
+2. **Same Signing Keys**: All instances must have identical `signingKey` values
+3. **Same Issuer**: All instances must use the same `issuer` value
+
+**Note**: STS now uses stateless JWT tokens, eliminating the need for shared session storage.
+
+### High Availability Setup
+
+```yaml
+# docker-compose.yml for production deployment
+services:
+  filer:
+    image: seaweedfs/seaweedfs:latest
+    command: "filer -master=master:9333"
+    volumes:
+      - filer-data:/data
+    
+  s3-gateway-1:
+    image: seaweedfs/seaweedfs:latest
+    command: "s3 -filer=filer:8888 -port=8333 -iam.config=/config/sts_distributed.json"
+    ports:
+      - "8333:8333"
+    volumes:
+      - ./sts_distributed.json:/config/sts_distributed.json:ro
+    depends_on: [filer]
+    
+  s3-gateway-2:
+    image: seaweedfs/seaweedfs:latest 
+    command: "s3 -filer=filer:8888 -port=8333 -iam.config=/config/sts_distributed.json"
+    ports:
+      - "8334:8333"
+    volumes:
+      - ./sts_distributed.json:/config/sts_distributed.json:ro
+    depends_on: [filer]
+    
+  s3-gateway-3:
+    image: seaweedfs/seaweedfs:latest
+    command: "s3 -filer=filer:8888 -port=8333 -iam.config=/config/sts_distributed.json"
+    ports:
+      - "8335:8333"
+    volumes:
+      - ./sts_distributed.json:/config/sts_distributed.json:ro
+    depends_on: [filer]
+    
+  load-balancer:
+    image: nginx:alpine
+    ports:
+      - "80:80"
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+    depends_on: [s3-gateway-1, s3-gateway-2, s3-gateway-3]
+```
+
+## Authentication Flow
+
+### 1. OIDC Authentication Flow
+
+```
+1. User authenticates with OIDC provider (Keycloak, Auth0, etc.)
+   ↓
+2. User receives OIDC JWT token from provider
+   ↓  
+3. User calls SeaweedFS STS AssumeRoleWithWebIdentity
+   POST /sts/assume-role-with-web-identity
+   {
+     "RoleArn": "arn:aws:iam::role/S3AdminRole",
+     "WebIdentityToken": "eyJ0eXAiOiJKV1QiLCJhbGc...",
+     "RoleSessionName": "user-session"
+   }
+   ↓
+4. STS validates OIDC token with configured provider
+   - Verifies JWT signature using provider's JWKS
+   - Validates issuer, audience, expiration
+   - Extracts user identity and groups
+   ↓
+5. STS checks role trust policy
+   - Verifies user/groups can assume the requested role
+   - Validates conditions in trust policy
+   ↓
+6. STS generates temporary credentials
+   - Creates temporary access key, secret key, session token
+   - Session token is signed JWT with all session information embedded (stateless)
+   ↓
+7. User receives temporary credentials
+   {
+     "Credentials": {
+       "AccessKeyId": "AKIA...",
+       "SecretAccessKey": "base64-secret",
+       "SessionToken": "eyJ0eXAiOiJKV1QiLCJhbGc...",
+       "Expiration": "2024-01-01T12:00:00Z"
+     }
+   }
+   ↓
+8. User makes S3 requests with temporary credentials
+   - AWS SDK signs requests with temporary credentials
+   - SeaweedFS S3 gateway validates session token
+   - Gateway checks permissions via policy engine
+```
+
+### 2. Cross-Instance Token Validation
+
+```
+User Request → Load Balancer → Any S3 Gateway Instance
+                                      ↓
+                              Extract JWT Session Token
+                                      ↓
+                              Validate JWT Token
+                              (Self-contained - no external storage needed)
+                                      ↓
+                              Check Permissions
+                              (Shared policy engine)
+                                      ↓
+                              Allow/Deny Request
+```
+
+## Trust Policy Conditions
+
+Step 5 above evaluates the role's trust policy against context keys derived from the
+OIDC token's claims. The available keys are:
+
+| Condition key | Source |
+|---------------|--------|
+| `oidc:iss` | `iss` claim (issuer URL) |
+| `oidc:sub` | `sub` claim |
+| `oidc:aud` | `aud` claim |
+| `oidc:<claim>` | any other token claim, e.g. `oidc:roles`, `oidc:groups`, `oidc:email` |
+| `aws:FederatedProvider` | the provider `name` (e.g. `keycloak-oidc`) when its configured issuer matches the token, otherwise the raw issuer URL |
+| `aws:userid` | `sub` claim (same value as `oidc:sub` during trust-policy evaluation) |
+| `sts:DurationSeconds` | requested session duration, when supplied |
+
+During trust-policy evaluation `aws:userid` is the raw `sub` claim. Once the
+role has been assumed, the keys seen by request authorization differ: there
+`aws:userid` is a stable per-identity hash of `sub` and `iss` (see
+`ComputeParentUser`), so do not assume the two contexts carry the same value.
+
+Custom claims are always exposed under the `oidc:` prefix, so a trust policy must use
+`oidc:roles` (not a bare `roles`) to match a `roles` claim:
+
+```json
+"Condition": {
+  "StringEquals": {
+    "oidc:roles": "s3-admin"
+  }
+}
+```
+
+A multi-valued claim (such as a `roles` array) matches when any of its values equals
+the condition value. The same `oidc:` keys can be interpolated into policy resources,
+e.g. `arn:aws:s3:::bucket/${oidc:sub}/*`.
+
+### roleMapping vs. trust policy
+
+A provider's `roleMapping` and a role's trust policy apply to two different entry
+points and are not interchangeable:
+
+- **Direct OIDC** — an S3 request carrying `Authorization: Bearer <OIDC-JWT>`. The
+  gateway applies `roleMapping` to choose the caller's role from the token claims; the
+  first matching rule (or `defaultRole`) wins.
+- **STS `AssumeRoleWithWebIdentity`** — the caller names the role explicitly via
+  `RoleArn`, and that role's trust policy decides whether the assumption is allowed.
+  `roleMapping` does not select the role on this path; instead the token claims are
+  surfaced as the `oidc:` condition keys above for the trust policy to evaluate.
+
+## Configuration Management
+
+### Development Environment
+
+```json
+{
+  "sts": {
+    "tokenDuration": "1h",
+    "maxSessionLength": "12h",
+    "issuer": "seaweedfs-dev-sts",
+    "signingKey": "ZGV2LXNpZ25pbmcta2V5LTMyLWNoYXJhY3RlcnMtbG9uZw==",
+    "providers": [
+      {
+        "name": "dev-mock",
+        "type": "mock",
+        "enabled": true,
+        "config": {
+          "issuer": "http://localhost:9999",
+          "clientId": "dev-mock-client"
+        }
+      }
+    ]
+  }
+}
+```
+
+### Production Environment
+
+```json
+{
+  "sts": {
+    "tokenDuration": "1h",
+    "maxSessionLength": "12h",
+    "issuer": "seaweedfs-prod-sts",
+    "signingKey": "cHJvZC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmctcmFuZG9t",
+    "providers": [
+      {
+        "name": "corporate-sso",
+        "type": "oidc",
+        "enabled": true,
+        "config": {
+          "issuer": "https://sso.company.com/realms/production",
+          "clientId": "seaweedfs-prod",
+          "clientSecret": "${SSO_CLIENT_SECRET}",
+          "scopes": ["openid", "profile", "email", "groups"],
+          "claimsMapping": {
+            "usernameClaim": "preferred_username",
+            "groupsClaim": "groups"
+          }
+        }
+      },
+      {
+        "name": "backup-auth",
+        "type": "oidc", 
+        "enabled": false,
+        "config": {
+          "issuer": "https://backup-sso.company.com",
+          "clientId": "seaweedfs-backup"
+        }
+      }
+    ]
+  }
+}
+```
+
+## Operational Best Practices
+
+### 1. Configuration Management
+
+- **Version Control**: Store configurations in Git with proper versioning
+- **Environment Separation**: Use separate configs for dev/staging/production
+- **Secret Management**: Use environment variable substitution for secrets
+- **Configuration Validation**: Test configurations before deployment
+
+### 2. Security Considerations
+
+- **Signing Key Security**: Use strong, randomly generated signing keys (32+ bytes)
+- **Key Rotation**: Implement signing key rotation procedures
+- **Secret Storage**: Store client secrets in secure secret management systems
+- **TLS Encryption**: Always use HTTPS for OIDC providers in production
+
+### 3. Monitoring and Troubleshooting
+
+- **Provider Health**: Monitor OIDC provider availability and response times
+- **Session Metrics**: Track active sessions, token validation errors
+- **Configuration Drift**: Alert on configuration inconsistencies between instances
+- **Authentication Logs**: Log authentication attempts for security auditing
+
+### 4. Capacity Planning
+
+- **Provider Performance**: Monitor OIDC provider response times and rate limits
+- **Token Validation**: Monitor JWT validation performance and caching
+- **Memory Usage**: Monitor JWT token validation caching and provider metadata
+
+## Migration Guide
+
+### From Manual Provider Registration
+
+**Before (Manual Registration):**
+```go
+// Each instance needs this code
+keycloakProvider := oidc.NewOIDCProvider("keycloak-oidc")
+keycloakProvider.Initialize(keycloakConfig)
+stsService.RegisterProvider(keycloakProvider)
+```
+
+**After (Configuration-Driven):**
+```json
+{
+  "sts": {
+    "providers": [
+      {
+        "name": "keycloak-oidc",
+        "type": "oidc",
+        "enabled": true,
+        "config": {
+          "issuer": "https://keycloak.company.com/realms/seaweedfs",
+          "clientId": "seaweedfs-s3"
+        }
+      }
+    ]
+  }
+}
+```
+
+### Migration Steps
+
+1. **Create Configuration File**: Convert manual provider registrations to JSON config
+2. **Test Single Instance**: Deploy config to one instance and verify functionality
+3. **Validate Consistency**: Ensure all instances load identical providers
+4. **Rolling Deployment**: Update instances one by one with new configuration
+5. **Remove Manual Code**: Clean up manual provider registration code
+
+## Troubleshooting
+
+### Common Issues
+
+#### 1. Provider Inconsistency
+
+**Symptoms**: Authentication works on some instances but not others
+**Diagnosis**: 
+```bash
+# Check provider counts on each instance
+curl http://instance1:8333/sts/providers | jq '.providers | length'
+curl http://instance2:8334/sts/providers | jq '.providers | length'
+```
+**Solution**: Ensure all instances use identical configuration files
+
+#### 2. Token Validation Failures
+
+**Symptoms**: "Invalid signature" or "Invalid issuer" errors
+**Diagnosis**: Check signing key and issuer consistency
+**Solution**: Verify `signingKey` and `issuer` are identical across all instances
+
+#### 3. Provider Loading Failures
+
+**Symptoms**: Providers not loaded at startup
+**Diagnosis**: Check logs for provider initialization errors
+**Solution**: Validate provider configuration against schema
+
+#### 4. OIDC Provider Connectivity
+
+**Symptoms**: "Failed to fetch JWKS" errors
+**Diagnosis**: Test OIDC provider connectivity from all instances
+**Solution**: Check network connectivity, DNS resolution, certificates
+
+### Debug Commands
+
+```bash
+# Test configuration loading
+weed s3 -iam.config=/path/to/config.json -test.config
+
+# Validate JWT tokens
+curl -X POST http://localhost:8333/sts/validate-token \
+  -H "Content-Type: application/json" \
+  -d '{"sessionToken": "eyJ0eXAiOiJKV1QiLCJhbGc..."}'
+
+# List loaded providers
+curl http://localhost:8333/sts/providers
+
+# Check session store
+curl http://localhost:8333/sts/sessions/count
+```
+
+## Performance Considerations
+
+### Token Validation Performance
+
+- **JWT Validation**: ~1-5ms per token validation
+- **JWKS Caching**: Cache JWKS responses to reduce OIDC provider load
+- **Session Lookup**: Filer session lookup adds ~10-20ms latency
+- **Concurrent Requests**: Each instance can handle 1000+ concurrent validations
+
+### Scaling Recommendations
+
+- **Horizontal Scaling**: Add more S3 gateway instances behind load balancer
+- **Session Store Optimization**: Use SSD storage for filer session store
+- **Provider Caching**: Implement JWKS caching to reduce provider load
+- **Connection Pooling**: Use connection pooling for filer communication
+
+## Summary
+
+The configuration-driven provider system solves critical distributed deployment issues:
+
+- ✅ **Automatic Provider Loading**: No manual registration code required
+- ✅ **Configuration Consistency**: All instances load identical providers from config
+- ✅ **Easy Management**: Update config file, restart services
+- ✅ **Production Ready**: Supports OIDC, proper session management, distributed storage
+- ✅ **Backwards Compatible**: Existing manual registration still works
+
+This enables SeaweedFS S3 Gateway to **scale horizontally** with **consistent authentication** across all instances, making it truly **production-ready for enterprise deployments**.

test/s3/iam/docker-compose-simple.yml

@@ -0,0 +1,20 @@
+services:
+  # Keycloak Identity Provider
+  keycloak:
+    image: quay.io/keycloak/keycloak:26.0.7
+    container_name: keycloak-test-simple
+    ports:
+      - "8080:8080"
+    environment:
+      KC_BOOTSTRAP_ADMIN_USERNAME: admin
+      KC_BOOTSTRAP_ADMIN_PASSWORD: admin
+      KC_HTTP_ENABLED: "true"
+      KC_HOSTNAME_STRICT: "false"
+      KC_HOSTNAME_STRICT_HTTPS: "false"
+    command: start-dev
+    networks:
+      - test-network
+
+networks:
+  test-network:
+    driver: bridge

test/s3/iam/docker-compose.test.yml

@@ -0,0 +1,160 @@
+# Docker Compose for SeaweedFS S3 IAM Integration Tests
+services:
+  # SeaweedFS Master
+  seaweedfs-master:
+    image: chrislusf/seaweedfs:latest
+    container_name: seaweedfs-master-test
+    command: master -mdir=/data -defaultReplication=000 -port=9333
+    ports:
+      - "9333:9333"
+    volumes:
+      - master-data:/data
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9333/cluster/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - seaweedfs-test
+
+  # SeaweedFS Volume  
+  seaweedfs-volume:
+    image: chrislusf/seaweedfs:latest
+    container_name: seaweedfs-volume-test
+    command: volume -dir=/data -port=8083 -master=seaweedfs-master:9333
+    ports:
+      - "8083:8083"
+    volumes:
+      - volume-data:/data
+    depends_on:
+      seaweedfs-master:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8083/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - seaweedfs-test
+
+  # SeaweedFS Filer
+  seaweedfs-filer:
+    image: chrislusf/seaweedfs:latest
+    container_name: seaweedfs-filer-test
+    command: filer -port=8888 -master=seaweedfs-master:9333 -defaultStoreDir=/data
+    ports:
+      - "8888:8888"
+    volumes:
+      - filer-data:/data
+    depends_on:
+      seaweedfs-master:
+        condition: service_healthy
+      seaweedfs-volume:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8888/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - seaweedfs-test
+
+  # SeaweedFS S3 API
+  seaweedfs-s3:
+    image: chrislusf/seaweedfs:latest
+    container_name: seaweedfs-s3-test
+    command: s3 -port=8333 -filer=seaweedfs-filer:8888 -config=/config/test_config.json
+    ports:
+      - "8333:8333"
+    volumes:
+      - ./test_config.json:/config/test_config.json:ro
+    depends_on:
+      seaweedfs-filer:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8333/"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - seaweedfs-test
+
+  # Test Runner
+  integration-tests:
+    build:
+      context: ../../../
+      dockerfile: test/s3/iam/Dockerfile.s3
+    container_name: seaweedfs-s3-iam-tests
+    environment:
+      - WEED_BINARY=weed
+      - S3_PORT=8333
+      - FILER_PORT=8888
+      - MASTER_PORT=9333
+      - VOLUME_PORT=8083
+      - TEST_TIMEOUT=30m
+      - LOG_LEVEL=2
+    depends_on:
+      seaweedfs-s3:
+        condition: service_healthy
+    volumes:
+      - .:/app/test/s3/iam
+      - test-results:/app/test-results
+    networks:
+      - seaweedfs-test
+    command: ["make", "test"]
+
+  # Optional: Mock LDAP Server for LDAP testing
+  ldap-server:
+    image: osixia/openldap:1.5.0
+    container_name: ldap-server-test
+    environment:
+      LDAP_ORGANISATION: "Example Corp"
+      LDAP_DOMAIN: "example.com"
+      LDAP_ADMIN_PASSWORD: "admin-password"
+      LDAP_CONFIG_PASSWORD: "config-password"
+      LDAP_READONLY_USER: "true"
+      LDAP_READONLY_USER_USERNAME: "readonly"
+      LDAP_READONLY_USER_PASSWORD: "readonly-password"
+    ports:
+      - "389:389"
+      - "636:636"
+    volumes:
+      - ldap-data:/var/lib/ldap
+      - ldap-config:/etc/ldap/slapd.d
+    networks:
+      - seaweedfs-test
+
+  # Optional: LDAP Admin UI
+  ldap-admin:
+    image: osixia/phpldapadmin:latest
+    container_name: ldap-admin-test
+    environment:
+      PHPLDAPADMIN_LDAP_HOSTS: "ldap-server"
+      PHPLDAPADMIN_HTTPS: "false"
+    ports:
+      - "8080:80"
+    depends_on:
+      - ldap-server
+    networks:
+      - seaweedfs-test
+
+volumes:
+  master-data:
+    driver: local
+  volume-data:
+    driver: local
+  filer-data:
+    driver: local
+  ldap-data:
+    driver: local
+  ldap-config:
+    driver: local
+  test-results:
+    driver: local
+
+networks:
+  seaweedfs-test:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.20.0.0/16

test/s3/iam/docker-compose.yml

@@ -0,0 +1,160 @@
+services:
+  # Keycloak Identity Provider
+  keycloak:
+    image: quay.io/keycloak/keycloak:26.0.7
+    container_name: keycloak-iam-test
+    hostname: keycloak
+    environment:
+      KC_BOOTSTRAP_ADMIN_USERNAME: admin
+      KC_BOOTSTRAP_ADMIN_PASSWORD: admin
+      KC_HTTP_ENABLED: "true"
+      KC_HOSTNAME_STRICT: "false"
+      KC_HOSTNAME_STRICT_HTTPS: "false"
+      KC_HTTP_RELATIVE_PATH: /
+    ports:
+      - "8080:8080"
+    command: start-dev
+    networks:
+      - seaweedfs-iam
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080/health/ready"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 60s
+
+  # SeaweedFS Master
+  weed-master:
+    image: ${SEAWEEDFS_IMAGE:-local/seaweedfs:latest}
+    container_name: weed-master
+    hostname: weed-master
+    ports:
+      - "9333:9333"
+      - "19333:19333"
+    command: "master -ip=weed-master -port=9333 -mdir=/data"
+    volumes:
+      - master-data:/data
+    networks:
+      - seaweedfs-iam
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://localhost:9333/cluster/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+
+  # SeaweedFS Volume Server
+  weed-volume:
+    image: ${SEAWEEDFS_IMAGE:-local/seaweedfs:latest}
+    container_name: weed-volume
+    hostname: weed-volume
+    ports:
+      - "8083:8083"
+      - "18083:18083"
+    command: "volume -ip=weed-volume -port=8083 -dir=/data -master=weed-master:9333 -dataCenter=dc1 -rack=rack1"
+    volumes:
+      - volume-data:/data
+    networks:
+      - seaweedfs-iam
+    depends_on:
+      weed-master:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://localhost:8083/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+
+  # SeaweedFS Filer
+  weed-filer:
+    image: ${SEAWEEDFS_IMAGE:-local/seaweedfs:latest}
+    container_name: weed-filer
+    hostname: weed-filer
+    ports:
+      - "8888:8888"
+      - "18888:18888"
+    command: "filer -ip=weed-filer -port=8888 -master=weed-master:9333 -defaultStoreDir=/data"
+    volumes:
+      - filer-data:/data
+    networks:
+      - seaweedfs-iam
+    depends_on:
+      weed-master:
+        condition: service_healthy
+      weed-volume:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://localhost:8888/status"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+
+  # SeaweedFS S3 API with IAM
+  weed-s3:
+    image: ${SEAWEEDFS_IMAGE:-local/seaweedfs:latest}
+    container_name: weed-s3
+    hostname: weed-s3
+    ports:
+      - "8333:8333"
+    environment:
+      WEED_FILER: "weed-filer:8888"
+      WEED_IAM_CONFIG: "/config/iam_config.json"
+      WEED_S3_CONFIG: "/config/test_config.json"
+      GLOG_v: "3"
+    command: >
+      sh -c "
+        echo 'Starting S3 API with IAM...' &&
+        weed -v=3 s3 -ip=weed-s3 -port=8333 
+          -filer=weed-filer:8888 
+          -config=/config/test_config.json 
+          -iam.config=/config/iam_config.json
+      "
+    volumes:
+      - ./iam_config.json:/config/iam_config.json:ro
+      - ./test_config.json:/config/test_config.json:ro
+    networks:
+      - seaweedfs-iam
+    depends_on:
+      weed-filer:
+        condition: service_healthy
+      keycloak:
+        condition: service_healthy
+      keycloak-setup:
+        condition: service_completed_successfully
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://localhost:8333"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+
+  # Keycloak Setup Service
+  keycloak-setup:
+    image: alpine/curl:8.4.0
+    container_name: keycloak-setup
+    volumes:
+      - ./setup_keycloak_docker.sh:/setup.sh:ro
+      - .:/workspace:rw
+    working_dir: /workspace
+    networks:
+      - seaweedfs-iam
+    depends_on:
+      keycloak:
+        condition: service_healthy
+    command: >
+      sh -c "
+        apk add --no-cache bash jq &&
+        chmod +x /setup.sh &&
+        /setup.sh
+      "
+
+volumes:
+  master-data:
+  volume-data:
+  filer-data:
+
+networks:
+  seaweedfs-iam:
+    driver: bridge

test/s3/iam/empty_s3_config.json

@@ -0,0 +1,3 @@
+{
+    "identities": []
+}

test/s3/iam/go.mod

@@ -0,0 +1,16 @@
+module github.com/seaweedfs/seaweedfs/test/s3/iam
+
+go 1.24
+
+require (
+	github.com/aws/aws-sdk-go v1.44.0
+	github.com/golang-jwt/jwt/v5 v5.3.0
+	github.com/stretchr/testify v1.8.4
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

test/s3/iam/go.sum

@@ -0,0 +1,31 @@
+github.com/aws/aws-sdk-go v1.44.0 h1:jwtHuNqfnJxL4DKHBUVUmQlfueQqBW7oXP6yebZR/R0=
+github.com/aws/aws-sdk-go v1.44.0/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
+github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

test/s3/iam/iam_config.github.json

@@ -0,0 +1,293 @@
+{
+  "sts": {
+    "tokenDuration": "1h",
+    "maxSessionLength": "12h", 
+    "issuer": "seaweedfs-sts",
+    "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc="
+  },
+  "providers": [
+    {
+      "name": "test-oidc",
+      "type": "mock",
+      "config": {
+        "issuer": "test-oidc-issuer",
+        "clientId": "test-oidc-client"
+      }
+    },
+    {
+      "name": "keycloak",
+      "type": "oidc",
+      "enabled": true,
+      "config": {
+        "issuer": "http://localhost:8080/realms/seaweedfs-test",
+        "clientId": "seaweedfs-s3",
+        "clientSecret": "seaweedfs-s3-secret",
+        "jwksUri": "http://localhost:8080/realms/seaweedfs-test/protocol/openid-connect/certs",
+        "userInfoUri": "http://localhost:8080/realms/seaweedfs-test/protocol/openid-connect/userinfo",
+        "scopes": ["openid", "profile", "email"],
+        "claimsMapping": {
+          "username": "preferred_username",
+          "email": "email",
+          "name": "name"
+        },
+        "roleMapping": {
+          "rules": [
+            {
+              "claim": "roles",
+              "value": "s3-admin",
+              "role": "arn:aws:iam::role/KeycloakAdminRole"
+            },
+            {
+              "claim": "roles", 
+              "value": "s3-read-only",
+              "role": "arn:aws:iam::role/KeycloakReadOnlyRole"
+            },
+            {
+              "claim": "roles",
+              "value": "s3-write-only", 
+              "role": "arn:aws:iam::role/KeycloakWriteOnlyRole"
+            },
+            {
+              "claim": "roles",
+              "value": "s3-read-write",
+              "role": "arn:aws:iam::role/KeycloakReadWriteRole"
+            }
+          ],
+          "defaultRole": "arn:aws:iam::role/KeycloakReadOnlyRole"
+        }
+      }
+    }
+  ],
+  "policy": {
+    "defaultEffect": "Deny"
+  },
+  "roles": [
+    {
+      "roleName": "TestAdminRole",
+      "roleArn": "arn:aws:iam::role/TestAdminRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "test-oidc"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"]
+          }
+        ]
+      },
+      "attachedPolicies": ["S3AdminPolicy"],
+      "description": "Admin role for testing"
+    },
+    {
+      "roleName": "TestReadOnlyRole", 
+      "roleArn": "arn:aws:iam::role/TestReadOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "test-oidc"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"]
+          }
+        ]
+      },
+      "attachedPolicies": ["S3ReadOnlyPolicy"],
+      "description": "Read-only role for testing"
+    },
+    {
+      "roleName": "TestWriteOnlyRole", 
+      "roleArn": "arn:aws:iam::role/TestWriteOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "test-oidc"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"]
+          }
+        ]
+      },
+      "attachedPolicies": ["S3WriteOnlyPolicy"],
+      "description": "Write-only role for testing"
+    },
+    {
+      "roleName": "KeycloakAdminRole",
+      "roleArn": "arn:aws:iam::role/KeycloakAdminRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"]
+          }
+        ]
+      },
+      "attachedPolicies": ["S3AdminPolicy"],
+      "description": "Admin role for Keycloak users"
+    },
+    {
+      "roleName": "KeycloakReadOnlyRole",
+      "roleArn": "arn:aws:iam::role/KeycloakReadOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"]
+          }
+        ]
+      },
+      "attachedPolicies": ["S3ReadOnlyPolicy"],
+      "description": "Read-only role for Keycloak users"
+    },
+    {
+      "roleName": "KeycloakWriteOnlyRole",
+      "roleArn": "arn:aws:iam::role/KeycloakWriteOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"]
+          }
+        ]
+      },
+      "attachedPolicies": ["S3WriteOnlyPolicy"],
+      "description": "Write-only role for Keycloak users"
+    },
+    {
+      "roleName": "KeycloakReadWriteRole",
+      "roleArn": "arn:aws:iam::role/KeycloakReadWriteRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"]
+          }
+        ]
+      },
+      "attachedPolicies": ["S3ReadWritePolicy"],
+      "description": "Read-write role for Keycloak users"
+    }
+  ],
+  "policies": [
+    {
+      "name": "S3AdminPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": ["*"]
+          },
+          {
+            "Effect": "Allow",
+            "Action": ["sts:ValidateSession"],
+            "Resource": ["*"]
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3ReadOnlyPolicy",
+      "document": {
+        "Version": "2012-10-17", 
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:GetObject",
+              "s3:ListBucket"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Allow",
+            "Action": ["sts:ValidateSession"],
+            "Resource": ["*"]
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3WriteOnlyPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:*"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Deny",
+            "Action": [
+              "s3:GetObject",
+              "s3:ListBucket"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Allow",
+            "Action": ["sts:ValidateSession"],
+            "Resource": ["*"]
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3ReadWritePolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:*"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Allow",
+            "Action": ["sts:ValidateSession"],
+            "Resource": ["*"]
+          }
+        ]
+      }
+    }
+  ]
+}

test/s3/iam/iam_config.json

@@ -0,0 +1,347 @@
+{
+  "sts": {
+    "tokenDuration": "1h",
+    "maxSessionLength": "12h",
+    "issuer": "seaweedfs-sts",
+    "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc="
+  },
+  "providers": [
+    {
+      "name": "test-oidc",
+      "type": "mock",
+      "config": {
+        "issuer": "test-oidc-issuer",
+        "clientId": "test-oidc-client"
+      }
+    },
+    {
+      "name": "keycloak",
+      "type": "oidc",
+      "enabled": true,
+      "config": {
+        "issuer": "http://localhost:8080/realms/seaweedfs-test",
+        "clientId": "seaweedfs-s3",
+        "clientSecret": "seaweedfs-s3-secret",
+        "jwksUri": "http://localhost:8080/realms/seaweedfs-test/protocol/openid-connect/certs",
+        "userInfoUri": "http://localhost:8080/realms/seaweedfs-test/protocol/openid-connect/userinfo",
+        "scopes": [
+          "openid",
+          "profile",
+          "email"
+        ],
+        "claimsMapping": {
+          "username": "preferred_username",
+          "email": "email",
+          "name": "name"
+        },
+        "roleMapping": {
+          "rules": [
+            {
+              "claim": "roles",
+              "value": "s3-admin",
+              "role": "arn:aws:iam::role/KeycloakAdminRole"
+            },
+            {
+              "claim": "roles",
+              "value": "s3-read-only",
+              "role": "arn:aws:iam::role/KeycloakReadOnlyRole"
+            },
+            {
+              "claim": "roles",
+              "value": "s3-write-only",
+              "role": "arn:aws:iam::role/KeycloakWriteOnlyRole"
+            },
+            {
+              "claim": "roles",
+              "value": "s3-read-write",
+              "role": "arn:aws:iam::role/KeycloakReadWriteRole"
+            }
+          ],
+          "defaultRole": "arn:aws:iam::role/KeycloakReadOnlyRole"
+        }
+      }
+    }
+  ],
+  "policy": {
+    "defaultEffect": "Deny"
+  },
+  "roles": [
+    {
+      "roleName": "TestAdminRole",
+      "roleArn": "arn:aws:iam::role/TestAdminRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "test-oidc"
+            },
+            "Action": [
+              "sts:AssumeRoleWithWebIdentity"
+            ]
+          }
+        ]
+      },
+      "attachedPolicies": [
+        "S3AdminPolicy"
+      ],
+      "description": "Admin role for testing"
+    },
+    {
+      "roleName": "TestReadOnlyRole",
+      "roleArn": "arn:aws:iam::role/TestReadOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "test-oidc"
+            },
+            "Action": [
+              "sts:AssumeRoleWithWebIdentity"
+            ]
+          }
+        ]
+      },
+      "attachedPolicies": [
+        "S3ReadOnlyPolicy"
+      ],
+      "description": "Read-only role for testing"
+    },
+    {
+      "roleName": "TestWriteOnlyRole",
+      "roleArn": "arn:aws:iam::role/TestWriteOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "test-oidc"
+            },
+            "Action": [
+              "sts:AssumeRoleWithWebIdentity"
+            ]
+          }
+        ]
+      },
+      "attachedPolicies": [
+        "S3WriteOnlyPolicy"
+      ],
+      "description": "Write-only role for testing"
+    },
+    {
+      "roleName": "KeycloakAdminRole",
+      "roleArn": "arn:aws:iam::role/KeycloakAdminRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": [
+              "sts:AssumeRoleWithWebIdentity"
+            ]
+          }
+        ]
+      },
+      "attachedPolicies": [
+        "S3AdminPolicy"
+      ],
+      "description": "Admin role for Keycloak users"
+    },
+    {
+      "roleName": "KeycloakReadOnlyRole",
+      "roleArn": "arn:aws:iam::role/KeycloakReadOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": [
+              "sts:AssumeRoleWithWebIdentity"
+            ]
+          }
+        ]
+      },
+      "attachedPolicies": [
+        "S3ReadOnlyPolicy"
+      ],
+      "description": "Read-only role for Keycloak users"
+    },
+    {
+      "roleName": "KeycloakWriteOnlyRole",
+      "roleArn": "arn:aws:iam::role/KeycloakWriteOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": [
+              "sts:AssumeRoleWithWebIdentity"
+            ]
+          }
+        ]
+      },
+      "attachedPolicies": [
+        "S3WriteOnlyPolicy"
+      ],
+      "description": "Write-only role for Keycloak users"
+    },
+    {
+      "roleName": "KeycloakReadWriteRole",
+      "roleArn": "arn:aws:iam::role/KeycloakReadWriteRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": [
+              "sts:AssumeRoleWithWebIdentity"
+            ]
+          }
+        ]
+      },
+      "attachedPolicies": [
+        "S3ReadWritePolicy"
+      ],
+      "description": "Read-write role for Keycloak users"
+    }
+  ],
+  "policies": [
+    {
+      "name": "S3AdminPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:*",
+              "iam:*"
+            ],
+            "Resource": [
+              "*",
+              "arn:aws:iam:::*"
+            ]
+          },
+          {
+            "Effect": "Allow",
+            "Action": [
+              "sts:ValidateSession"
+            ],
+            "Resource": [
+              "*"
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3ReadOnlyPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:GetObject",
+              "s3:ListBucket"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Allow",
+            "Action": [
+              "sts:ValidateSession"
+            ],
+            "Resource": [
+              "*"
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3WriteOnlyPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:*"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Deny",
+            "Action": [
+              "s3:GetObject",
+              "s3:ListBucket"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Allow",
+            "Action": [
+              "sts:ValidateSession"
+            ],
+            "Resource": [
+              "*"
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3ReadWritePolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:*"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Allow",
+            "Action": [
+              "sts:ValidateSession"
+            ],
+            "Resource": [
+              "*"
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

test/s3/iam/iam_config.local.json

@@ -0,0 +1,347 @@
+{
+  "sts": {
+    "tokenDuration": "1h",
+    "maxSessionLength": "12h",
+    "issuer": "seaweedfs-sts",
+    "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc="
+  },
+  "providers": [
+    {
+      "name": "test-oidc",
+      "type": "mock",
+      "config": {
+        "issuer": "test-oidc-issuer",
+        "clientId": "test-oidc-client"
+      }
+    },
+    {
+      "name": "keycloak",
+      "type": "oidc",
+      "enabled": true,
+      "config": {
+        "issuer": "http://localhost:8080/realms/seaweedfs-test",
+        "clientId": "seaweedfs-s3",
+        "clientSecret": "seaweedfs-s3-secret",
+        "jwksUri": "http://localhost:8080/realms/seaweedfs-test/protocol/openid-connect/certs",
+        "userInfoUri": "http://localhost:8080/realms/seaweedfs-test/protocol/openid-connect/userinfo",
+        "scopes": [
+          "openid",
+          "profile",
+          "email"
+        ],
+        "claimsMapping": {
+          "username": "preferred_username",
+          "email": "email",
+          "name": "name"
+        },
+        "roleMapping": {
+          "rules": [
+            {
+              "claim": "roles",
+              "value": "s3-admin",
+              "role": "arn:aws:iam::role/KeycloakAdminRole"
+            },
+            {
+              "claim": "roles",
+              "value": "s3-read-only",
+              "role": "arn:aws:iam::role/KeycloakReadOnlyRole"
+            },
+            {
+              "claim": "roles",
+              "value": "s3-write-only",
+              "role": "arn:aws:iam::role/KeycloakWriteOnlyRole"
+            },
+            {
+              "claim": "roles",
+              "value": "s3-read-write",
+              "role": "arn:aws:iam::role/KeycloakReadWriteRole"
+            }
+          ],
+          "defaultRole": "arn:aws:iam::role/KeycloakReadOnlyRole"
+        }
+      }
+    }
+  ],
+  "policy": {
+    "defaultEffect": "Deny"
+  },
+  "roles": [
+    {
+      "roleName": "TestAdminRole",
+      "roleArn": "arn:aws:iam::role/TestAdminRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "test-oidc"
+            },
+            "Action": [
+              "sts:AssumeRoleWithWebIdentity"
+            ]
+          }
+        ]
+      },
+      "attachedPolicies": [
+        "S3AdminPolicy"
+      ],
+      "description": "Admin role for testing"
+    },
+    {
+      "roleName": "TestReadOnlyRole",
+      "roleArn": "arn:aws:iam::role/TestReadOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "test-oidc"
+            },
+            "Action": [
+              "sts:AssumeRoleWithWebIdentity"
+            ]
+          }
+        ]
+      },
+      "attachedPolicies": [
+        "S3ReadOnlyPolicy"
+      ],
+      "description": "Read-only role for testing"
+    },
+    {
+      "roleName": "TestWriteOnlyRole",
+      "roleArn": "arn:aws:iam::role/TestWriteOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "test-oidc"
+            },
+            "Action": [
+              "sts:AssumeRoleWithWebIdentity"
+            ]
+          }
+        ]
+      },
+      "attachedPolicies": [
+        "S3WriteOnlyPolicy"
+      ],
+      "description": "Write-only role for testing"
+    },
+    {
+      "roleName": "KeycloakAdminRole",
+      "roleArn": "arn:aws:iam::role/KeycloakAdminRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": [
+              "sts:AssumeRoleWithWebIdentity"
+            ]
+          }
+        ]
+      },
+      "attachedPolicies": [
+        "S3AdminPolicy"
+      ],
+      "description": "Admin role for Keycloak users"
+    },
+    {
+      "roleName": "KeycloakReadOnlyRole",
+      "roleArn": "arn:aws:iam::role/KeycloakReadOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": [
+              "sts:AssumeRoleWithWebIdentity"
+            ]
+          }
+        ]
+      },
+      "attachedPolicies": [
+        "S3ReadOnlyPolicy"
+      ],
+      "description": "Read-only role for Keycloak users"
+    },
+    {
+      "roleName": "KeycloakWriteOnlyRole",
+      "roleArn": "arn:aws:iam::role/KeycloakWriteOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": [
+              "sts:AssumeRoleWithWebIdentity"
+            ]
+          }
+        ]
+      },
+      "attachedPolicies": [
+        "S3WriteOnlyPolicy"
+      ],
+      "description": "Write-only role for Keycloak users"
+    },
+    {
+      "roleName": "KeycloakReadWriteRole",
+      "roleArn": "arn:aws:iam::role/KeycloakReadWriteRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": [
+              "sts:AssumeRoleWithWebIdentity"
+            ]
+          }
+        ]
+      },
+      "attachedPolicies": [
+        "S3ReadWritePolicy"
+      ],
+      "description": "Read-write role for Keycloak users"
+    }
+  ],
+  "policies": [
+    {
+      "name": "S3AdminPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:*",
+              "iam:*"
+            ],
+            "Resource": [
+              "*",
+              "arn:aws:iam:::*"
+            ]
+          },
+          {
+            "Effect": "Allow",
+            "Action": [
+              "sts:ValidateSession"
+            ],
+            "Resource": [
+              "*"
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3ReadOnlyPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:GetObject",
+              "s3:ListBucket"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Allow",
+            "Action": [
+              "sts:ValidateSession"
+            ],
+            "Resource": [
+              "*"
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3WriteOnlyPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:*"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Deny",
+            "Action": [
+              "s3:GetObject",
+              "s3:ListBucket"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Allow",
+            "Action": [
+              "sts:ValidateSession"
+            ],
+            "Resource": [
+              "*"
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3ReadWritePolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:*"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Allow",
+            "Action": [
+              "sts:ValidateSession"
+            ],
+            "Resource": [
+              "*"
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

test/s3/iam/iam_config_distributed.json

@@ -0,0 +1,173 @@
+{
+  "sts": {
+    "tokenDuration": "1h",
+    "maxSessionLength": "12h",
+    "issuer": "seaweedfs-sts",
+    "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc=",
+    "providers": [
+      {
+        "name": "keycloak-oidc",
+        "type": "oidc",
+        "enabled": true,
+        "config": {
+          "issuer": "http://keycloak:8080/realms/seaweedfs-test",
+          "clientId": "seaweedfs-s3",
+          "clientSecret": "seaweedfs-s3-secret",
+          "jwksUri": "http://keycloak:8080/realms/seaweedfs-test/protocol/openid-connect/certs",
+          "scopes": ["openid", "profile", "email", "roles"],
+          "claimsMapping": {
+            "usernameClaim": "preferred_username",
+            "groupsClaim": "roles"
+          }
+        }
+      },
+      {
+        "name": "mock-provider",
+        "type": "mock",
+        "enabled": false,
+        "config": {
+          "issuer": "http://localhost:9999",
+          "jwksEndpoint": "http://localhost:9999/jwks"
+        }
+      }
+    ]
+  },
+  "policy": {
+    "defaultEffect": "Deny"
+  },
+  "roleStore": {},
+
+  "roles": [
+    {
+      "roleName": "S3AdminRole",
+      "roleArn": "arn:aws:iam::role/S3AdminRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak-oidc"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"],
+            "Condition": {
+              "StringEquals": {
+                "roles": "s3-admin"
+              }
+            }
+          }
+        ]
+      },
+      "attachedPolicies": ["S3AdminPolicy"],
+      "description": "Full S3 administrator access role"
+    },
+    {
+      "roleName": "S3ReadOnlyRole",
+      "roleArn": "arn:aws:iam::role/S3ReadOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak-oidc"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"],
+            "Condition": {
+              "StringEquals": {
+                "roles": "s3-read-only"
+              }
+            }
+          }
+        ]
+      },
+      "attachedPolicies": ["S3ReadOnlyPolicy"],
+      "description": "Read-only access to S3 resources"
+    },
+    {
+      "roleName": "S3ReadWriteRole",
+      "roleArn": "arn:aws:iam::role/S3ReadWriteRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak-oidc"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"],
+            "Condition": {
+              "StringEquals": {
+                "roles": "s3-read-write"
+              }
+            }
+          }
+        ]
+      },
+      "attachedPolicies": ["S3ReadWritePolicy"],
+      "description": "Read-write access to S3 resources"
+    }
+  ],
+  "policies": [
+    {
+      "name": "S3AdminPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": "s3:*",
+            "Resource": "*"
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3ReadOnlyPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:GetObject",
+              "s3:GetObjectAcl",
+              "s3:GetObjectVersion",
+              "s3:ListBucket",
+              "s3:ListBucketVersions"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3ReadWritePolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:GetObject",
+              "s3:GetObjectAcl",
+              "s3:GetObjectVersion",
+              "s3:PutObject",
+              "s3:PutObjectAcl",
+              "s3:DeleteObject",
+              "s3:ListBucket",
+              "s3:ListBucketVersions"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

test/s3/iam/iam_config_docker.json

@@ -0,0 +1,158 @@
+{
+    "sts": {
+    "tokenDuration": "1h",
+    "maxSessionLength": "12h",
+    "issuer": "seaweedfs-sts",
+    "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc=",
+    "providers": [
+      {
+        "name": "keycloak-oidc",
+        "type": "oidc",
+        "enabled": true,
+        "config": {
+          "issuer": "http://keycloak:8080/realms/seaweedfs-test",
+          "clientId": "seaweedfs-s3",
+          "clientSecret": "seaweedfs-s3-secret",
+          "jwksUri": "http://keycloak:8080/realms/seaweedfs-test/protocol/openid-connect/certs",
+          "scopes": ["openid", "profile", "email", "roles"]
+        }
+      }
+    ]
+  },
+  "policy": {
+    "defaultEffect": "Deny"
+  },
+  "roles": [
+    {
+      "roleName": "S3AdminRole",
+      "roleArn": "arn:aws:iam::role/S3AdminRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak-oidc"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"],
+            "Condition": {
+              "StringEquals": {
+                "oidc:roles": "s3-admin"
+              }
+            }
+          }
+        ]
+      },
+      "attachedPolicies": ["S3AdminPolicy"],
+      "description": "Full S3 administrator access role"
+    },
+    {
+      "roleName": "S3ReadOnlyRole",
+      "roleArn": "arn:aws:iam::role/S3ReadOnlyRole", 
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak-oidc"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"],
+            "Condition": {
+              "StringEquals": {
+                "oidc:roles": "s3-read-only"
+              }
+            }
+          }
+        ]
+      },
+      "attachedPolicies": ["S3ReadOnlyPolicy"],
+      "description": "Read-only access to S3 resources"
+    },
+    {
+      "roleName": "S3ReadWriteRole",
+      "roleArn": "arn:aws:iam::role/S3ReadWriteRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak-oidc"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"],
+            "Condition": {
+              "StringEquals": {
+                "oidc:roles": "s3-read-write"
+              }
+            }
+          }
+        ]
+      },
+      "attachedPolicies": ["S3ReadWritePolicy"],
+      "description": "Read-write access to S3 resources"
+    }
+  ],
+  "policies": [
+    {
+      "name": "S3AdminPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": "s3:*",
+            "Resource": "*"
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3ReadOnlyPolicy", 
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:GetObject",
+              "s3:GetObjectAcl",
+              "s3:GetObjectVersion",
+              "s3:ListBucket",
+              "s3:ListBucketVersions"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3ReadWritePolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:GetObject",
+              "s3:GetObjectAcl", 
+              "s3:GetObjectVersion",
+              "s3:PutObject",
+              "s3:PutObjectAcl",
+              "s3:DeleteObject",
+              "s3:ListBucket",
+              "s3:ListBucketVersions"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          }
+        ]
+      }
+    }
+  ]
+}

test/s3/iam/run_all_tests.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+
+# Master Test Runner - Enables and runs all previously skipped tests
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+echo -e "${BLUE}🎯 SeaweedFS S3 IAM Complete Test Suite${NC}"
+echo -e "${BLUE}=====================================${NC}"
+
+# Set environment variables to enable all tests
+export ENABLE_DISTRIBUTED_TESTS=true
+export ENABLE_PERFORMANCE_TESTS=true
+export ENABLE_STRESS_TESTS=true
+export KEYCLOAK_URL="http://localhost:8080"
+export S3_ENDPOINT="http://localhost:8333"
+export TEST_TIMEOUT=60m
+export CGO_ENABLED=0
+
+# Function to run test category
+run_test_category() {
+    local category="$1"
+    local test_pattern="$2"
+    local description="$3"
+    
+    echo -e "${YELLOW}🧪 Running $description...${NC}"
+    
+    if go test -v -timeout=$TEST_TIMEOUT -run "$test_pattern" ./...; then
+        echo -e "${GREEN}[OK] $description completed successfully${NC}"
+        return 0
+    else
+        echo -e "${RED}[FAIL] $description failed${NC}"
+        return 1
+    fi
+}
+
+# Track results
+TOTAL_CATEGORIES=0
+PASSED_CATEGORIES=0
+
+# 1. Standard IAM Integration Tests
+echo -e "\n${BLUE}1. Standard IAM Integration Tests${NC}"
+TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1))
+if run_test_category "standard" "TestS3IAM(?!.*Distributed|.*Performance)" "Standard IAM Integration Tests"; then
+    PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1))
+fi
+
+# 2. Keycloak Integration Tests (if Keycloak is available)
+echo -e "\n${BLUE}2. Keycloak Integration Tests${NC}"
+TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1))
+if curl -s "http://localhost:8080/health/ready" > /dev/null 2>&1; then
+    if run_test_category "keycloak" "TestKeycloak" "Keycloak Integration Tests"; then
+        PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1))
+    fi
+else
+    echo -e "${YELLOW}⚠️ Keycloak not available, skipping Keycloak tests${NC}"
+    echo -e "${YELLOW}💡 Run './setup_all_tests.sh' to start Keycloak${NC}"
+fi
+
+# 3. Distributed Tests
+echo -e "\n${BLUE}3. Distributed IAM Tests${NC}"
+TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1))
+if run_test_category "distributed" "TestS3IAMDistributedTests" "Distributed IAM Tests"; then
+    PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1))
+fi
+
+# 4. Performance Tests
+echo -e "\n${BLUE}4. Performance Tests${NC}"
+TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1))
+if run_test_category "performance" "TestS3IAMPerformanceTests" "Performance Tests"; then
+    PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1))
+fi
+
+# 5. Benchmarks
+echo -e "\n${BLUE}5. Benchmark Tests${NC}"
+TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1))
+if go test -bench=. -benchmem -timeout=$TEST_TIMEOUT ./...; then
+    echo -e "${GREEN}[OK] Benchmark tests completed successfully${NC}"
+    PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1))
+else
+    echo -e "${RED}[FAIL] Benchmark tests failed${NC}"
+fi
+
+# 6. Versioning Stress Tests
+echo -e "\n${BLUE}6. S3 Versioning Stress Tests${NC}"
+TOTAL_CATEGORIES=$((TOTAL_CATEGORIES + 1))
+if [ -f "../versioning/enable_stress_tests.sh" ]; then
+    if (cd ../versioning && ./enable_stress_tests.sh); then
+        echo -e "${GREEN}[OK] Versioning stress tests completed successfully${NC}"
+        PASSED_CATEGORIES=$((PASSED_CATEGORIES + 1))
+    else
+        echo -e "${RED}[FAIL] Versioning stress tests failed${NC}"
+    fi
+else
+    echo -e "${YELLOW}⚠️ Versioning stress tests not available${NC}"
+fi
+
+# Summary
+echo -e "\n${BLUE}📊 Test Summary${NC}"
+echo -e "${BLUE}===============${NC}"
+echo -e "Total test categories: $TOTAL_CATEGORIES"
+echo -e "Passed: ${GREEN}$PASSED_CATEGORIES${NC}"
+echo -e "Failed: ${RED}$((TOTAL_CATEGORIES - PASSED_CATEGORIES))${NC}"
+
+if [ $PASSED_CATEGORIES -eq $TOTAL_CATEGORIES ]; then
+    echo -e "\n${GREEN}🎉 All test categories passed!${NC}"
+    exit 0
+else
+    echo -e "\n${RED}[FAIL] Some test categories failed${NC}"
+    exit 1
+fi

test/s3/iam/run_custom_test.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+set -e
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+
+# Build weed binary
+echo "Building weed binary..."
+cd "$PROJECT_ROOT/weed" && go install
+
+# Kill existing server
+if lsof -Pi :8333 -sTCP:LISTEN -t >/dev/null 2>&1 ; then
+    kill $(lsof -t -i:8333) 2>/dev/null || true
+fi
+
+# Start server using weed mini for simpler all-in-one deployment
+weed mini \
+    -s3 \
+    -s3.port=8333 \
+    -s3.config="$SCRIPT_DIR/empty_s3_config.json" \
+    -s3.iam.config="$SCRIPT_DIR/test_iam_config.json" \
+    -s3.allowDeleteBucketNotEmpty=true \
+    > /tmp/weed_test_server_custom.log 2>&1 &
+SERVER_PID=$!
+
+# Wait for server
+MAX_WAIT=30
+COUNTER=0
+while ! curl -s http://localhost:8333/status > /dev/null 2>&1; do
+    sleep 1
+    COUNTER=$((COUNTER + 1))
+    if [ $COUNTER -ge $MAX_WAIT ]; then
+        echo "Server failed to start"
+        cat /tmp/weed_test_server_custom.log
+        kill $SERVER_PID
+        exit 1
+    fi
+done
+
+trap "kill $SERVER_PID" EXIT
+
+cd "$SCRIPT_DIR"
+if [ $# -eq 0 ]; then
+    go test -v -run TestS3IAMMultipartUploadPolicyEnforcement .
+else
+    go test -v "$@" .
+fi

test/s3/iam/run_performance_tests.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Performance Test Runner for SeaweedFS S3 IAM
+
+set -e
+
+# Colors
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+echo -e "${YELLOW}🏁 Running S3 IAM Performance Tests${NC}"
+
+# Enable performance tests
+export ENABLE_PERFORMANCE_TESTS=true
+export TEST_TIMEOUT=60m
+
+# Run benchmarks
+echo -e "${YELLOW}📊 Running benchmarks...${NC}"
+go test -bench=. -benchmem -timeout=$TEST_TIMEOUT ./...
+
+# Run performance tests
+echo -e "${YELLOW}🧪 Running performance test suite...${NC}"
+go test -v -timeout=$TEST_TIMEOUT -run "TestS3IAMPerformanceTests" ./...
+
+echo -e "${GREEN}[OK] Performance tests completed${NC}"

test/s3/iam/run_stress_tests.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Stress Test Runner for SeaweedFS S3 IAM
+
+set -e
+
+# Colors
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+NC='\033[0m'
+
+echo -e "${YELLOW}💪 Running S3 IAM Stress Tests${NC}"
+
+# Enable stress tests
+export ENABLE_STRESS_TESTS=true
+export TEST_TIMEOUT=60m
+
+# Run stress tests multiple times
+STRESS_ITERATIONS=5
+
+echo -e "${YELLOW}🔄 Running stress tests with $STRESS_ITERATIONS iterations...${NC}"
+
+for i in $(seq 1 $STRESS_ITERATIONS); do
+    echo -e "${YELLOW}📊 Iteration $i/$STRESS_ITERATIONS${NC}"
+    
+    if ! go test -v -timeout=$TEST_TIMEOUT -run "TestS3IAMDistributedTests.*concurrent" ./... -count=1; then
+        echo -e "${RED}❌ Stress test failed on iteration $i${NC}"
+        exit 1
+    fi
+    
+    # Brief pause between iterations
+    sleep 2
+done
+
+echo -e "${GREEN}[OK] All stress test iterations completed successfully${NC}"

test/s3/iam/run_tests.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+# Test runner for S3 policy variables integration tests
+# This script starts a SeaweedFS server with the required IAM configuration
+# and runs the integration tests.
+
+set -e
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+echo -e "${GREEN}=== S3 Policy Variables Integration Test Runner ===${NC}"
+
+# Get the directory of this script
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
+
+# Always build to ensure latest changes are tested
+echo -e "${YELLOW}Building weed binary...${NC}"
+cd "$PROJECT_ROOT/weed" && go install
+if ! command -v weed &> /dev/null; then
+    echo -e "${RED}Failed to build weed binary${NC}"
+    exit 1
+fi
+
+# Kill any existing weed server on port 8333
+echo "Checking for existing weed server..."
+if lsof -Pi :8333 -sTCP:LISTEN -t >/dev/null 2>&1 ; then
+    echo -e "${YELLOW}Killing existing weed server on port 8333...${NC}"
+    kill $(lsof -t -i:8333) 2>/dev/null || true
+    sleep 2
+fi
+
+# Start weed server with IAM configuration
+echo -e "${GREEN}Starting weed server with IAM configuration...${NC}"
+weed server \
+    -s3 \
+    -s3.port=8333 \
+    -s3.iam.config="$SCRIPT_DIR/test_iam_config.json" \
+    -filer \
+    -volume.max=0 \
+    -master.volumeSizeLimitMB=100 \
+    -s3.allowDeleteBucketNotEmpty=true \
+    -s3.iam.readOnly=false \
+    > /tmp/weed_test_server.log 2>&1 &
+
+SERVER_PID=$!
+echo "Server started with PID: $SERVER_PID"
+
+# Wait for server to be ready
+echo "Waiting for server to be ready..."
+MAX_WAIT=30
+COUNTER=0
+while ! curl -s http://localhost:8333/status > /dev/null 2>&1; do
+    sleep 1
+    COUNTER=$((COUNTER + 1))
+    if [ $COUNTER -ge $MAX_WAIT ]; then
+        echo -e "${RED}Server failed to start within ${MAX_WAIT} seconds${NC}"
+        echo "Server log:"
+        cat /tmp/weed_test_server.log
+        kill $SERVER_PID 2>/dev/null || true
+        exit 1
+    fi
+done
+
+echo -e "${GREEN}Server is ready!${NC}"
+
+# Run the tests
+echo -e "${GREEN}Running integration tests...${NC}"
+cd "$SCRIPT_DIR"
+
+# Trap to ensure server is killed on exit
+trap "echo -e '${YELLOW}Shutting down server...${NC}'; kill $SERVER_PID 2>/dev/null || true" EXIT
+
+# Run the tests
+go test -v -run TestS3PolicyVariables .
+
+TEST_RESULT=$?
+
+if [ $TEST_RESULT -eq 0 ]; then
+    echo -e "${GREEN}=== All tests passed! ===${NC}"
+else
+    echo -e "${RED}=== Tests failed ===${NC}"
+    echo "Server log (last 50 lines):"
+    tail -50 /tmp/weed_test_server.log
+fi
+
+exit $TEST_RESULT

test/s3/iam/s3_iam_admin_test.go

@@ -0,0 +1,384 @@
+package iam
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestIAMUserManagement tests user management operations
+func TestIAMUserManagement(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	// Create IAM client with admin privileges
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	t.Run("create_and_get_user", func(t *testing.T) {
+		userName := "test-user-mgm"
+
+		// Create user
+		createResp, err := iamClient.CreateUser(&iam.CreateUserInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, userName, *createResp.User.UserName)
+
+		// Get user
+		getResp, err := iamClient.GetUser(&iam.GetUserInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, userName, *getResp.User.UserName)
+
+		// List users to verify existence
+		listResp, err := iamClient.ListUsers(&iam.ListUsersInput{})
+		require.NoError(t, err)
+		found := false
+		for _, user := range listResp.Users {
+			if *user.UserName == userName {
+				found = true
+				break
+			}
+		}
+		assert.True(t, found, "Created user should be listed")
+
+		// Clean up
+		_, err = iamClient.DeleteUser(&iam.DeleteUserInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+	})
+
+	t.Run("update_user", func(t *testing.T) {
+		userName := "user-to-update"
+		newUserName := "user-updated"
+
+		// Create user
+		_, err := iamClient.CreateUser(&iam.CreateUserInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		defer func() {
+			// Try to delete both just in case
+			iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
+			iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(newUserName)})
+		}()
+
+		// Update user name
+		_, err = iamClient.UpdateUser(&iam.UpdateUserInput{
+			UserName:    aws.String(userName),
+			NewUserName: aws.String(newUserName),
+		})
+		require.NoError(t, err)
+
+		// Verify update (GetUser with NEW name should work)
+		getResp, err := iamClient.GetUser(&iam.GetUserInput{
+			UserName: aws.String(newUserName),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, newUserName, *getResp.User.UserName)
+
+		// GetUser with OLD name should fail
+		_, err = iamClient.GetUser(&iam.GetUserInput{
+			UserName: aws.String(userName),
+		})
+		require.Error(t, err)
+	})
+}
+
+// TestIAMAccessKeyManagement tests access key operations
+func TestIAMAccessKeyManagement(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	userName := "test-user-keys"
+	_, err = iamClient.CreateUser(&iam.CreateUserInput{
+		UserName: aws.String(userName),
+	})
+	require.NoError(t, err)
+	defer iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
+
+	t.Run("create_list_delete_access_key", func(t *testing.T) {
+		// Create access key
+		createResp, err := iamClient.CreateAccessKey(&iam.CreateAccessKeyInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		assert.NotEmpty(t, *createResp.AccessKey.AccessKeyId)
+		assert.NotEmpty(t, *createResp.AccessKey.SecretAccessKey)
+		assert.Equal(t, "Active", *createResp.AccessKey.Status)
+
+		// List access keys
+		listResp, err := iamClient.ListAccessKeys(&iam.ListAccessKeysInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, 1, len(listResp.AccessKeyMetadata))
+		assert.Equal(t, *createResp.AccessKey.AccessKeyId, *listResp.AccessKeyMetadata[0].AccessKeyId)
+
+		// Delete access key
+		_, err = iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{
+			UserName:    aws.String(userName),
+			AccessKeyId: createResp.AccessKey.AccessKeyId,
+		})
+		require.NoError(t, err)
+
+		// Verify deletion
+		listResp, err = iamClient.ListAccessKeys(&iam.ListAccessKeysInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, 0, len(listResp.AccessKeyMetadata))
+	})
+
+	t.Run("update_access_key_status", func(t *testing.T) {
+		// Create access key
+		createResp, err := iamClient.CreateAccessKey(&iam.CreateAccessKeyInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		defer iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{
+			UserName:    aws.String(userName),
+			AccessKeyId: createResp.AccessKey.AccessKeyId,
+		})
+
+		// Update to Inactive
+		_, err = iamClient.UpdateAccessKey(&iam.UpdateAccessKeyInput{
+			UserName:    aws.String(userName),
+			AccessKeyId: createResp.AccessKey.AccessKeyId,
+			Status:      aws.String("Inactive"),
+		})
+		require.NoError(t, err)
+
+		// Verify update in ListAccessKeys
+		listResp, err := iamClient.ListAccessKeys(&iam.ListAccessKeysInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		found := false
+		for _, key := range listResp.AccessKeyMetadata {
+			if *key.AccessKeyId == *createResp.AccessKey.AccessKeyId {
+				assert.Equal(t, "Inactive", *key.Status)
+				found = true
+				break
+			}
+		}
+		assert.True(t, found)
+	})
+}
+
+// TestIAMPolicyManagement tests policy operations
+func TestIAMPolicyManagement(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	t.Run("create_managed_policy", func(t *testing.T) {
+		policyName := "test-managed-policy"
+		policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListBucket","Resource":"*"}]}`
+
+		createResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(policyDoc),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, policyName, *createResp.Policy.PolicyName)
+		assert.NotEmpty(t, *createResp.Policy.Arn)
+
+		t.Cleanup(func() {
+			_, _ = iamClient.DeletePolicy(&iam.DeletePolicyInput{
+				PolicyArn: createResp.Policy.Arn,
+			})
+		})
+	})
+
+	t.Run("managed_policy_crud_lifecycle", func(t *testing.T) {
+		policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::*"}]}`
+
+		policyNames := []string{"test-managed-policy-lifecycle-a", "test-managed-policy-lifecycle-b"}
+		policyArns := make([]*string, 0, len(policyNames))
+		for _, policyName := range policyNames {
+			createResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
+				PolicyName:     aws.String(policyName),
+				PolicyDocument: aws.String(policyDoc),
+			})
+			require.NoError(t, err)
+			policyArns = append(policyArns, createResp.Policy.Arn)
+		}
+
+		t.Cleanup(func() {
+			for _, policyArn := range policyArns {
+				_, _ = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policyArn})
+			}
+		})
+
+		listResp, err := iamClient.ListPolicies(&iam.ListPoliciesInput{})
+		require.NoError(t, err)
+
+		foundByName := map[string]bool{}
+		for _, policy := range listResp.Policies {
+			if policy.PolicyName != nil {
+				foundByName[*policy.PolicyName] = true
+			}
+		}
+		for _, policyName := range policyNames {
+			assert.True(t, foundByName[policyName], "policy %s should be listed", policyName)
+		}
+
+		getResp, err := iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: policyArns[0]})
+		require.NoError(t, err)
+		require.NotNil(t, getResp.Policy)
+		assert.Equal(t, policyNames[0], aws.StringValue(getResp.Policy.PolicyName))
+		assert.Equal(t, aws.StringValue(policyArns[0]), aws.StringValue(getResp.Policy.Arn))
+
+		_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policyArns[0]})
+		require.NoError(t, err)
+
+		_, err = iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: policyArns[0]})
+		require.Error(t, err)
+		awsErr, ok := err.(awserr.Error)
+		require.True(t, ok)
+		assert.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+
+		listAfterDeleteResp, err := iamClient.ListPolicies(&iam.ListPoliciesInput{})
+		require.NoError(t, err)
+		deletedPolicyFound := false
+		remainingPolicyFound := false
+		for _, policy := range listAfterDeleteResp.Policies {
+			if policy.PolicyName == nil {
+				continue
+			}
+			if *policy.PolicyName == policyNames[0] {
+				deletedPolicyFound = true
+			}
+			if *policy.PolicyName == policyNames[1] {
+				remainingPolicyFound = true
+			}
+		}
+		assert.False(t, deletedPolicyFound, "deleted policy should no longer be listed")
+		assert.True(t, remainingPolicyFound, "remaining policy should still be listed")
+
+		policyArns[0] = nil
+	})
+
+	t.Run("managed_policy_versions", func(t *testing.T) {
+		policyName := "test-managed-policy-version"
+		policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListBucket","Resource":"*"}]}`
+
+		createResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(policyDoc),
+		})
+		require.NoError(t, err)
+
+		t.Cleanup(func() {
+			_, _ = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: createResp.Policy.Arn})
+		})
+
+		listVersionsResp, err := iamClient.ListPolicyVersions(&iam.ListPolicyVersionsInput{
+			PolicyArn: createResp.Policy.Arn,
+		})
+		require.NoError(t, err)
+		require.NotEmpty(t, listVersionsResp.Versions)
+		assert.Equal(t, "v1", aws.StringValue(listVersionsResp.Versions[0].VersionId))
+		assert.Equal(t, true, aws.BoolValue(listVersionsResp.Versions[0].IsDefaultVersion))
+
+		getVersionResp, err := iamClient.GetPolicyVersion(&iam.GetPolicyVersionInput{
+			PolicyArn:  createResp.Policy.Arn,
+			VersionId:  aws.String("v1"),
+		})
+		require.NoError(t, err)
+		require.NotNil(t, getVersionResp.PolicyVersion)
+		assert.Equal(t, "v1", aws.StringValue(getVersionResp.PolicyVersion.VersionId))
+		assert.Contains(t, aws.StringValue(getVersionResp.PolicyVersion.Document), "s3:ListBucket")
+
+		_, err = iamClient.GetPolicyVersion(&iam.GetPolicyVersionInput{
+			PolicyArn: createResp.Policy.Arn,
+			VersionId: aws.String("v2"),
+		})
+		require.Error(t, err)
+		awsErr, ok := err.(awserr.Error)
+		require.True(t, ok)
+		assert.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+
+	t.Run("user_inline_policy", func(t *testing.T) {
+		userName := "test-user-policy"
+		_, err := iamClient.CreateUser(&iam.CreateUserInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		defer iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
+
+		// List user policies before any inline policy is attached
+		listResp, err := iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		assert.Empty(t, listResp.PolicyNames, "New user should have no inline policies")
+
+		policyName := "test-inline-policy"
+		policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::*"}]}`
+
+		// Put user policy
+		_, err = iamClient.PutUserPolicy(&iam.PutUserPolicyInput{
+			UserName:       aws.String(userName),
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(policyDoc),
+		})
+		require.NoError(t, err)
+
+		// List user policies after attaching inline policy
+		listResp, err = iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		assert.NotEmpty(t, listResp.PolicyNames, "User should have inline policies after PutUserPolicy")
+		assert.False(t, *listResp.IsTruncated)
+
+		// Get user policy
+		getResp, err := iamClient.GetUserPolicy(&iam.GetUserPolicyInput{
+			UserName:   aws.String(userName),
+			PolicyName: aws.String(policyName),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, userName, *getResp.UserName)
+		assert.Equal(t, policyName, *getResp.PolicyName)
+		assert.Contains(t, *getResp.PolicyDocument, "s3:Get")
+
+		// Delete user policy
+		_, err = iamClient.DeleteUserPolicy(&iam.DeleteUserPolicyInput{
+			UserName:   aws.String(userName),
+			PolicyName: aws.String(policyName),
+		})
+		require.NoError(t, err)
+
+		// List user policies after deletion
+		listResp, err = iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		assert.Empty(t, listResp.PolicyNames, "User should have no inline policies after DeleteUserPolicy")
+	})
+
+	t.Run("list_user_policies_nonexistent_user", func(t *testing.T) {
+		_, err := iamClient.ListUserPolicies(&iam.ListUserPoliciesInput{
+			UserName: aws.String("nonexistent-user-for-list-policies"),
+		})
+		require.Error(t, err)
+		awsErr, ok := err.(awserr.Error)
+		require.True(t, ok, "Expected AWS error type")
+		assert.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+}

test/s3/iam/s3_iam_distributed_test.go

@@ -0,0 +1,435 @@
+package iam
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestS3IAMDistributedTests tests IAM functionality across multiple S3 gateway instances
+func TestS3IAMDistributedTests(t *testing.T) {
+	// Skip if not in distributed test mode
+	if os.Getenv("ENABLE_DISTRIBUTED_TESTS") != "true" {
+		t.Skip("Distributed tests not enabled. Set ENABLE_DISTRIBUTED_TESTS=true")
+	}
+
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	t.Run("distributed_session_consistency", func(t *testing.T) {
+		// Test that sessions created on one instance are visible on others
+		// This requires filer-based session storage
+
+		// Create S3 clients that would connect to different gateway instances
+		// In a real distributed setup, these would point to different S3 gateway ports
+		client1, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+		require.NoError(t, err)
+
+		client2, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+		require.NoError(t, err)
+
+		// Both clients should be able to perform operations
+		bucketName := "test-distributed-session"
+
+		err = framework.CreateBucket(client1, bucketName)
+		require.NoError(t, err)
+
+		// Client2 should see the bucket created by client1
+		// Retry logic for eventually consistent storage
+		var found bool
+		for i := 0; i < 20; i++ {
+			listResult, err := client2.ListBuckets(&s3.ListBucketsInput{})
+			require.NoError(t, err)
+
+			found = false
+			for _, bucket := range listResult.Buckets {
+				if *bucket.Name == bucketName {
+					found = true
+					break
+				}
+			}
+			if found {
+				break
+			}
+			time.Sleep(250 * time.Millisecond)
+		}
+		assert.True(t, found, "Bucket should be visible across distributed instances")
+
+		// Cleanup
+		_, err = client1.DeleteBucket(&s3.DeleteBucketInput{
+			Bucket: aws.String(bucketName),
+		})
+		require.NoError(t, err)
+	})
+
+	t.Run("distributed_role_consistency", func(t *testing.T) {
+		// Test that role definitions are consistent across instances
+		// This requires filer-based role storage
+
+		// Create clients with different roles
+		adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+		require.NoError(t, err)
+
+		readOnlyClient, err := framework.CreateS3ClientWithJWT("read-user", "TestReadOnlyRole")
+		require.NoError(t, err)
+
+		bucketName := "test-distributed-roles"
+		objectKey := "test-object.txt"
+
+		// Admin should be able to create bucket
+		err = framework.CreateBucket(adminClient, bucketName)
+		require.NoError(t, err)
+
+		// Admin should be able to put object
+		err = framework.PutTestObject(adminClient, bucketName, objectKey, "test content")
+		require.NoError(t, err)
+
+		// Read-only user should be able to get object
+		content, err := framework.GetTestObject(readOnlyClient, bucketName, objectKey)
+		require.NoError(t, err)
+		assert.Equal(t, "test content", content)
+
+		// Read-only user should NOT be able to put object
+		err = framework.PutTestObject(readOnlyClient, bucketName, "forbidden-object.txt", "forbidden content")
+		require.Error(t, err, "Read-only user should not be able to put objects")
+
+		// Cleanup
+		err = framework.DeleteTestObject(adminClient, bucketName, objectKey)
+		require.NoError(t, err)
+		_, err = adminClient.DeleteBucket(&s3.DeleteBucketInput{
+			Bucket: aws.String(bucketName),
+		})
+		require.NoError(t, err)
+	})
+
+	t.Run("distributed_concurrent_operations", func(t *testing.T) {
+		// Test concurrent operations across distributed instances with robust retry mechanisms
+		// This approach implements proper retry logic instead of tolerating errors to catch real concurrency issues
+		const numGoroutines = 3                   // Reduced concurrency for better CI reliability
+		const numOperationsPerGoroutine = 2       // Minimal operations per goroutine
+		const maxRetries = 3                      // Maximum retry attempts for transient failures
+		const retryDelay = 200 * time.Millisecond // Increased delay for better stability
+
+		var wg sync.WaitGroup
+		errors := make(chan error, numGoroutines*numOperationsPerGoroutine)
+
+		// Helper function to determine if an error is retryable
+		isRetryableError := func(err error) bool {
+			if err == nil {
+				return false
+			}
+			errorMsg := err.Error()
+			return strings.Contains(errorMsg, "timeout") ||
+				strings.Contains(errorMsg, "connection reset") ||
+				strings.Contains(errorMsg, "connection refused") ||
+				strings.Contains(errorMsg, "temporary failure") ||
+				strings.Contains(errorMsg, "TooManyRequests") ||
+				strings.Contains(errorMsg, "ServiceUnavailable") ||
+				strings.Contains(errorMsg, "InternalError")
+		}
+
+		// Helper function to execute operations with retry logic
+		executeWithRetry := func(operation func() error, operationName string) error {
+			var lastErr error
+			for attempt := 0; attempt <= maxRetries; attempt++ {
+				if attempt > 0 {
+					time.Sleep(retryDelay * time.Duration(attempt)) // Linear backoff
+				}
+
+				lastErr = operation()
+				if lastErr == nil {
+					return nil // Success
+				}
+
+				if !isRetryableError(lastErr) {
+					// Non-retryable error - fail immediately
+					return fmt.Errorf("%s failed with non-retryable error: %w", operationName, lastErr)
+				}
+
+				// Retryable error - continue to next attempt
+				if attempt < maxRetries {
+					t.Logf("Retrying %s (attempt %d/%d) after error: %v", operationName, attempt+1, maxRetries, lastErr)
+				}
+			}
+
+			// All retries exhausted
+			return fmt.Errorf("%s failed after %d retries, last error: %w", operationName, maxRetries, lastErr)
+		}
+
+		for i := 0; i < numGoroutines; i++ {
+			wg.Add(1)
+			go func(goroutineID int) {
+				defer wg.Done()
+
+				client, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+				if err != nil {
+					errors <- fmt.Errorf("failed to create S3 client for goroutine %d: %w", goroutineID, err)
+					return
+				}
+
+				for j := 0; j < numOperationsPerGoroutine; j++ {
+					bucketName := fmt.Sprintf("test-concurrent-%d-%d", goroutineID, j)
+					objectKey := "test-object.txt"
+					objectContent := fmt.Sprintf("content-%d-%d", goroutineID, j)
+
+					// Execute full operation sequence with individual retries
+					operationFailed := false
+
+					// 1. Create bucket with retry
+					if err := executeWithRetry(func() error {
+						return framework.CreateBucket(client, bucketName)
+					}, fmt.Sprintf("CreateBucket-%s", bucketName)); err != nil {
+						errors <- err
+						operationFailed = true
+					}
+
+					if !operationFailed {
+						// 2. Put object with retry
+						if err := executeWithRetry(func() error {
+							return framework.PutTestObject(client, bucketName, objectKey, objectContent)
+						}, fmt.Sprintf("PutObject-%s/%s", bucketName, objectKey)); err != nil {
+							errors <- err
+							operationFailed = true
+						}
+					}
+
+					if !operationFailed {
+						// 3. Get object with retry
+						if err := executeWithRetry(func() error {
+							_, err := framework.GetTestObject(client, bucketName, objectKey)
+							return err
+						}, fmt.Sprintf("GetObject-%s/%s", bucketName, objectKey)); err != nil {
+							errors <- err
+							operationFailed = true
+						}
+					}
+
+					if !operationFailed {
+						// 4. Delete object with retry
+						if err := executeWithRetry(func() error {
+							return framework.DeleteTestObject(client, bucketName, objectKey)
+						}, fmt.Sprintf("DeleteObject-%s/%s", bucketName, objectKey)); err != nil {
+							errors <- err
+							operationFailed = true
+						}
+					}
+
+					// 5. Always attempt bucket cleanup, even if previous operations failed
+					if err := executeWithRetry(func() error {
+						_, err := client.DeleteBucket(&s3.DeleteBucketInput{
+							Bucket: aws.String(bucketName),
+						})
+						return err
+					}, fmt.Sprintf("DeleteBucket-%s", bucketName)); err != nil {
+						// Only log cleanup failures, don't fail the test
+						t.Logf("Warning: Failed to cleanup bucket %s: %v", bucketName, err)
+					}
+
+					// Increased delay between operation sequences to reduce server load and improve stability
+					time.Sleep(100 * time.Millisecond)
+				}
+			}(i)
+		}
+
+		wg.Wait()
+		close(errors)
+
+		// Collect and analyze errors - with retry logic, we should see very few errors
+		var errorList []error
+		for err := range errors {
+			errorList = append(errorList, err)
+		}
+
+		totalOperations := numGoroutines * numOperationsPerGoroutine
+
+		// Report results
+		if len(errorList) == 0 {
+			t.Logf("All %d concurrent operations completed successfully with retry mechanisms!", totalOperations)
+		} else {
+			t.Logf("Concurrent operations summary:")
+			t.Logf("  Total operations: %d", totalOperations)
+			t.Logf("  Failed operations: %d (%.1f%% error rate)", len(errorList), float64(len(errorList))/float64(totalOperations)*100)
+
+			// Log first few errors for debugging
+			for i, err := range errorList {
+				if i >= 3 { // Limit to first 3 errors
+					t.Logf("  ... and %d more errors", len(errorList)-3)
+					break
+				}
+				t.Logf("  Error %d: %v", i+1, err)
+			}
+		}
+
+		// With proper retry mechanisms, we should expect near-zero failures
+		// Any remaining errors likely indicate real concurrency issues or system problems
+		if len(errorList) > 0 {
+			t.Errorf("%d operation(s) failed even after retry mechanisms (%.1f%% failure rate). This indicates potential system issues or race conditions that need investigation.",
+				len(errorList), float64(len(errorList))/float64(totalOperations)*100)
+		}
+	})
+}
+
+// TestS3IAMPerformanceTests tests IAM performance characteristics
+func TestS3IAMPerformanceTests(t *testing.T) {
+	// Skip if not in performance test mode
+	if os.Getenv("ENABLE_PERFORMANCE_TESTS") != "true" {
+		t.Skip("Performance tests not enabled. Set ENABLE_PERFORMANCE_TESTS=true")
+	}
+
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	t.Run("authentication_performance", func(t *testing.T) {
+		// Test authentication performance
+		const numRequests = 100
+
+		client, err := framework.CreateS3ClientWithJWT("perf-user", "TestAdminRole")
+		require.NoError(t, err)
+
+		bucketName := "test-auth-performance"
+		err = framework.CreateBucket(client, bucketName)
+		require.NoError(t, err)
+		defer func() {
+			_, err := client.DeleteBucket(&s3.DeleteBucketInput{
+				Bucket: aws.String(bucketName),
+			})
+			require.NoError(t, err)
+		}()
+
+		start := time.Now()
+
+		for i := 0; i < numRequests; i++ {
+			_, err := client.ListBuckets(&s3.ListBucketsInput{})
+			require.NoError(t, err)
+		}
+
+		duration := time.Since(start)
+		avgLatency := duration / numRequests
+
+		t.Logf("Authentication performance: %d requests in %v (avg: %v per request)",
+			numRequests, duration, avgLatency)
+
+		// Performance assertion - should be under 100ms per request on average
+		assert.Less(t, avgLatency, 100*time.Millisecond,
+			"Average authentication latency should be under 100ms")
+	})
+
+	t.Run("authorization_performance", func(t *testing.T) {
+		// Test authorization performance with different policy complexities
+		const numRequests = 50
+
+		client, err := framework.CreateS3ClientWithJWT("perf-user", "TestAdminRole")
+		require.NoError(t, err)
+
+		bucketName := "test-authz-performance"
+		err = framework.CreateBucket(client, bucketName)
+		require.NoError(t, err)
+		defer func() {
+			_, err := client.DeleteBucket(&s3.DeleteBucketInput{
+				Bucket: aws.String(bucketName),
+			})
+			require.NoError(t, err)
+		}()
+
+		start := time.Now()
+
+		for i := 0; i < numRequests; i++ {
+			objectKey := fmt.Sprintf("perf-object-%d.txt", i)
+			err := framework.PutTestObject(client, bucketName, objectKey, "performance test content")
+			require.NoError(t, err)
+
+			_, err = framework.GetTestObject(client, bucketName, objectKey)
+			require.NoError(t, err)
+
+			err = framework.DeleteTestObject(client, bucketName, objectKey)
+			require.NoError(t, err)
+		}
+
+		duration := time.Since(start)
+		avgLatency := duration / (numRequests * 3) // 3 operations per iteration
+
+		t.Logf("Authorization performance: %d operations in %v (avg: %v per operation)",
+			numRequests*3, duration, avgLatency)
+
+		// Performance assertion - should be under 50ms per operation on average
+		assert.Less(t, avgLatency, 50*time.Millisecond,
+			"Average authorization latency should be under 50ms")
+	})
+}
+
+// BenchmarkS3IAMAuthentication benchmarks JWT authentication
+func BenchmarkS3IAMAuthentication(b *testing.B) {
+	if os.Getenv("ENABLE_PERFORMANCE_TESTS") != "true" {
+		b.Skip("Performance tests not enabled. Set ENABLE_PERFORMANCE_TESTS=true")
+	}
+
+	framework := NewS3IAMTestFramework(&testing.T{})
+	defer framework.Cleanup()
+
+	client, err := framework.CreateS3ClientWithJWT("bench-user", "TestAdminRole")
+	require.NoError(b, err)
+
+	bucketName := "test-bench-auth"
+	err = framework.CreateBucket(client, bucketName)
+	require.NoError(b, err)
+	defer func() {
+		_, err := client.DeleteBucket(&s3.DeleteBucketInput{
+			Bucket: aws.String(bucketName),
+		})
+		require.NoError(b, err)
+	}()
+
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			_, err := client.ListBuckets(&s3.ListBucketsInput{})
+			if err != nil {
+				b.Error(err)
+			}
+		}
+	})
+}
+
+// BenchmarkS3IAMAuthorization benchmarks policy evaluation
+func BenchmarkS3IAMAuthorization(b *testing.B) {
+	if os.Getenv("ENABLE_PERFORMANCE_TESTS") != "true" {
+		b.Skip("Performance tests not enabled. Set ENABLE_PERFORMANCE_TESTS=true")
+	}
+
+	framework := NewS3IAMTestFramework(&testing.T{})
+	defer framework.Cleanup()
+
+	client, err := framework.CreateS3ClientWithJWT("bench-user", "TestAdminRole")
+	require.NoError(b, err)
+
+	bucketName := "test-bench-authz"
+	err = framework.CreateBucket(client, bucketName)
+	require.NoError(b, err)
+	defer func() {
+		_, err := client.DeleteBucket(&s3.DeleteBucketInput{
+			Bucket: aws.String(bucketName),
+		})
+		require.NoError(b, err)
+	}()
+
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		i := 0
+		for pb.Next() {
+			objectKey := fmt.Sprintf("bench-object-%d.txt", i)
+			err := framework.PutTestObject(client, bucketName, objectKey, "benchmark content")
+			if err != nil {
+				b.Error(err)
+			}
+			i++
+		}
+	})
+}

test/s3/iam/s3_iam_framework.go

@@ -0,0 +1,982 @@
+package iam
+
+import (
+	"context"
+	cryptorand "crypto/rand"
+	"crypto/rsa"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	mathrand "math/rand"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/golang-jwt/jwt/v5"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	TestS3Endpoint = "http://localhost:8333"
+	TestRegion     = "us-west-2"
+
+	// Keycloak configuration
+	DefaultKeycloakURL   = "http://localhost:8080"
+	KeycloakRealm        = "seaweedfs-test"
+	KeycloakClientID     = "seaweedfs-s3"
+	KeycloakClientSecret = "seaweedfs-s3-secret"
+)
+
+// S3IAMTestFramework provides utilities for S3+IAM integration testing
+type S3IAMTestFramework struct {
+	t              *testing.T
+	mockOIDC       *httptest.Server
+	privateKey     *rsa.PrivateKey
+	publicKey      *rsa.PublicKey
+	createdBuckets []string
+	ctx            context.Context
+	keycloakClient *KeycloakClient
+	useKeycloak    bool
+}
+
+// KeycloakClient handles authentication with Keycloak
+type KeycloakClient struct {
+	baseURL      string
+	realm        string
+	clientID     string
+	clientSecret string
+	httpClient   *http.Client
+}
+
+// KeycloakTokenResponse represents Keycloak token response
+type KeycloakTokenResponse struct {
+	AccessToken  string `json:"access_token"`
+	TokenType    string `json:"token_type"`
+	ExpiresIn    int    `json:"expires_in"`
+	RefreshToken string `json:"refresh_token,omitempty"`
+	Scope        string `json:"scope,omitempty"`
+}
+
+// NewS3IAMTestFramework creates a new test framework instance
+func NewS3IAMTestFramework(t *testing.T) *S3IAMTestFramework {
+	framework := &S3IAMTestFramework{
+		t:              t,
+		ctx:            context.Background(),
+		createdBuckets: make([]string, 0),
+	}
+
+	// Check if we should use Keycloak or mock OIDC
+	keycloakURL := os.Getenv("KEYCLOAK_URL")
+	if keycloakURL == "" {
+		keycloakURL = DefaultKeycloakURL
+	}
+
+	// Test if Keycloak is available
+	framework.useKeycloak = framework.isKeycloakAvailable(keycloakURL)
+
+	if framework.useKeycloak {
+		t.Logf("Using real Keycloak instance at %s", keycloakURL)
+		framework.keycloakClient = NewKeycloakClient(keycloakURL, KeycloakRealm, KeycloakClientID, KeycloakClientSecret)
+	} else {
+		t.Logf("Using mock OIDC server for testing")
+		// Generate RSA keys for JWT signing (mock mode)
+		var err error
+		framework.privateKey, err = rsa.GenerateKey(cryptorand.Reader, 2048)
+		require.NoError(t, err)
+		framework.publicKey = &framework.privateKey.PublicKey
+
+		// Setup mock OIDC server
+		framework.setupMockOIDCServer()
+	}
+
+	return framework
+}
+
+// NewKeycloakClient creates a new Keycloak client
+func NewKeycloakClient(baseURL, realm, clientID, clientSecret string) *KeycloakClient {
+	return &KeycloakClient{
+		baseURL:      baseURL,
+		realm:        realm,
+		clientID:     clientID,
+		clientSecret: clientSecret,
+		httpClient:   &http.Client{Timeout: 30 * time.Second},
+	}
+}
+
+// isKeycloakAvailable checks if Keycloak is running and accessible
+func (f *S3IAMTestFramework) isKeycloakAvailable(keycloakURL string) bool {
+	client := &http.Client{Timeout: 5 * time.Second}
+	// Use realms endpoint instead of health/ready for Keycloak v26+
+	// First, verify master realm is reachable
+	masterURL := fmt.Sprintf("%s/realms/master", keycloakURL)
+
+	resp, err := client.Get(masterURL)
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return false
+	}
+
+	// Also ensure the specific test realm exists; otherwise fall back to mock
+	testRealmURL := fmt.Sprintf("%s/realms/%s", keycloakURL, KeycloakRealm)
+	resp2, err := client.Get(testRealmURL)
+	if err != nil {
+		return false
+	}
+	defer resp2.Body.Close()
+	return resp2.StatusCode == http.StatusOK
+}
+
+// AuthenticateUser authenticates a user with Keycloak and returns an access token
+func (kc *KeycloakClient) AuthenticateUser(username, password string) (*KeycloakTokenResponse, error) {
+	tokenURL := fmt.Sprintf("%s/realms/%s/protocol/openid-connect/token", kc.baseURL, kc.realm)
+
+	data := url.Values{}
+	data.Set("grant_type", "password")
+	data.Set("client_id", kc.clientID)
+	data.Set("client_secret", kc.clientSecret)
+	data.Set("username", username)
+	data.Set("password", password)
+	data.Set("scope", "openid profile email")
+
+	resp, err := kc.httpClient.PostForm(tokenURL, data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to authenticate with Keycloak: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		// Read the response body for debugging
+		body, readErr := io.ReadAll(resp.Body)
+		bodyStr := ""
+		if readErr == nil {
+			bodyStr = string(body)
+		}
+		return nil, fmt.Errorf("Keycloak authentication failed with status: %d, response: %s", resp.StatusCode, bodyStr)
+	}
+
+	var tokenResp KeycloakTokenResponse
+	if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
+		return nil, fmt.Errorf("failed to decode token response: %w", err)
+	}
+
+	return &tokenResp, nil
+}
+
+// getKeycloakToken authenticates with Keycloak and returns a JWT token
+func (f *S3IAMTestFramework) getKeycloakToken(username string) (string, error) {
+	if f.keycloakClient == nil {
+		return "", fmt.Errorf("Keycloak client not initialized")
+	}
+
+	// Map username to password for test users
+	password := f.getTestUserPassword(username)
+	if password == "" {
+		return "", fmt.Errorf("unknown test user: %s", username)
+	}
+
+	tokenResp, err := f.keycloakClient.AuthenticateUser(username, password)
+	if err != nil {
+		return "", fmt.Errorf("failed to authenticate user %s: %w", username, err)
+	}
+
+	return tokenResp.AccessToken, nil
+}
+
+// getTestUserPassword returns the password for test users
+func (f *S3IAMTestFramework) getTestUserPassword(username string) string {
+	// Password generation matches setup_keycloak_docker.sh logic:
+	// password="${username//[^a-zA-Z]/}123" (removes non-alphabetic chars + "123")
+	userPasswords := map[string]string{
+		"admin-user":      "adminuser123",     // "admin-user" -> "adminuser" + "123"
+		"read-user":       "readuser123",      // "read-user" -> "readuser" + "123"
+		"write-user":      "writeuser123",     // "write-user" -> "writeuser" + "123"
+		"write-only-user": "writeonlyuser123", // "write-only-user" -> "writeonlyuser" + "123"
+	}
+
+	return userPasswords[username]
+}
+
+// setupMockOIDCServer creates a mock OIDC server for testing
+func (f *S3IAMTestFramework) setupMockOIDCServer() {
+
+	f.mockOIDC = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/.well-known/openid_configuration":
+			config := map[string]interface{}{
+				"issuer":            "http://" + r.Host,
+				"jwks_uri":          "http://" + r.Host + "/jwks",
+				"userinfo_endpoint": "http://" + r.Host + "/userinfo",
+			}
+			w.Header().Set("Content-Type", "application/json")
+			fmt.Fprintf(w, `{
+				"issuer": "%s",
+				"jwks_uri": "%s",
+				"userinfo_endpoint": "%s"
+			}`, config["issuer"], config["jwks_uri"], config["userinfo_endpoint"])
+
+		case "/jwks":
+			w.Header().Set("Content-Type", "application/json")
+			fmt.Fprintf(w, `{
+				"keys": [
+					{
+						"kty": "RSA",
+						"kid": "test-key-id",
+						"use": "sig",
+						"alg": "RS256",
+						"n": "%s",
+						"e": "AQAB"
+					}
+				]
+			}`, f.encodePublicKey())
+
+		case "/userinfo":
+			authHeader := r.Header.Get("Authorization")
+			if !strings.HasPrefix(authHeader, "Bearer ") {
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+
+			token := strings.TrimPrefix(authHeader, "Bearer ")
+			userInfo := map[string]interface{}{
+				"sub":    "test-user",
+				"email":  "test@example.com",
+				"name":   "Test User",
+				"groups": []string{"users", "developers"},
+			}
+
+			if strings.Contains(token, "admin") {
+				userInfo["groups"] = []string{"admins"}
+			}
+
+			w.Header().Set("Content-Type", "application/json")
+			fmt.Fprintf(w, `{
+				"sub": "%s",
+				"email": "%s",
+				"name": "%s",
+				"groups": %v
+			}`, userInfo["sub"], userInfo["email"], userInfo["name"], userInfo["groups"])
+
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+}
+
+// encodePublicKey encodes the RSA public key for JWKS
+func (f *S3IAMTestFramework) encodePublicKey() string {
+	return base64.RawURLEncoding.EncodeToString(f.publicKey.N.Bytes())
+}
+
+// BearerTokenTransport is an HTTP transport that adds Bearer token authentication
+type BearerTokenTransport struct {
+	Transport http.RoundTripper
+	Token     string
+}
+
+// RoundTrip implements the http.RoundTripper interface
+func (t *BearerTokenTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Clone the request to avoid modifying the original
+	newReq := req.Clone(req.Context())
+
+	// Remove ALL existing Authorization headers first to prevent conflicts
+	newReq.Header.Del("Authorization")
+	newReq.Header.Del("X-Amz-Date")
+	newReq.Header.Del("X-Amz-Content-Sha256")
+	newReq.Header.Del("X-Amz-Signature")
+	newReq.Header.Del("X-Amz-Algorithm")
+	newReq.Header.Del("X-Amz-Credential")
+	newReq.Header.Del("X-Amz-SignedHeaders")
+	newReq.Header.Del("X-Amz-Security-Token")
+
+	// Add Bearer token authorization header
+	newReq.Header.Set("Authorization", "Bearer "+t.Token)
+
+	// Extract and set the principal ARN from JWT token for security compliance
+	if principal := t.extractPrincipalFromJWT(t.Token); principal != "" {
+		newReq.Header.Set("X-SeaweedFS-Principal", principal)
+	}
+
+	// Token preview for logging (first 50 chars for security)
+	tokenPreview := t.Token
+	if len(tokenPreview) > 50 {
+		tokenPreview = tokenPreview[:50] + "..."
+	}
+
+	// Use underlying transport
+	transport := t.Transport
+	if transport == nil {
+		transport = http.DefaultTransport
+	}
+
+	return transport.RoundTrip(newReq)
+}
+
+// extractPrincipalFromJWT extracts the principal ARN from a JWT token without validating it
+// This is used to set the X-SeaweedFS-Principal header that's required after our security fix
+func (t *BearerTokenTransport) extractPrincipalFromJWT(tokenString string) string {
+	// Parse the JWT token without validation to extract the principal claim
+	token, _ := jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) {
+		// We don't validate the signature here, just extract the claims
+		// This is safe because the actual validation happens server-side
+		return []byte("dummy-key"), nil
+	})
+
+	// Even if parsing fails due to signature verification, we might still get claims
+	if claims, ok := token.Claims.(jwt.MapClaims); ok {
+		// Try multiple possible claim names for the principal ARN
+		if principal, exists := claims["principal"]; exists {
+			if principalStr, ok := principal.(string); ok {
+				return principalStr
+			}
+		}
+		if assumed, exists := claims["assumed"]; exists {
+			if assumedStr, ok := assumed.(string); ok {
+				return assumedStr
+			}
+		}
+	}
+
+	return ""
+}
+
+// generateSTSSessionToken creates a session token using the actual STS service for proper validation
+func (f *S3IAMTestFramework) generateSTSSessionToken(username, roleName string, validDuration time.Duration, account string, customClaims map[string]interface{}) (string, error) {
+	now := time.Now()
+	signingKeyB64 := "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc="
+	signingKey, err := base64.StdEncoding.DecodeString(signingKeyB64)
+	if err != nil {
+		return "", fmt.Errorf("failed to decode signing key: %v", err)
+	}
+
+	// Generate a session ID that would be created by the STS service
+	sessionId := fmt.Sprintf("test-session-%s-%s-%d", username, roleName, now.Unix())
+
+	if account == "" {
+		account = "123456789012" // Default test account
+	}
+
+	// Create session token claims exactly matching STSSessionClaims struct
+	roleArn := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, roleName)
+	sessionName := username
+	principalArn := fmt.Sprintf("arn:aws:sts::%s:assumed-role/%s/%s", account, roleName, sessionName)
+
+	// Use jwt.MapClaims but with exact field names that STSSessionClaims expects
+	sessionClaims := jwt.MapClaims{
+		// RegisteredClaims fields
+		"iss": "seaweedfs-sts",
+		"sub": sessionId,
+		"iat": now.Unix(),
+		"exp": now.Add(validDuration).Unix(),
+		"nbf": now.Unix(),
+
+		// STSSessionClaims fields (using exact JSON tags from the struct)
+		"sid":        sessionId,                      // SessionId
+		"snam":       sessionName,                    // SessionName
+		"typ":        "session",                      // TokenType
+		"role":       roleArn,                        // RoleArn
+		"assumed":    principalArn,                   // AssumedRole
+		"principal":  principalArn,                   // Principal
+		"idp":        "test-oidc",                    // IdentityProvider
+		"ext_uid":    username,                       // ExternalUserId
+		"assumed_at": now.Format(time.RFC3339Nano),   // AssumedAt
+		"max_dur":    int64(validDuration.Seconds()), // MaxDuration
+	}
+
+	// Add custom claims (e.g., for ldap:* or jwt:* testing)
+	for k, v := range customClaims {
+		sessionClaims[k] = v
+	}
+
+	token := jwt.NewWithClaims(jwt.SigningMethodHS256, sessionClaims)
+	tokenString, err := token.SignedString(signingKey)
+	if err != nil {
+		return "", err
+	}
+
+	return tokenString, nil
+}
+
+// CreateS3ClientWithJWT creates an S3 client authenticated with a JWT token for the specified role
+func (f *S3IAMTestFramework) CreateS3ClientWithJWT(username, roleName string) (*s3.S3, error) {
+	return f.CreateS3ClientWithCustomClaims(username, roleName, "", nil)
+}
+
+// CreateS3ClientWithCustomClaims creates an S3 client with specific account ID and custom claims
+func (f *S3IAMTestFramework) CreateS3ClientWithCustomClaims(username, roleName, account string, claims map[string]interface{}) (*s3.S3, error) {
+	var token string
+	var err error
+
+	if f.useKeycloak && claims == nil && account == "" {
+		// Use real Keycloak authentication if no custom requirements
+		token, err = f.getKeycloakToken(username)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get Keycloak token: %v", err)
+		}
+	} else {
+		// Generate STS session token (mock mode or custom requirements)
+		token, err = f.generateSTSSessionToken(username, roleName, time.Hour, account, claims)
+		if err != nil {
+			return nil, fmt.Errorf("failed to generate STS session token: %v", err)
+		}
+	}
+
+	// Create custom HTTP client with Bearer token transport
+	httpClient := &http.Client{
+		Transport: &BearerTokenTransport{
+			Token: token,
+		},
+	}
+
+	sess, err := session.NewSession(&aws.Config{
+		Region:     aws.String(TestRegion),
+		Endpoint:   aws.String(TestS3Endpoint),
+		HTTPClient: httpClient,
+		// Use anonymous credentials to avoid AWS signature generation
+		Credentials:      credentials.AnonymousCredentials,
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to create AWS session: %v", err)
+	}
+
+	return s3.New(sess), nil
+}
+
+// CreateS3ClientWithInvalidJWT creates an S3 client with an invalid JWT token
+func (f *S3IAMTestFramework) CreateS3ClientWithInvalidJWT() (*s3.S3, error) {
+	invalidToken := "invalid.jwt.token"
+
+	// Create custom HTTP client with Bearer token transport
+	httpClient := &http.Client{
+		Transport: &BearerTokenTransport{
+			Token: invalidToken,
+		},
+	}
+
+	sess, err := session.NewSession(&aws.Config{
+		Region:     aws.String(TestRegion),
+		Endpoint:   aws.String(TestS3Endpoint),
+		HTTPClient: httpClient,
+		// Use anonymous credentials to avoid AWS signature generation
+		Credentials:      credentials.AnonymousCredentials,
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to create AWS session: %v", err)
+	}
+
+	return s3.New(sess), nil
+}
+
+// CreateS3ClientWithExpiredJWT creates an S3 client with an expired JWT token
+func (f *S3IAMTestFramework) CreateS3ClientWithExpiredJWT(username, roleName string) (*s3.S3, error) {
+	// Generate expired STS session token (expired 1 hour ago)
+	token, err := f.generateSTSSessionToken(username, roleName, -time.Hour, "", nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate expired STS session token: %v", err)
+	}
+
+	// Create custom HTTP client with Bearer token transport
+	httpClient := &http.Client{
+		Transport: &BearerTokenTransport{
+			Token: token,
+		},
+	}
+
+	sess, err := session.NewSession(&aws.Config{
+		Region:     aws.String(TestRegion),
+		Endpoint:   aws.String(TestS3Endpoint),
+		HTTPClient: httpClient,
+		// Use anonymous credentials to avoid AWS signature generation
+		Credentials:      credentials.AnonymousCredentials,
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to create AWS session: %v", err)
+	}
+
+	return s3.New(sess), nil
+}
+
+// CreateS3ClientWithSessionToken creates an S3 client with a session token
+func (f *S3IAMTestFramework) CreateS3ClientWithSessionToken(sessionToken string) (*s3.S3, error) {
+	sess, err := session.NewSession(&aws.Config{
+		Region:   aws.String(TestRegion),
+		Endpoint: aws.String(TestS3Endpoint),
+		Credentials: credentials.NewStaticCredentials(
+			"session-access-key",
+			"session-secret-key",
+			sessionToken,
+		),
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to create AWS session: %v", err)
+	}
+
+	return s3.New(sess), nil
+}
+
+// CreateS3ClientWithKeycloakToken creates an S3 client using a Keycloak JWT token
+func (f *S3IAMTestFramework) CreateS3ClientWithKeycloakToken(keycloakToken string) (*s3.S3, error) {
+	// Determine response header timeout based on environment
+	responseHeaderTimeout := 10 * time.Second
+	overallTimeout := 30 * time.Second
+	if os.Getenv("GITHUB_ACTIONS") == "true" {
+		responseHeaderTimeout = 30 * time.Second // Longer timeout for CI JWT validation
+		overallTimeout = 60 * time.Second
+	}
+
+	// Create a fresh HTTP transport with appropriate timeouts
+	transport := &http.Transport{
+		DisableKeepAlives:     true, // Force new connections for each request
+		DisableCompression:    true, // Disable compression to simplify requests
+		MaxIdleConns:          0,    // No connection pooling
+		MaxIdleConnsPerHost:   0,    // No connection pooling per host
+		IdleConnTimeout:       1 * time.Second,
+		TLSHandshakeTimeout:   5 * time.Second,
+		ResponseHeaderTimeout: responseHeaderTimeout, // Adjustable for CI environments
+		ExpectContinueTimeout: 1 * time.Second,
+	}
+
+	// Create a custom HTTP client with appropriate timeouts
+	httpClient := &http.Client{
+		Timeout: overallTimeout, // Overall request timeout (adjustable for CI)
+		Transport: &BearerTokenTransport{
+			Token:     keycloakToken,
+			Transport: transport,
+		},
+	}
+
+	sess, err := session.NewSession(&aws.Config{
+		Region:           aws.String(TestRegion),
+		Endpoint:         aws.String(TestS3Endpoint),
+		Credentials:      credentials.AnonymousCredentials,
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+		HTTPClient:       httpClient,
+		MaxRetries:       aws.Int(0), // No retries to avoid delays
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to create AWS session: %v", err)
+	}
+
+	return s3.New(sess), nil
+}
+
+// TestKeycloakTokenDirectly tests a Keycloak token with direct HTTP request (bypassing AWS SDK)
+func (f *S3IAMTestFramework) TestKeycloakTokenDirectly(keycloakToken string) error {
+	// Create a simple HTTP client with timeout
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	// Create request to list buckets
+	req, err := http.NewRequest("GET", TestS3Endpoint, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create request: %v", err)
+	}
+
+	// Add Bearer token
+	req.Header.Set("Authorization", "Bearer "+keycloakToken)
+	req.Header.Set("Host", "localhost:8333")
+
+	// Make request
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("request failed: %v", err)
+	}
+	defer resp.Body.Close()
+
+	// Read response
+	_, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed to read response: %v", err)
+	}
+
+	return nil
+}
+
+// generateJWTToken creates a JWT token for testing
+func (f *S3IAMTestFramework) generateJWTToken(username, roleName string, validDuration time.Duration) (string, error) {
+	now := time.Now()
+	claims := jwt.MapClaims{
+		"sub":   username,
+		"iss":   f.mockOIDC.URL,
+		"aud":   "test-client",
+		"exp":   now.Add(validDuration).Unix(),
+		"iat":   now.Unix(),
+		"email": username + "@example.com",
+		"name":  strings.Title(username),
+	}
+
+	// Add role-specific groups
+	switch roleName {
+	case "TestAdminRole":
+		claims["groups"] = []string{"admins"}
+	case "TestReadOnlyRole":
+		claims["groups"] = []string{"users"}
+	case "TestWriteOnlyRole":
+		claims["groups"] = []string{"writers"}
+	default:
+		claims["groups"] = []string{"users"}
+	}
+
+	token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims)
+	token.Header["kid"] = "test-key-id"
+
+	tokenString, err := token.SignedString(f.privateKey)
+	if err != nil {
+		return "", fmt.Errorf("failed to sign token: %v", err)
+	}
+
+	return tokenString, nil
+}
+
+// CreateShortLivedSessionToken creates a mock session token for testing
+func (f *S3IAMTestFramework) CreateShortLivedSessionToken(username, roleName string, durationSeconds int64) (string, error) {
+	// For testing purposes, create a mock session token
+	// In reality, this would be generated by the STS service
+	return fmt.Sprintf("mock-session-token-%s-%s-%d", username, roleName, time.Now().Unix()), nil
+}
+
+// ExpireSessionForTesting simulates session expiration for testing
+func (f *S3IAMTestFramework) ExpireSessionForTesting(sessionToken string) error {
+	// For integration tests, this would typically involve calling the STS service
+	// For now, we just simulate success since the actual expiration will be handled by SeaweedFS
+	return nil
+}
+
+// GenerateUniqueBucketName generates a unique bucket name for testing
+func (f *S3IAMTestFramework) GenerateUniqueBucketName(prefix string) string {
+	// Use test name and timestamp to ensure uniqueness
+	testName := strings.ToLower(f.t.Name())
+	testName = strings.ReplaceAll(testName, "/", "-")
+	testName = strings.ReplaceAll(testName, "_", "-")
+
+	// Truncate test name to keep total length under 63 characters
+	// S3 bucket names must be 3-63 characters, lowercase, no underscores
+	// Format: prefix-testname-random (need room for random suffix)
+	maxTestNameLen := 63 - len(prefix) - 5 - 4 // account for dashes and random suffix
+	if len(testName) > maxTestNameLen {
+		testName = testName[:maxTestNameLen]
+	}
+
+	// Add random suffix to handle parallel tests
+	randomSuffix := mathrand.Intn(10000)
+
+	bucketName := fmt.Sprintf("%s-%s-%d", prefix, testName, randomSuffix)
+
+	// Ensure final name is valid
+	if len(bucketName) > 63 {
+		// Truncate further if necessary
+		bucketName = bucketName[:63]
+	}
+
+	return bucketName
+}
+
+// CreateBucket creates a bucket and tracks it for cleanup
+func (f *S3IAMTestFramework) CreateBucket(s3Client *s3.S3, bucketName string) error {
+	_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		return err
+	}
+
+	// Track bucket for cleanup
+	f.createdBuckets = append(f.createdBuckets, bucketName)
+	return nil
+}
+
+// CreateBucketWithCleanup creates a bucket, cleaning up any existing bucket first
+func (f *S3IAMTestFramework) CreateBucketWithCleanup(s3Client *s3.S3, bucketName string) error {
+	// First try to create the bucket normally
+	_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+
+	if err != nil {
+		// If bucket already exists, clean it up first
+		if awsErr, ok := err.(awserr.Error); ok && (awsErr.Code() == "BucketAlreadyExists" || awsErr.Code() == "BucketAlreadyOwnedByYou") {
+			f.t.Logf("Bucket %s already exists, cleaning up first", bucketName)
+
+			// First try to delete the bucket completely
+			f.emptyBucket(s3Client, bucketName)
+			_, deleteErr := s3Client.DeleteBucket(&s3.DeleteBucketInput{
+				Bucket: aws.String(bucketName),
+			})
+			if deleteErr != nil {
+				f.t.Logf("Warning: Failed to delete existing bucket %s: %v", bucketName, deleteErr)
+			}
+
+			// Add a small delay to allow deletion to propagate
+			time.Sleep(100 * time.Millisecond)
+
+			// Now create it fresh
+			_, err = s3Client.CreateBucket(&s3.CreateBucketInput{
+				Bucket: aws.String(bucketName),
+			})
+			if err != nil {
+				// If it still says bucket exists after cleanup, it might be in an inconsistent state
+				// In this case, just use the existing bucket since we emptied it
+				if awsErr, ok := err.(awserr.Error); ok && (awsErr.Code() == "BucketAlreadyExists" || awsErr.Code() == "BucketAlreadyOwnedByYou") {
+					f.t.Logf("Bucket %s still exists after cleanup, reusing it", bucketName)
+					// Bucket exists and is empty, so we can proceed
+				} else {
+					return fmt.Errorf("failed to recreate bucket after cleanup: %v", err)
+				}
+			}
+		} else {
+			return err
+		}
+	}
+
+	// Track bucket for cleanup
+	f.createdBuckets = append(f.createdBuckets, bucketName)
+	return nil
+}
+
+// emptyBucket removes all objects from a bucket
+func (f *S3IAMTestFramework) emptyBucket(s3Client *s3.S3, bucketName string) {
+	// Delete all objects
+	listResult, err := s3Client.ListObjects(&s3.ListObjectsInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err == nil {
+		for _, obj := range listResult.Contents {
+			_, err := s3Client.DeleteObject(&s3.DeleteObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    obj.Key,
+			})
+			if err != nil {
+				f.t.Logf("Warning: Failed to delete object %s/%s: %v", bucketName, *obj.Key, err)
+			}
+		}
+	}
+}
+
+// Cleanup cleans up test resources
+func (f *S3IAMTestFramework) Cleanup() {
+	// Clean up buckets (best effort)
+	if len(f.createdBuckets) > 0 {
+		// Create admin client for cleanup
+		adminClient, err := f.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+		if err == nil {
+			for _, bucket := range f.createdBuckets {
+				// Try to empty bucket first
+				listResult, err := adminClient.ListObjects(&s3.ListObjectsInput{
+					Bucket: aws.String(bucket),
+				})
+				if err == nil {
+					for _, obj := range listResult.Contents {
+						adminClient.DeleteObject(&s3.DeleteObjectInput{
+							Bucket: aws.String(bucket),
+							Key:    obj.Key,
+						})
+					}
+				}
+
+				// Delete bucket
+				adminClient.DeleteBucket(&s3.DeleteBucketInput{
+					Bucket: aws.String(bucket),
+				})
+			}
+		}
+	}
+
+	// Close mock OIDC server
+	if f.mockOIDC != nil {
+		f.mockOIDC.Close()
+	}
+}
+
+// WaitForS3Service waits for the S3 service to be available and checks for IAM write permissions
+func (f *S3IAMTestFramework) WaitForS3Service() error {
+	// Create a basic S3 client
+	sess, err := session.NewSession(&aws.Config{
+		Region:   aws.String(TestRegion),
+		Endpoint: aws.String(TestS3Endpoint),
+		Credentials: credentials.NewStaticCredentials(
+			"test-access-key",
+			"test-secret-key",
+			"",
+		),
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+	})
+	if err != nil {
+		return fmt.Errorf("failed to create AWS session: %v", err)
+	}
+
+	s3Client := s3.New(sess)
+
+	// Create IAM client for write permission check
+	iamClient := iam.New(sess)
+
+	// Try to list buckets to check if S3 service is available
+	maxRetries := 30
+	for i := 0; i < maxRetries; i++ {
+		_, err := s3Client.ListBuckets(&s3.ListBucketsInput{})
+		if err == nil {
+			// S3 is up, now check if IAM is writable
+			// We try to create a dummy user. If it fails with "AccessDenied: IAM write operations are disabled",
+			// we know we are still in read-only mode (or the flag didn't take effect).
+			// If it fails with other errors (e.g. invalid auth), that's fine for this connectivity check.
+			// Only the explicit read-only error is a blocker for our specific test scenario.
+
+			// Note: We use a random name to avoid conflicts if it actually succeeds
+			dummyUser := fmt.Sprintf("check-writable-%d", time.Now().UnixNano())
+			_, iamErr := iamClient.CreateUser(&iam.CreateUserInput{
+				UserName: aws.String(dummyUser),
+			})
+
+			if iamErr != nil {
+				if reqErr, ok := iamErr.(awserr.RequestFailure); ok {
+					if reqErr.Code() == "AccessDenied" && strings.Contains(reqErr.Message(), "IAM write operations are disabled") {
+						f.t.Logf("Waiting for IAM to become writable... (attempt %d/%d)", i+1, maxRetries)
+						time.Sleep(1 * time.Second)
+						continue
+					}
+				}
+				// Ignore other errors (like auth errors), we just want to ensure we aren't explicitly blocked by read-only mode
+			} else {
+				// Cleanup if it actually succeeded
+				iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(dummyUser)})
+			}
+
+			return nil
+		}
+		time.Sleep(1 * time.Second)
+	}
+
+	return fmt.Errorf("S3 service not available or not writable after %d retries", maxRetries)
+}
+
+// PutTestObject puts a test object in the specified bucket
+func (f *S3IAMTestFramework) PutTestObject(client *s3.S3, bucket, key, content string) error {
+	_, err := client.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+		Body:   strings.NewReader(content),
+	})
+	return err
+}
+
+// GetTestObject retrieves a test object from the specified bucket
+func (f *S3IAMTestFramework) GetTestObject(client *s3.S3, bucket, key string) (string, error) {
+	result, err := client.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		return "", err
+	}
+	defer result.Body.Close()
+
+	content := strings.Builder{}
+	_, err = io.Copy(&content, result.Body)
+	if err != nil {
+		return "", err
+	}
+
+	return content.String(), nil
+}
+
+// ListTestObjects lists objects in the specified bucket
+func (f *S3IAMTestFramework) ListTestObjects(client *s3.S3, bucket string) ([]string, error) {
+	result, err := client.ListObjects(&s3.ListObjectsInput{
+		Bucket: aws.String(bucket),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	var keys []string
+	for _, obj := range result.Contents {
+		keys = append(keys, *obj.Key)
+	}
+
+	return keys, nil
+}
+
+// DeleteTestObject deletes a test object from the specified bucket
+func (f *S3IAMTestFramework) DeleteTestObject(client *s3.S3, bucket, key string) error {
+	_, err := client.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	return err
+}
+
+// WaitForS3Service waits for the S3 service to be available (simplified version)
+func (f *S3IAMTestFramework) WaitForS3ServiceSimple() error {
+	// This is a simplified version that just checks if the endpoint responds
+	// The full implementation would be in the Makefile's wait-for-services target
+	return nil
+}
+
+// CreateIAMClientWithJWT creates an IAM client authenticated with a JWT token for the specified role
+func (f *S3IAMTestFramework) CreateIAMClientWithJWT(username, roleName string) (*iam.IAM, error) {
+	return f.CreateIAMClientWithCustomClaims(username, roleName, "", nil)
+}
+
+// CreateIAMClientWithCustomClaims creates an IAM client with specific account ID and custom claims
+func (f *S3IAMTestFramework) CreateIAMClientWithCustomClaims(username, roleName, account string, claims map[string]interface{}) (*iam.IAM, error) {
+	var token string
+	var err error
+
+	if f.useKeycloak && claims == nil && account == "" {
+		// Use real Keycloak authentication if no custom requirements
+		token, err = f.getKeycloakToken(username)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get Keycloak token: %v", err)
+		}
+	} else {
+		// Generate STS session token (mock mode or custom requirements)
+		token, err = f.generateSTSSessionToken(username, roleName, time.Hour, account, claims)
+		if err != nil {
+			return nil, fmt.Errorf("failed to generate STS session token: %v", err)
+		}
+	}
+
+	// Create custom HTTP client with Bearer token transport
+	httpClient := &http.Client{
+		Transport: &BearerTokenTransport{
+			Token: token,
+		},
+	}
+
+	sess, err := session.NewSession(&aws.Config{
+		Region:     aws.String(TestRegion),
+		Endpoint:   aws.String(TestS3Endpoint),
+		HTTPClient: httpClient,
+		// Use anonymous credentials to avoid AWS signature generation
+		Credentials: credentials.AnonymousCredentials,
+		DisableSSL:  aws.Bool(true),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to create AWS session: %v", err)
+	}
+
+	return iam.New(sess), nil
+}

test/s3/iam/s3_iam_group_test.go

@@ -0,0 +1,891 @@
+package iam
+
+import (
+	"encoding/xml"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestIAMGroupLifecycle tests the full lifecycle of group management:
+// CreateGroup, GetGroup, ListGroups, DeleteGroup
+func TestIAMGroupLifecycle(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	groupName := "test-group-lifecycle"
+
+	t.Run("create_group", func(t *testing.T) {
+		resp, err := iamClient.CreateGroup(&iam.CreateGroupInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, groupName, *resp.Group.GroupName)
+	})
+
+	t.Run("get_group", func(t *testing.T) {
+		resp, err := iamClient.GetGroup(&iam.GetGroupInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, groupName, *resp.Group.GroupName)
+	})
+
+	t.Run("list_groups_contains_created", func(t *testing.T) {
+		resp, err := iamClient.ListGroups(&iam.ListGroupsInput{})
+		require.NoError(t, err)
+		found := false
+		for _, g := range resp.Groups {
+			if *g.GroupName == groupName {
+				found = true
+				break
+			}
+		}
+		assert.True(t, found, "Created group should appear in ListGroups")
+	})
+
+	t.Run("create_duplicate_group_fails", func(t *testing.T) {
+		_, err := iamClient.CreateGroup(&iam.CreateGroupInput{
+			GroupName: aws.String(groupName),
+		})
+		assert.Error(t, err, "Creating a duplicate group should fail")
+	})
+
+	t.Run("delete_group", func(t *testing.T) {
+		_, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+
+		// Verify it's gone
+		resp, err := iamClient.ListGroups(&iam.ListGroupsInput{})
+		require.NoError(t, err)
+		for _, g := range resp.Groups {
+			assert.NotEqual(t, groupName, *g.GroupName,
+				"Deleted group should not appear in ListGroups")
+		}
+	})
+
+	t.Run("delete_nonexistent_group_fails", func(t *testing.T) {
+		_, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{
+			GroupName: aws.String("nonexistent-group-xyz"),
+		})
+		assert.Error(t, err)
+	})
+}
+
+// TestIAMGroupMembership tests adding and removing users from groups
+func TestIAMGroupMembership(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	groupName := "test-group-members"
+	userName := "test-user-for-group"
+
+	// Setup: create group and user
+	_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
+		GroupName: aws.String(groupName),
+	})
+	require.NoError(t, err)
+	defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)})
+
+	_, err = iamClient.CreateUser(&iam.CreateUserInput{
+		UserName: aws.String(userName),
+	})
+	require.NoError(t, err)
+	defer iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
+
+	t.Run("add_user_to_group", func(t *testing.T) {
+		_, err := iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
+			GroupName: aws.String(groupName),
+			UserName:  aws.String(userName),
+		})
+		require.NoError(t, err)
+	})
+
+	t.Run("get_group_shows_member", func(t *testing.T) {
+		resp, err := iamClient.GetGroup(&iam.GetGroupInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		found := false
+		for _, u := range resp.Users {
+			if *u.UserName == userName {
+				found = true
+				break
+			}
+		}
+		assert.True(t, found, "Added user should appear in GetGroup members")
+	})
+
+	t.Run("list_groups_for_user", func(t *testing.T) {
+		resp, err := iamClient.ListGroupsForUser(&iam.ListGroupsForUserInput{
+			UserName: aws.String(userName),
+		})
+		require.NoError(t, err)
+		found := false
+		for _, g := range resp.Groups {
+			if *g.GroupName == groupName {
+				found = true
+				break
+			}
+		}
+		assert.True(t, found, "Group should appear in ListGroupsForUser")
+	})
+
+	t.Run("add_duplicate_member_is_idempotent", func(t *testing.T) {
+		_, err := iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
+			GroupName: aws.String(groupName),
+			UserName:  aws.String(userName),
+		})
+		// Should succeed (idempotent) or return a benign error
+		// AWS IAM allows duplicate add without error
+		assert.NoError(t, err)
+	})
+
+	t.Run("remove_user_from_group", func(t *testing.T) {
+		_, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
+			GroupName: aws.String(groupName),
+			UserName:  aws.String(userName),
+		})
+		require.NoError(t, err)
+
+		// Verify removal
+		resp, err := iamClient.GetGroup(&iam.GetGroupInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		for _, u := range resp.Users {
+			assert.NotEqual(t, userName, *u.UserName,
+				"Removed user should not appear in group members")
+		}
+	})
+}
+
+// TestIAMGroupPolicyAttachment tests attaching and detaching policies from groups
+func TestIAMGroupPolicyAttachment(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	groupName := "test-group-policies"
+	policyName := "test-group-attach-policy"
+	policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListBucket","Resource":"*"}]}`
+
+	// Setup: create group and policy
+	_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
+		GroupName: aws.String(groupName),
+	})
+	require.NoError(t, err)
+
+	createPolicyResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
+		PolicyName:     aws.String(policyName),
+		PolicyDocument: aws.String(policyDoc),
+	})
+	require.NoError(t, err)
+	policyArn := createPolicyResp.Policy.Arn
+
+	// Cleanup in correct order: detach policy, delete group, delete policy
+	t.Cleanup(func() {
+		if _, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
+			GroupName: aws.String(groupName),
+			PolicyArn: policyArn,
+		}); err != nil {
+			t.Logf("cleanup: failed to detach group policy: %v", err)
+		}
+		if _, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)}); err != nil {
+			t.Logf("cleanup: failed to delete group: %v", err)
+		}
+		if _, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policyArn}); err != nil {
+			t.Logf("cleanup: failed to delete policy: %v", err)
+		}
+	})
+
+	t.Run("attach_group_policy", func(t *testing.T) {
+		_, err := iamClient.AttachGroupPolicy(&iam.AttachGroupPolicyInput{
+			GroupName: aws.String(groupName),
+			PolicyArn: policyArn,
+		})
+		require.NoError(t, err)
+	})
+
+	t.Run("list_attached_group_policies", func(t *testing.T) {
+		resp, err := iamClient.ListAttachedGroupPolicies(&iam.ListAttachedGroupPoliciesInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		found := false
+		for _, p := range resp.AttachedPolicies {
+			if *p.PolicyName == policyName {
+				found = true
+				break
+			}
+		}
+		assert.True(t, found, "Attached policy should appear in ListAttachedGroupPolicies")
+	})
+
+	t.Run("detach_group_policy", func(t *testing.T) {
+		_, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
+			GroupName: aws.String(groupName),
+			PolicyArn: policyArn,
+		})
+		require.NoError(t, err)
+
+		// Verify detachment
+		resp, err := iamClient.ListAttachedGroupPolicies(&iam.ListAttachedGroupPoliciesInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		for _, p := range resp.AttachedPolicies {
+			assert.NotEqual(t, policyName, *p.PolicyName,
+				"Detached policy should not appear in ListAttachedGroupPolicies")
+		}
+	})
+}
+
+// TestIAMGroupPolicyEnforcement tests that group policies are enforced during S3 operations.
+// Creates a user with no direct policies, adds them to a group with S3 access,
+// and verifies they can access S3 through the group policy.
+func TestIAMGroupPolicyEnforcement(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	groupName := "test-enforcement-group"
+	userName := "test-enforcement-user"
+	policyName := "test-enforcement-policy"
+	bucketName := "test-group-enforce-bucket"
+	policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::` + bucketName + `","arn:aws:s3:::` + bucketName + `/*"]}]}`
+
+	// Create user
+	_, err = iamClient.CreateUser(&iam.CreateUserInput{
+		UserName: aws.String(userName),
+	})
+	require.NoError(t, err)
+
+	// Create access key for the user
+	keyResp, err := iamClient.CreateAccessKey(&iam.CreateAccessKeyInput{
+		UserName: aws.String(userName),
+	})
+	require.NoError(t, err)
+
+	accessKeyId := *keyResp.AccessKey.AccessKeyId
+	secretKey := *keyResp.AccessKey.SecretAccessKey
+
+	// Create an S3 client with the user's credentials
+	userS3Client := createS3Client(t, accessKeyId, secretKey)
+
+	// Create group
+	_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
+		GroupName: aws.String(groupName),
+	})
+	require.NoError(t, err)
+
+	// Create policy
+	createPolicyResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
+		PolicyName:     aws.String(policyName),
+		PolicyDocument: aws.String(policyDoc),
+	})
+	require.NoError(t, err)
+	policyArn := createPolicyResp.Policy.Arn
+
+	// Cleanup in correct order: remove user from group, detach policy,
+	// delete access key, delete user, delete group, delete policy
+	t.Cleanup(func() {
+		if _, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
+			GroupName: aws.String(groupName),
+			UserName:  aws.String(userName),
+		}); err != nil {
+			t.Logf("cleanup: failed to remove user from group: %v", err)
+		}
+		if _, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
+			GroupName: aws.String(groupName),
+			PolicyArn: policyArn,
+		}); err != nil {
+			t.Logf("cleanup: failed to detach group policy: %v", err)
+		}
+		if _, err := iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{
+			UserName:    aws.String(userName),
+			AccessKeyId: keyResp.AccessKey.AccessKeyId,
+		}); err != nil {
+			t.Logf("cleanup: failed to delete access key: %v", err)
+		}
+		if _, err := iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)}); err != nil {
+			t.Logf("cleanup: failed to delete user: %v", err)
+		}
+		if _, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)}); err != nil {
+			t.Logf("cleanup: failed to delete group: %v", err)
+		}
+		if _, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policyArn}); err != nil {
+			t.Logf("cleanup: failed to delete policy: %v", err)
+		}
+	})
+
+	// Register bucket cleanup on parent test with admin credentials
+	// (userS3Client may lack permissions by cleanup time)
+	adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		if _, err := adminS3.DeleteObject(&s3.DeleteObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String("test-key"),
+		}); err != nil {
+			t.Logf("cleanup: failed to delete object: %v", err)
+		}
+		if _, err := adminS3.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)}); err != nil {
+			t.Logf("cleanup: failed to delete bucket: %v", err)
+		}
+	})
+
+	t.Run("user_without_group_denied", func(t *testing.T) {
+		// User has no policies and is not in any group — should be denied
+		_, err := userS3Client.CreateBucket(&s3.CreateBucketInput{
+			Bucket: aws.String(bucketName),
+		})
+		require.Error(t, err, "User without any policies should be denied")
+		awsErr, ok := err.(awserr.Error)
+		require.True(t, ok, "Expected awserr.Error")
+		assert.Equal(t, "AccessDenied", awsErr.Code())
+	})
+
+	t.Run("user_with_group_policy_allowed", func(t *testing.T) {
+		// Attach policy to group
+		_, err := iamClient.AttachGroupPolicy(&iam.AttachGroupPolicyInput{
+			GroupName: aws.String(groupName),
+			PolicyArn: policyArn,
+		})
+		require.NoError(t, err)
+
+		// Add user to group
+		_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
+			GroupName: aws.String(groupName),
+			UserName:  aws.String(userName),
+		})
+		require.NoError(t, err)
+
+		// Wait for policy propagation, then create bucket
+		require.Eventually(t, func() bool {
+			_, err = userS3Client.CreateBucket(&s3.CreateBucketInput{
+				Bucket: aws.String(bucketName),
+			})
+			return err == nil
+		}, 10*time.Second, 500*time.Millisecond, "User with group policy should be allowed")
+
+		// Should also be able to put/get objects
+		_, err = userS3Client.PutObject(&s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String("test-key"),
+			Body:   aws.ReadSeekCloser(strings.NewReader("test-data")),
+		})
+		require.NoError(t, err, "User should be able to put objects through group policy")
+	})
+
+	t.Run("user_removed_from_group_denied", func(t *testing.T) {
+		// Remove user from group
+		_, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
+			GroupName: aws.String(groupName),
+			UserName:  aws.String(userName),
+		})
+		require.NoError(t, err)
+
+		// Wait for policy propagation — user should now be denied
+		var lastErr error
+		require.Eventually(t, func() bool {
+			_, lastErr = userS3Client.ListObjects(&s3.ListObjectsInput{
+				Bucket: aws.String(bucketName),
+			})
+			return lastErr != nil
+		}, 10*time.Second, 500*time.Millisecond, "User removed from group should be denied")
+		awsErr, ok := lastErr.(awserr.Error)
+		require.True(t, ok, "Expected awserr.Error")
+		assert.Equal(t, "AccessDenied", awsErr.Code())
+	})
+}
+
+// TestIAMGroupDisabledPolicyEnforcement tests that disabled groups do not contribute policies.
+// Uses the raw IAM API (callIAMAPI) since the AWS SDK doesn't support custom group status.
+func TestIAMGroupDisabledPolicyEnforcement(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	if !isSeaweedFSRunning(t) {
+		t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
+	}
+
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	groupName := "test-disabled-group"
+	userName := "test-disabled-grp-user"
+	policyName := "test-disabled-grp-policy"
+	bucketName := "test-disabled-grp-bucket"
+	policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::` + bucketName + `","arn:aws:s3:::` + bucketName + `/*"]}]}`
+
+	// Create user, group, policy
+	_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
+	require.NoError(t, err)
+
+	keyResp, err := iamClient.CreateAccessKey(&iam.CreateAccessKeyInput{UserName: aws.String(userName)})
+	require.NoError(t, err)
+
+	_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(groupName)})
+	require.NoError(t, err)
+
+	createPolicyResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
+		PolicyName: aws.String(policyName), PolicyDocument: aws.String(policyDoc),
+	})
+	require.NoError(t, err)
+
+	// Cleanup in correct order: remove user from group, detach policy,
+	// delete access key, delete user, delete group, delete policy
+	t.Cleanup(func() {
+		if _, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
+			GroupName: aws.String(groupName), UserName: aws.String(userName),
+		}); err != nil {
+			t.Logf("cleanup: failed to remove user from group: %v", err)
+		}
+		if _, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
+			GroupName: aws.String(groupName),
+			PolicyArn: aws.String("arn:aws:iam:::policy/" + policyName),
+		}); err != nil {
+			t.Logf("cleanup: failed to detach group policy: %v", err)
+		}
+		if _, err := iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{
+			UserName: aws.String(userName), AccessKeyId: keyResp.AccessKey.AccessKeyId,
+		}); err != nil {
+			t.Logf("cleanup: failed to delete access key: %v", err)
+		}
+		if _, err := iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)}); err != nil {
+			t.Logf("cleanup: failed to delete user: %v", err)
+		}
+		if _, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)}); err != nil {
+			t.Logf("cleanup: failed to delete group: %v", err)
+		}
+		if _, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: createPolicyResp.Policy.Arn}); err != nil {
+			t.Logf("cleanup: failed to delete policy: %v", err)
+		}
+	})
+
+	// Setup: attach policy, add user, create bucket with admin
+	_, err = iamClient.AttachGroupPolicy(&iam.AttachGroupPolicyInput{
+		GroupName: aws.String(groupName), PolicyArn: createPolicyResp.Policy.Arn,
+	})
+	require.NoError(t, err)
+
+	_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
+		GroupName: aws.String(groupName), UserName: aws.String(userName),
+	})
+	require.NoError(t, err)
+
+	userS3Client := createS3Client(t, *keyResp.AccessKey.AccessKeyId, *keyResp.AccessKey.SecretAccessKey)
+
+	// Create bucket using admin first so we can test listing
+	adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+	_, err = adminS3.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucketName)})
+	require.NoError(t, err)
+	defer adminS3.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
+
+	t.Run("enabled_group_allows_access", func(t *testing.T) {
+		require.Eventually(t, func() bool {
+			_, err := userS3Client.ListObjects(&s3.ListObjectsInput{
+				Bucket: aws.String(bucketName),
+			})
+			return err == nil
+		}, 10*time.Second, 500*time.Millisecond, "User in enabled group should have access")
+	})
+
+	t.Run("disabled_group_denies_access", func(t *testing.T) {
+		// Disable group via raw IAM API (no SDK support for this extension)
+		resp, err := callIAMAPIAuthenticated(t, framework, "UpdateGroup", url.Values{
+			"GroupName": {groupName},
+			"Disabled":  {"true"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+		require.Equal(t, http.StatusOK, resp.StatusCode, "UpdateGroup (disable) should return 200")
+
+		// Wait for propagation — user should be denied
+		var lastErr error
+		require.Eventually(t, func() bool {
+			_, lastErr = userS3Client.ListObjects(&s3.ListObjectsInput{
+				Bucket: aws.String(bucketName),
+			})
+			return lastErr != nil
+		}, 10*time.Second, 500*time.Millisecond, "User in disabled group should be denied access")
+		awsErr, ok := lastErr.(awserr.Error)
+		require.True(t, ok, "Expected awserr.Error")
+		assert.Equal(t, "AccessDenied", awsErr.Code())
+	})
+
+	t.Run("re_enabled_group_restores_access", func(t *testing.T) {
+		// Re-enable the group
+		resp, err := callIAMAPIAuthenticated(t, framework, "UpdateGroup", url.Values{
+			"GroupName": {groupName},
+			"Disabled":  {"false"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+		require.Equal(t, http.StatusOK, resp.StatusCode, "UpdateGroup (re-enable) should return 200")
+
+		// Wait for propagation — user should have access again
+		require.Eventually(t, func() bool {
+			_, err = userS3Client.ListObjects(&s3.ListObjectsInput{
+				Bucket: aws.String(bucketName),
+			})
+			return err == nil
+		}, 10*time.Second, 500*time.Millisecond, "User in re-enabled group should have access again")
+	})
+}
+
+// TestIAMGroupUserDeletionSideEffect tests that deleting a user removes them from all groups.
+func TestIAMGroupUserDeletionSideEffect(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	groupName := "test-deletion-group"
+	userName := "test-deletion-user"
+
+	// Create group and user
+	_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(groupName)})
+	require.NoError(t, err)
+	defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)})
+
+	_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		// Best-effort: user may already be deleted by the test
+		iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
+	})
+
+	// Add user to group
+	_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
+		GroupName: aws.String(groupName),
+		UserName:  aws.String(userName),
+	})
+	require.NoError(t, err)
+
+	// Verify user is in group
+	getResp, err := iamClient.GetGroup(&iam.GetGroupInput{GroupName: aws.String(groupName)})
+	require.NoError(t, err)
+	assert.Len(t, getResp.Users, 1, "Group should have 1 member before deletion")
+
+	// Delete the user
+	_, err = iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
+	require.NoError(t, err)
+
+	// Verify user was removed from the group
+	getResp, err = iamClient.GetGroup(&iam.GetGroupInput{GroupName: aws.String(groupName)})
+	require.NoError(t, err)
+	assert.Empty(t, getResp.Users, "Group should have no members after user deletion")
+}
+
+// TestIAMGroupMultipleGroups tests that a user can belong to multiple groups
+// and inherits policies from all of them.
+func TestIAMGroupMultipleGroups(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	group1 := "test-multi-group-1"
+	group2 := "test-multi-group-2"
+	userName := "test-multi-group-user"
+
+	// Create two groups
+	_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(group1)})
+	require.NoError(t, err)
+	defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(group1)})
+
+	_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(group2)})
+	require.NoError(t, err)
+	defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(group2)})
+
+	// Create user
+	_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
+	require.NoError(t, err)
+	defer func() {
+		iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
+			GroupName: aws.String(group1), UserName: aws.String(userName),
+		})
+		iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
+			GroupName: aws.String(group2), UserName: aws.String(userName),
+		})
+		iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
+	}()
+
+	// Add user to both groups
+	_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
+		GroupName: aws.String(group1), UserName: aws.String(userName),
+	})
+	require.NoError(t, err)
+
+	_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
+		GroupName: aws.String(group2), UserName: aws.String(userName),
+	})
+	require.NoError(t, err)
+
+	// Verify user appears in both groups
+	resp, err := iamClient.ListGroupsForUser(&iam.ListGroupsForUserInput{
+		UserName: aws.String(userName),
+	})
+	require.NoError(t, err)
+	groupNames := make(map[string]bool)
+	for _, g := range resp.Groups {
+		groupNames[*g.GroupName] = true
+	}
+	assert.True(t, groupNames[group1], "User should be in group 1")
+	assert.True(t, groupNames[group2], "User should be in group 2")
+}
+
+// --- Response types for raw IAM API calls ---
+
+type CreateGroupResponse struct {
+	XMLName           xml.Name `xml:"CreateGroupResponse"`
+	CreateGroupResult struct {
+		Group struct {
+			GroupName string `xml:"GroupName"`
+		} `xml:"Group"`
+	} `xml:"CreateGroupResult"`
+}
+
+type ListGroupsResponse struct {
+	XMLName          xml.Name `xml:"ListGroupsResponse"`
+	ListGroupsResult struct {
+		Groups []struct {
+			GroupName string `xml:"GroupName"`
+		} `xml:"Groups>member"`
+	} `xml:"ListGroupsResult"`
+}
+
+// callIAMAPIAuthenticated sends an authenticated raw IAM API request using the
+// framework's JWT token. This is needed for custom extensions not in the AWS SDK
+// (like UpdateGroup with Disabled parameter).
+func callIAMAPIAuthenticated(_ *testing.T, framework *S3IAMTestFramework, action string, params url.Values) (*http.Response, error) {
+	params.Set("Action", action)
+
+	req, err := http.NewRequest(http.MethodPost, TestIAMEndpoint+"/",
+		strings.NewReader(params.Encode()))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	token, err := framework.generateSTSSessionToken("admin-user", "TestAdminRole", time.Hour, "", nil)
+	if err != nil {
+		return nil, err
+	}
+
+	client := &http.Client{
+		Timeout:   30 * time.Second,
+		Transport: &BearerTokenTransport{Token: token},
+	}
+	return client.Do(req)
+}
+
+// TestIAMGroupRawAPI tests group operations using raw HTTP IAM API calls,
+// verifying XML response format for group operations.
+func TestIAMGroupRawAPI(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+	if !isSeaweedFSRunning(t) {
+		t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
+	}
+
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	groupName := "test-raw-api-group"
+
+	t.Run("create_group_raw", func(t *testing.T) {
+		resp, err := callIAMAPIAuthenticated(t, framework, "CreateGroup", url.Values{
+			"GroupName": {groupName},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		var createResp CreateGroupResponse
+		err = xml.Unmarshal(body, &createResp)
+		require.NoError(t, err)
+		assert.Equal(t, groupName, createResp.CreateGroupResult.Group.GroupName)
+	})
+
+	t.Run("list_groups_raw", func(t *testing.T) {
+		resp, err := callIAMAPIAuthenticated(t, framework, "ListGroups", url.Values{})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		var listResp ListGroupsResponse
+		err = xml.Unmarshal(body, &listResp)
+		require.NoError(t, err)
+
+		found := false
+		for _, g := range listResp.ListGroupsResult.Groups {
+			if g.GroupName == groupName {
+				found = true
+				break
+			}
+		}
+		assert.True(t, found, "Created group should appear in raw ListGroups")
+	})
+
+	t.Run("delete_group_raw", func(t *testing.T) {
+		resp, err := callIAMAPIAuthenticated(t, framework, "DeleteGroup", url.Values{
+			"GroupName": {groupName},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+	})
+}
+
+// TestIAMGroupInlinePolicy tests group inline policy operations:
+// PutGroupPolicy, GetGroupPolicy, ListGroupPolicies, DeleteGroupPolicy
+func TestIAMGroupInlinePolicy(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	// Skip if running against embedded IAM (which returns NotImplemented for group inline policies)
+	_, probeErr := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{GroupName: aws.String("probe-group-inline-support")})
+	if probeErr != nil {
+		if awsErr, ok := probeErr.(awserr.Error); ok && awsErr.Code() == "NotImplemented" {
+			t.Skip("Skipping: group inline policies not supported in embedded IAM mode")
+		}
+	}
+	require.NoError(t, err)
+
+	groupName := "test-group-inline-policy"
+	policyName := "TestInlinePolicy"
+	_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
+		GroupName: aws.String(groupName),
+	})
+	require.NoError(t, err)
+	defer func() {
+		// Clean up inline policies before deleting the group
+		iamClient.DeleteGroupPolicy(&iam.DeleteGroupPolicyInput{
+			GroupName:  aws.String(groupName),
+			PolicyName: aws.String(policyName),
+		})
+		iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)})
+	}()
+
+	t.Run("list_empty", func(t *testing.T) {
+		resp, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		assert.Empty(t, resp.PolicyNames)
+		assert.False(t, *resp.IsTruncated)
+	})
+
+	policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}`
+
+	t.Run("put_group_policy", func(t *testing.T) {
+		_, err := iamClient.PutGroupPolicy(&iam.PutGroupPolicyInput{
+			GroupName:      aws.String(groupName),
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(policyDoc),
+		})
+		require.NoError(t, err)
+	})
+
+	t.Run("list_after_put", func(t *testing.T) {
+		resp, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		assert.NotEmpty(t, resp.PolicyNames)
+	})
+
+	t.Run("get_group_policy", func(t *testing.T) {
+		resp, err := iamClient.GetGroupPolicy(&iam.GetGroupPolicyInput{
+			GroupName:  aws.String(groupName),
+			PolicyName: aws.String(policyName),
+		})
+		require.NoError(t, err)
+		assert.Equal(t, groupName, *resp.GroupName)
+		assert.Equal(t, policyName, *resp.PolicyName)
+		assert.Contains(t, *resp.PolicyDocument, "s3:GetObject")
+	})
+
+	t.Run("delete_group_policy", func(t *testing.T) {
+		_, err := iamClient.DeleteGroupPolicy(&iam.DeleteGroupPolicyInput{
+			GroupName:  aws.String(groupName),
+			PolicyName: aws.String(policyName),
+		})
+		require.NoError(t, err)
+	})
+
+	t.Run("list_after_delete", func(t *testing.T) {
+		resp, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		assert.Empty(t, resp.PolicyNames)
+	})
+
+	t.Run("nonexistent_group", func(t *testing.T) {
+		_, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
+			GroupName: aws.String("nonexistent-group-for-policies"),
+		})
+		require.Error(t, err)
+		awsErr, ok := err.(awserr.Error)
+		require.True(t, ok, "Expected AWS error type")
+		assert.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
+	})
+}
+
+// createS3Client creates an S3 client with static credentials
+func createS3Client(t *testing.T, accessKey, secretKey string) *s3.S3 {
+	sess, err := session.NewSession(&aws.Config{
+		Region:           aws.String("us-east-1"),
+		Endpoint:         aws.String(TestS3Endpoint),
+		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, ""),
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+	})
+	require.NoError(t, err)
+	return s3.New(sess)
+}

test/s3/iam/s3_iam_inline_policy_condition_test.go

@@ -0,0 +1,309 @@
+package iam
+
+import (
+	"fmt"
+	mathrand "math/rand"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// uniqueResourceSuffix returns a lowercased per-test, per-invocation suffix
+// safe for use in IAM resource and S3 bucket names. Avoids EntityAlreadyExists
+// / BucketAlreadyExists collisions when integration jobs retry or run in
+// parallel against a shared stack.
+func uniqueResourceSuffix(t *testing.T) string {
+	name := strings.ToLower(t.Name())
+	name = strings.ReplaceAll(name, "/", "-")
+	name = strings.ReplaceAll(name, "_", "-")
+	return fmt.Sprintf("%s-%d", name, mathrand.Intn(10000))
+}
+
+// isAccessDenied returns true when err is an AWS error with code "AccessDenied".
+// Used to gate deny-path polling so transient setup errors don't end the
+// Eventually loop prematurely.
+func isAccessDenied(err error) bool {
+	if err == nil {
+		return false
+	}
+	awsErr, ok := err.(awserr.Error)
+	return ok && awsErr.Code() == "AccessDenied"
+}
+
+// localSourceAllowCIDRs lists every address a loopback-targeted test client may
+// present as aws:SourceIp. The SDK reaches the server over localhost, but
+// depending on resolver order and host routing the source the server observes
+// can be IPv4 loopback, IPv6 loopback, or one of the host's RFC1918 addresses
+// (CI runners advertise the S3 endpoint on a 10.x interface). An allow policy
+// meant to match "this local client" must cover all of them or the
+// positive-condition assertion flakes.
+var localSourceAllowCIDRs = []string{
+	"127.0.0.0/8",
+	"::1/128",
+	"10.0.0.0/8",
+	"172.16.0.0/12",
+	"192.168.0.0/16",
+}
+
+// sourceIpAllowPolicy builds an Allow policy for s3:* on bucketName gated by an
+// aws:SourceIp IpAddress condition over the given CIDRs.
+func sourceIpAllowPolicy(bucketName string, cidrs ...string) string {
+	quoted := make([]string, len(cidrs))
+	for i, c := range cidrs {
+		quoted[i] = `"` + c + `"`
+	}
+	return `{
+		"Version":"2012-10-17",
+		"Statement":[{
+			"Effect":"Allow",
+			"Action":"s3:*",
+			"Resource":["arn:aws:s3:::` + bucketName + `","arn:aws:s3:::` + bucketName + `/*"],
+			"Condition":{"IpAddress":{"aws:SourceIp":[` + strings.Join(quoted, ",") + `]}}
+		}]
+	}`
+}
+
+// TestIAMUserInlinePolicySourceIpCondition verifies that an aws:SourceIp condition
+// on a user inline policy is honored. Tests run against a local server, so a
+// policy that only allows a non-local CIDR must deny the request, and a policy
+// that allows the local client's address (see localSourceAllowCIDRs) must allow it.
+func TestIAMUserInlinePolicySourceIpCondition(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	suffix := uniqueResourceSuffix(t)
+	userName := "user-" + suffix
+	policyName := "policy-" + suffix
+	bucketName := "bucket-" + suffix
+
+	_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
+	require.NoError(t, err)
+
+	keyResp, err := iamClient.CreateAccessKey(&iam.CreateAccessKeyInput{
+		UserName: aws.String(userName),
+	})
+	require.NoError(t, err)
+	accessKeyId := *keyResp.AccessKey.AccessKeyId
+	secretKey := *keyResp.AccessKey.SecretAccessKey
+
+	userS3 := createS3Client(t, accessKeyId, secretKey)
+
+	adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+	require.NoError(t, framework.CreateBucketWithCleanup(adminS3, bucketName))
+
+	t.Cleanup(func() {
+		if _, err := iamClient.DeleteUserPolicy(&iam.DeleteUserPolicyInput{
+			UserName:   aws.String(userName),
+			PolicyName: aws.String(policyName),
+		}); err != nil {
+			t.Logf("cleanup: failed to delete user policy: %v", err)
+		}
+		if _, err := iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{
+			UserName:    aws.String(userName),
+			AccessKeyId: keyResp.AccessKey.AccessKeyId,
+		}); err != nil {
+			t.Logf("cleanup: failed to delete access key: %v", err)
+		}
+		if _, err := iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)}); err != nil {
+			t.Logf("cleanup: failed to delete user: %v", err)
+		}
+	})
+
+	t.Run("denies_when_source_ip_does_not_match", func(t *testing.T) {
+		// SourceIp 198.51.100.0/24 is RFC5737 TEST-NET-2; the test client is on
+		// the local host, so the condition must fail and the action be denied.
+		_, err = iamClient.PutUserPolicy(&iam.PutUserPolicyInput{
+			UserName:       aws.String(userName),
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(sourceIpAllowPolicy(bucketName, "198.51.100.0/24")),
+		})
+		require.NoError(t, err)
+
+		var lastErr error
+		require.Eventually(t, func() bool {
+			_, lastErr = userS3.PutObject(&s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String("denied.txt"),
+				Body:   aws.ReadSeekCloser(strings.NewReader("nope")),
+			})
+			return isAccessDenied(lastErr)
+		}, 10*time.Second, 500*time.Millisecond,
+			"PutObject must be denied with AccessDenied when aws:SourceIp condition does not match (last error: %v)", lastErr)
+	})
+
+	t.Run("allows_when_source_ip_matches", func(t *testing.T) {
+		_, err = iamClient.PutUserPolicy(&iam.PutUserPolicyInput{
+			UserName:       aws.String(userName),
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(sourceIpAllowPolicy(bucketName, localSourceAllowCIDRs...)),
+		})
+		require.NoError(t, err)
+
+		require.Eventually(t, func() bool {
+			_, putErr := userS3.PutObject(&s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String("allowed.txt"),
+				Body:   aws.ReadSeekCloser(strings.NewReader("ok")),
+			})
+			if putErr != nil {
+				t.Logf("allow attempt denied (source IP not in %v?): %v", localSourceAllowCIDRs, putErr)
+			}
+			return putErr == nil
+		}, 10*time.Second, 500*time.Millisecond,
+			"PutObject must succeed when aws:SourceIp condition matches the local client address")
+	})
+}
+
+// TestIAMGroupInlinePolicyEnforcement verifies that PutGroupPolicy is supported
+// and that the resulting inline policy is enforced for members of the group,
+// including its Condition block.
+func TestIAMGroupInlinePolicyEnforcement(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	suffix := uniqueResourceSuffix(t)
+	groupName := "group-" + suffix
+	userName := "user-" + suffix
+	policyName := "policy-" + suffix
+	bucketName := "bucket-" + suffix
+
+	_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
+	require.NoError(t, err)
+
+	keyResp, err := iamClient.CreateAccessKey(&iam.CreateAccessKeyInput{
+		UserName: aws.String(userName),
+	})
+	require.NoError(t, err)
+
+	_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(groupName)})
+	require.NoError(t, err)
+
+	_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
+		GroupName: aws.String(groupName),
+		UserName:  aws.String(userName),
+	})
+	require.NoError(t, err)
+
+	userS3 := createS3Client(t, *keyResp.AccessKey.AccessKeyId, *keyResp.AccessKey.SecretAccessKey)
+
+	adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+	require.NoError(t, framework.CreateBucketWithCleanup(adminS3, bucketName))
+
+	t.Cleanup(func() {
+		if _, err := iamClient.DeleteGroupPolicy(&iam.DeleteGroupPolicyInput{
+			GroupName:  aws.String(groupName),
+			PolicyName: aws.String(policyName),
+		}); err != nil {
+			t.Logf("cleanup: failed to delete group policy: %v", err)
+		}
+		if _, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
+			GroupName: aws.String(groupName),
+			UserName:  aws.String(userName),
+		}); err != nil {
+			t.Logf("cleanup: failed to remove user from group: %v", err)
+		}
+		if _, err := iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{
+			UserName:    aws.String(userName),
+			AccessKeyId: keyResp.AccessKey.AccessKeyId,
+		}); err != nil {
+			t.Logf("cleanup: failed to delete access key: %v", err)
+		}
+		if _, err := iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)}); err != nil {
+			t.Logf("cleanup: failed to delete user: %v", err)
+		}
+		if _, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)}); err != nil {
+			t.Logf("cleanup: failed to delete group: %v", err)
+		}
+	})
+
+	allowDoc := sourceIpAllowPolicy(bucketName, localSourceAllowCIDRs...)
+	denyDoc := sourceIpAllowPolicy(bucketName, "198.51.100.0/24")
+
+	t.Run("crud_round_trip", func(t *testing.T) {
+		_, err := iamClient.PutGroupPolicy(&iam.PutGroupPolicyInput{
+			GroupName:      aws.String(groupName),
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(allowDoc),
+		})
+		require.NoError(t, err, "PutGroupPolicy must succeed (no longer NotImplemented)")
+
+		listResp, err := iamClient.ListGroupPolicies(&iam.ListGroupPoliciesInput{
+			GroupName: aws.String(groupName),
+		})
+		require.NoError(t, err)
+		found := false
+		for _, name := range listResp.PolicyNames {
+			if name != nil && *name == policyName {
+				found = true
+				break
+			}
+		}
+		assert.True(t, found, "ListGroupPolicies must return the freshly added policy")
+
+		getResp, err := iamClient.GetGroupPolicy(&iam.GetGroupPolicyInput{
+			GroupName:  aws.String(groupName),
+			PolicyName: aws.String(policyName),
+		})
+		require.NoError(t, err)
+		require.NotNil(t, getResp.PolicyDocument)
+		assert.Contains(t, *getResp.PolicyDocument, "aws:SourceIp",
+			"GetGroupPolicy must round-trip the Condition block")
+	})
+
+	t.Run("enforces_allow_when_condition_matches", func(t *testing.T) {
+		_, err := iamClient.PutGroupPolicy(&iam.PutGroupPolicyInput{
+			GroupName:      aws.String(groupName),
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(allowDoc),
+		})
+		require.NoError(t, err)
+
+		require.Eventually(t, func() bool {
+			_, putErr := userS3.PutObject(&s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String("group-allowed.txt"),
+				Body:   aws.ReadSeekCloser(strings.NewReader("ok")),
+			})
+			if putErr != nil {
+				t.Logf("allow attempt denied (source IP not in %v?): %v", localSourceAllowCIDRs, putErr)
+			}
+			return putErr == nil
+		}, 10*time.Second, 500*time.Millisecond,
+			"group member must be allowed when the group policy condition matches")
+	})
+
+	t.Run("enforces_deny_when_condition_does_not_match", func(t *testing.T) {
+		_, err := iamClient.PutGroupPolicy(&iam.PutGroupPolicyInput{
+			GroupName:      aws.String(groupName),
+			PolicyName:     aws.String(policyName),
+			PolicyDocument: aws.String(denyDoc),
+		})
+		require.NoError(t, err)
+
+		var lastErr error
+		require.Eventually(t, func() bool {
+			_, lastErr = userS3.PutObject(&s3.PutObjectInput{
+				Bucket: aws.String(bucketName),
+				Key:    aws.String("group-denied.txt"),
+				Body:   aws.ReadSeekCloser(strings.NewReader("nope")),
+			})
+			return isAccessDenied(lastErr)
+		}, 10*time.Second, 500*time.Millisecond,
+			"group member must be denied with AccessDenied when the group policy condition does not match (last error: %v)", lastErr)
+	})
+}

test/s3/iam/s3_iam_integration_test.go

@@ -0,0 +1,479 @@
+package iam
+
+import (
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testEndpoint   = "http://localhost:8333"
+	testRegion     = "us-west-2"
+	testBucket     = "test-iam-bucket"
+	testObjectKey  = "test-object.txt"
+	testObjectData = "Hello, SeaweedFS IAM Integration!"
+)
+
+// TestS3IAMAuthentication tests S3 API authentication with IAM JWT tokens
+func TestS3IAMAuthentication(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	t.Run("valid_jwt_token_authentication", func(t *testing.T) {
+		// Create S3 client with valid JWT token
+		s3Client, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+		require.NoError(t, err)
+
+		// Test bucket operations
+		err = framework.CreateBucket(s3Client, testBucket)
+		require.NoError(t, err)
+
+		// Verify bucket exists
+		buckets, err := s3Client.ListBuckets(&s3.ListBucketsInput{})
+		require.NoError(t, err)
+
+		found := false
+		for _, bucket := range buckets.Buckets {
+			if *bucket.Name == testBucket {
+				found = true
+				break
+			}
+		}
+		assert.True(t, found, "Created bucket should be listed")
+	})
+
+	t.Run("invalid_jwt_token_authentication", func(t *testing.T) {
+		// Create S3 client with invalid JWT token
+		s3Client, err := framework.CreateS3ClientWithInvalidJWT()
+		require.NoError(t, err)
+
+		// Attempt bucket operations - should fail
+		err = framework.CreateBucket(s3Client, testBucket+"-invalid")
+		require.Error(t, err)
+
+		// Verify it's an access denied error
+		if awsErr, ok := err.(awserr.Error); ok {
+			assert.Equal(t, "AccessDenied", awsErr.Code())
+		} else {
+			t.Error("Expected AWS error with AccessDenied code")
+		}
+	})
+
+	t.Run("expired_jwt_token_authentication", func(t *testing.T) {
+		// Create S3 client with expired JWT token
+		s3Client, err := framework.CreateS3ClientWithExpiredJWT("expired-user", "TestAdminRole")
+		require.NoError(t, err)
+
+		// Attempt bucket operations - should fail
+		err = framework.CreateBucket(s3Client, testBucket+"-expired")
+		require.Error(t, err)
+
+		// Verify it's an access denied error
+		if awsErr, ok := err.(awserr.Error); ok {
+			assert.Equal(t, "AccessDenied", awsErr.Code())
+		} else {
+			t.Error("Expected AWS error with AccessDenied code")
+		}
+	})
+}
+
+// TestS3IAMPolicyEnforcement tests policy enforcement for different S3 operations
+// NOTE: This test is currently skipped because the IAM framework needs to set up role policies
+// The test assumes TestReadOnlyRole and TestWriteOnlyRole are configured in the IAM system,
+// but these roles and their associated policies are not yet being created during test setup.
+// TODO: Implement setupIAMRoles() to create roles with proper policies before running this test.
+// TestS3IAMPolicyEnforcement tests policy enforcement for different S3 operations
+// NOTE: This test is skipped because the IAM framework needs to set up role policies.
+// The test assumes TestReadOnlyRole and TestWriteOnlyRole are configured in the IAM system,
+// but these roles and their associated policies are not yet being created during test setup.
+// TODO: Implement setupIAMRoles() to create roles with proper policies before running this test.
+func TestS3IAMPolicyEnforcement(t *testing.T) {
+	t.Skip("Skipping: Requires IAM role and policy setup - TestReadOnlyRole and TestWriteOnlyRole policies not configured")
+}
+
+// TestS3IAMSessionExpiration tests session expiration handling
+func TestS3IAMSessionExpiration(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	t.Run("session_expiration_enforcement", func(t *testing.T) {
+		// Create S3 client with valid JWT token
+		s3Client, err := framework.CreateS3ClientWithJWT("session-user", "TestAdminRole")
+		require.NoError(t, err)
+
+		// Initially should work
+		err = framework.CreateBucket(s3Client, testBucket+"-session")
+		require.NoError(t, err)
+
+		// Create S3 client with expired JWT token
+		expiredClient, err := framework.CreateS3ClientWithExpiredJWT("session-user", "TestAdminRole")
+		require.NoError(t, err)
+
+		// Now operations should fail with expired token
+		err = framework.CreateBucket(expiredClient, testBucket+"-session-expired")
+		require.Error(t, err)
+		if awsErr, ok := err.(awserr.Error); ok {
+			assert.Equal(t, "AccessDenied", awsErr.Code())
+		}
+
+		// Cleanup the successful bucket
+		adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+		require.NoError(t, err)
+
+		_, err = adminClient.DeleteBucket(&s3.DeleteBucketInput{
+			Bucket: aws.String(testBucket + "-session"),
+		})
+		require.NoError(t, err)
+	})
+}
+
+// TestS3IAMMultipartUploadPolicyEnforcement tests multipart upload with IAM policies
+func TestS3IAMMultipartUploadPolicyEnforcement(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	// Setup test bucket with admin client
+	adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	err = framework.CreateBucket(adminClient, testBucket)
+	require.NoError(t, err)
+
+	// Set bucket policy to deny multipart uploads from read-only users
+	bucketPolicy := fmt.Sprintf(`{
+		"Version": "2012-10-17",
+		"Statement": [
+			{
+				"Effect": "Allow",
+				"Principal": "*",
+				"Action": "s3:*",
+				"Resource": ["arn:aws:s3:::%s", "arn:aws:s3:::%s/*"]
+			},
+			{
+				"Effect": "Deny",
+				"Principal": "arn:aws:sts::123456789012:assumed-role/TestReadOnlyRole/read-user",
+				"Action": ["s3:PutObject", "s3:CreateMultipartUpload", "s3:AbortMultipartUpload", "s3:CompleteMultipartUpload", "s3:ListMultipartUploadParts"],
+				"Resource": "arn:aws:s3:::%s/*"
+			}
+		]
+	}`, testBucket, testBucket, testBucket)
+
+	_, err = adminClient.PutBucketPolicy(&s3.PutBucketPolicyInput{
+		Bucket: aws.String(testBucket),
+		Policy: aws.String(bucketPolicy),
+	})
+	require.NoError(t, err)
+
+	t.Run("multipart_upload_with_write_permissions", func(t *testing.T) {
+		// Create S3 client with admin role (has multipart permissions)
+		s3Client := adminClient
+
+		// Initiate multipart upload
+		multipartKey := "large-test-file.txt"
+		initResult, err := s3Client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
+			Bucket: aws.String(testBucket),
+			Key:    aws.String(multipartKey),
+		})
+		require.NoError(t, err)
+
+		uploadId := initResult.UploadId
+
+		// Upload a part
+		partNumber := int64(1)
+		partData := strings.Repeat("Test data for multipart upload. ", 1000) // ~30KB
+
+		uploadResult, err := s3Client.UploadPart(&s3.UploadPartInput{
+			Bucket:     aws.String(testBucket),
+			Key:        aws.String(multipartKey),
+			PartNumber: aws.Int64(partNumber),
+			UploadId:   uploadId,
+			Body:       strings.NewReader(partData),
+		})
+		require.NoError(t, err)
+
+		// Complete multipart upload
+		_, err = s3Client.CompleteMultipartUpload(&s3.CompleteMultipartUploadInput{
+			Bucket:   aws.String(testBucket),
+			Key:      aws.String(multipartKey),
+			UploadId: uploadId,
+			MultipartUpload: &s3.CompletedMultipartUpload{
+				Parts: []*s3.CompletedPart{
+					{
+						ETag:       uploadResult.ETag,
+						PartNumber: aws.Int64(partNumber),
+					},
+				},
+			},
+		})
+		require.NoError(t, err)
+
+		// Verify object was created
+		result, err := s3Client.GetObject(&s3.GetObjectInput{
+			Bucket: aws.String(testBucket),
+			Key:    aws.String(multipartKey),
+		})
+		require.NoError(t, err)
+
+		data, err := io.ReadAll(result.Body)
+		require.NoError(t, err)
+		assert.Equal(t, partData, string(data))
+		result.Body.Close()
+
+		// Cleanup
+		_, err = s3Client.DeleteObject(&s3.DeleteObjectInput{
+			Bucket: aws.String(testBucket),
+			Key:    aws.String(multipartKey),
+		})
+		require.NoError(t, err)
+	})
+
+	t.Run("multipart_upload_denied_for_read_only", func(t *testing.T) {
+		// Create S3 client with read-only role
+		readOnlyClient, err := framework.CreateS3ClientWithJWT("read-user", "TestReadOnlyRole")
+		require.NoError(t, err)
+
+		// Attempt to initiate multipart upload - should fail due to bucket policy
+		multipartKey := "denied-multipart-file.txt"
+		_, err = readOnlyClient.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
+			Bucket: aws.String(testBucket),
+			Key:    aws.String(multipartKey),
+		})
+		require.Error(t, err)
+		if awsErr, ok := err.(awserr.Error); ok {
+			assert.Equal(t, "AccessDenied", awsErr.Code())
+		}
+	})
+
+	// Cleanup
+	_, err = adminClient.DeleteBucket(&s3.DeleteBucketInput{
+		Bucket: aws.String(testBucket),
+	})
+	require.NoError(t, err)
+}
+
+// TestS3IAMBucketPolicyIntegration tests bucket policy integration with IAM
+func TestS3IAMBucketPolicyIntegration(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	// Setup test bucket with admin client
+	adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	// Use unique bucket name to avoid collection conflicts
+	bucketName := framework.GenerateUniqueBucketName("test-iam-bucket-policy")
+	err = framework.CreateBucket(adminClient, bucketName)
+	require.NoError(t, err)
+	defer adminClient.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
+
+	t.Run("bucket_policy_allows_public_read", func(t *testing.T) {
+		testObjectKey := "test-object.txt"
+		testObjectData := "test data for public read"
+
+		// Set bucket policy to allow public read access
+		bucketPolicy := fmt.Sprintf(`{
+			"Version": "2012-10-17",
+			"Statement": [
+				{
+					"Sid": "PublicReadGetObject",
+					"Effect": "Allow",
+					"Principal": "*",
+					"Action": ["s3:GetObject"],
+					"Resource": ["arn:aws:s3:::%s/*"]
+				}
+			]
+		}`, bucketName)
+
+		_, err = adminClient.PutBucketPolicy(&s3.PutBucketPolicyInput{
+			Bucket: aws.String(bucketName),
+			Policy: aws.String(bucketPolicy),
+		})
+		require.NoError(t, err)
+
+		// Put test object
+		_, err = adminClient.PutObject(&s3.PutObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(testObjectKey),
+			Body:   strings.NewReader(testObjectData),
+		})
+		require.NoError(t, err)
+
+		// Test with read-only client - should now be allowed due to bucket policy
+		readOnlyClient, err := framework.CreateS3ClientWithJWT("read-user", "TestReadOnlyRole")
+		require.NoError(t, err)
+
+		result, err := readOnlyClient.GetObject(&s3.GetObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(testObjectKey),
+		})
+		require.NoError(t, err)
+
+		data, err := io.ReadAll(result.Body)
+		require.NoError(t, err)
+		assert.Equal(t, testObjectData, string(data))
+		result.Body.Close()
+
+		// Clean up object and bucket policy after this test
+		_, err = adminClient.DeleteObject(&s3.DeleteObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(testObjectKey),
+		})
+		require.NoError(t, err)
+
+		_, err = adminClient.DeleteBucketPolicy(&s3.DeleteBucketPolicyInput{
+			Bucket: aws.String(bucketName),
+		})
+		require.NoError(t, err)
+	})
+
+	t.Run("bucket_policy_denies_specific_action", func(t *testing.T) {
+		// Set bucket policy to deny delete operations
+		bucketPolicy := fmt.Sprintf(`{
+			"Version": "2012-10-17",
+			"Statement": [
+				{
+					"Sid": "DenyDelete",
+					"Effect": "Deny",
+					"Principal": "*",
+					"Action": ["s3:DeleteObject"],
+					"Resource": ["arn:aws:s3:::%s/*"]
+				}
+			]
+		}`, bucketName)
+
+		_, err = adminClient.PutBucketPolicy(&s3.PutBucketPolicyInput{
+			Bucket: aws.String(bucketName),
+			Policy: aws.String(bucketPolicy),
+		})
+		require.NoError(t, err)
+
+		// Verify that the bucket policy was stored successfully by retrieving it
+		policyResult, err := adminClient.GetBucketPolicy(&s3.GetBucketPolicyInput{
+			Bucket: aws.String(bucketName),
+		})
+		require.NoError(t, err)
+		assert.Contains(t, *policyResult.Policy, "s3:DeleteObject")
+		assert.Contains(t, *policyResult.Policy, "Deny")
+
+		// NOTE: Enforcement test is commented out due to known architectural limitation:
+		//
+		// KNOWN LIMITATION: DeleteObject uses the coarse-grained ACTION_WRITE constant,
+		// which convertActionToS3Format maps to "s3:PutObject" (not "s3:DeleteObject").
+		// This means the policy engine evaluates the deny policy against "s3:PutObject",
+		// doesn't find a match, and allows the delete operation.
+		//
+		// TODO: Uncomment this test once the action mapping is refactored to use
+		// specific S3 action strings throughout the S3 API handlers.
+		// See: weed/s3api/s3api_bucket_policy_engine.go lines 135-146
+		//
+		// _, err = adminClient.DeleteObject(&s3.DeleteObjectInput{
+		// 	Bucket: aws.String(bucketName),
+		// 	Key:    aws.String(testObjectKey),
+		// })
+		// require.Error(t, err, "DeleteObject should be denied by the bucket policy")
+		// awsErr, ok := err.(awserr.Error)
+		// require.True(t, ok, "Error should be an awserr.Error")
+		// assert.Equal(t, "AccessDenied", awsErr.Code(), "Expected AccessDenied error code")
+
+		// Clean up bucket policy after this test
+		_, err = adminClient.DeleteBucketPolicy(&s3.DeleteBucketPolicyInput{
+			Bucket: aws.String(bucketName),
+		})
+		require.NoError(t, err)
+	})
+}
+
+// TestS3IAMContextualPolicyEnforcement tests context-aware policy enforcement
+func TestS3IAMContextualPolicyEnforcement(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	// This test would verify IP-based restrictions, time-based restrictions,
+	// and other context-aware policy conditions
+	// For now, we'll focus on the basic structure
+
+	t.Run("ip_based_policy_enforcement", func(t *testing.T) {
+		// IMPLEMENTATION NOTE: IP-based policy testing framework planned for future release
+		// Requirements:
+		// - Configure IAM policies with IpAddress/NotIpAddress conditions
+		// - Multi-container test setup with controlled source IP addresses
+		// - Test policy enforcement from allowed vs denied IP ranges
+		t.Skip("IP-based policy testing requires advanced network configuration and multi-container setup")
+	})
+
+	t.Run("time_based_policy_enforcement", func(t *testing.T) {
+		// IMPLEMENTATION NOTE: Time-based policy testing framework planned for future release
+		// Requirements:
+		// - Configure IAM policies with DateGreaterThan/DateLessThan conditions
+		// - Time manipulation capabilities for testing different time windows
+		// - Test policy enforcement during allowed vs restricted time periods
+		t.Skip("Time-based policy testing requires time manipulation capabilities")
+	})
+}
+
+// TestS3IAMPresignedURLIntegration tests presigned URL generation with IAM
+func TestS3IAMPresignedURLIntegration(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	// Setup test bucket with admin client
+	adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	// Use unique bucket name to avoid conflicts with other tests
+	bucketName := framework.GenerateUniqueBucketName("test-iam-presigned")
+	err = framework.CreateBucket(adminClient, bucketName)
+	require.NoError(t, err)
+
+	// Put test object
+	_, err = adminClient.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(testObjectKey),
+		Body:   strings.NewReader(testObjectData),
+	})
+	require.NoError(t, err)
+
+	t.Run("presigned_url_generation_and_usage", func(t *testing.T) {
+		// ARCHITECTURAL NOTE: AWS SDK presigned URLs are incompatible with JWT Bearer authentication
+		//
+		// AWS SDK presigned URLs use AWS Signature Version 4 (SigV4) which requires:
+		// - Access Key ID and Secret Access Key for signing
+		// - Query parameter-based authentication in the URL
+		//
+		// SeaweedFS JWT authentication uses:
+		// - Bearer tokens in the Authorization header
+		// - Stateless JWT validation without AWS-style signing
+		//
+		// RECOMMENDATION: For JWT-authenticated applications, use direct API calls
+		// with Bearer tokens rather than presigned URLs.
+
+		// Test direct object access with JWT Bearer token (recommended approach)
+		_, err := adminClient.GetObject(&s3.GetObjectInput{
+			Bucket: aws.String(bucketName),
+			Key:    aws.String(testObjectKey),
+		})
+		require.NoError(t, err, "Direct object access with JWT Bearer token works correctly")
+
+		t.Log("JWT Bearer token authentication confirmed working for direct S3 API calls")
+		t.Log("Note: Presigned URLs are not supported with JWT Bearer authentication by design")
+	})
+
+	// Cleanup
+	_, err = adminClient.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String(testObjectKey),
+	})
+	require.NoError(t, err)
+
+	_, err = adminClient.DeleteBucket(&s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+}

test/s3/iam/s3_keycloak_integration_test.go

@@ -0,0 +1,307 @@
+package iam
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testKeycloakBucket = "test-keycloak-bucket"
+)
+
+// TestKeycloakIntegrationAvailable checks if Keycloak is available for testing
+func TestKeycloakIntegrationAvailable(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	if !framework.useKeycloak {
+		t.Skip("Keycloak not available, skipping integration tests")
+	}
+
+	// Test Keycloak health
+	assert.True(t, framework.useKeycloak, "Keycloak should be available")
+	assert.NotNil(t, framework.keycloakClient, "Keycloak client should be initialized")
+}
+
+// TestKeycloakAuthentication tests authentication flow with real Keycloak
+func TestKeycloakAuthentication(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	if !framework.useKeycloak {
+		t.Skip("Keycloak not available, skipping integration tests")
+	}
+
+	t.Run("admin_user_authentication", func(t *testing.T) {
+		// Test admin user authentication
+		token, err := framework.getKeycloakToken("admin-user")
+		require.NoError(t, err)
+		assert.NotEmpty(t, token, "JWT token should not be empty")
+
+		// Verify token can be used to create S3 client
+		s3Client, err := framework.CreateS3ClientWithKeycloakToken(token)
+		require.NoError(t, err)
+		assert.NotNil(t, s3Client, "S3 client should be created successfully")
+
+		// Test bucket operations with admin privileges
+		err = framework.CreateBucket(s3Client, testKeycloakBucket)
+		assert.NoError(t, err, "Admin user should be able to create buckets")
+
+		// Verify bucket exists
+		buckets, err := s3Client.ListBuckets(&s3.ListBucketsInput{})
+		require.NoError(t, err)
+
+		found := false
+		for _, bucket := range buckets.Buckets {
+			if *bucket.Name == testKeycloakBucket {
+				found = true
+				break
+			}
+		}
+		assert.True(t, found, "Created bucket should be listed")
+	})
+
+	t.Run("read_only_user_authentication", func(t *testing.T) {
+		// Test read-only user authentication
+		token, err := framework.getKeycloakToken("read-user")
+		require.NoError(t, err)
+		assert.NotEmpty(t, token, "JWT token should not be empty")
+
+		// Debug: decode token to verify it's for read-user
+		parts := strings.Split(token, ".")
+		if len(parts) >= 2 {
+			payload := parts[1]
+			// JWTs use URL-safe base64 encoding without padding (RFC 4648 §5)
+			decoded, err := base64.RawURLEncoding.DecodeString(payload)
+			if err == nil {
+				var claims map[string]interface{}
+				if json.Unmarshal(decoded, &claims) == nil {
+					t.Logf("Token username: %v", claims["preferred_username"])
+					t.Logf("Token roles: %v", claims["roles"])
+				}
+			}
+		}
+
+		// First test with direct HTTP request to verify OIDC authentication works
+		t.Logf("Testing with direct HTTP request...")
+		err = framework.TestKeycloakTokenDirectly(token)
+		require.NoError(t, err, "Direct HTTP test should succeed")
+
+		// Create S3 client with Keycloak token
+		s3Client, err := framework.CreateS3ClientWithKeycloakToken(token)
+		require.NoError(t, err)
+
+		// Test that read-only user can list buckets
+		t.Logf("Testing ListBuckets with AWS SDK...")
+		_, err = s3Client.ListBuckets(&s3.ListBucketsInput{})
+		assert.NoError(t, err, "Read-only user should be able to list buckets")
+
+		// Test that read-only user cannot create buckets
+		t.Logf("Testing CreateBucket with AWS SDK...")
+		err = framework.CreateBucket(s3Client, testKeycloakBucket+"-readonly")
+		assert.Error(t, err, "Read-only user should not be able to create buckets")
+	})
+
+	t.Run("invalid_user_authentication", func(t *testing.T) {
+		// Test authentication with invalid credentials
+		_, err := framework.keycloakClient.AuthenticateUser("invalid-user", "invalid-password")
+		assert.Error(t, err, "Authentication with invalid credentials should fail")
+	})
+}
+
+// TestKeycloakTokenExpiration tests JWT token expiration handling
+func TestKeycloakTokenExpiration(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	if !framework.useKeycloak {
+		t.Skip("Keycloak not available, skipping integration tests")
+	}
+
+	// Get a short-lived token (if Keycloak is configured for it)
+	// Use consistent password that matches Docker setup script logic: "adminuser123"
+	tokenResp, err := framework.keycloakClient.AuthenticateUser("admin-user", "adminuser123")
+	require.NoError(t, err)
+
+	// Verify token properties
+	assert.NotEmpty(t, tokenResp.AccessToken, "Access token should not be empty")
+	assert.Equal(t, "Bearer", tokenResp.TokenType, "Token type should be Bearer")
+	assert.Greater(t, tokenResp.ExpiresIn, 0, "Token should have expiration time")
+
+	// Test that token works initially
+	token, err := framework.getKeycloakToken("admin-user")
+	require.NoError(t, err)
+
+	s3Client, err := framework.CreateS3ClientWithKeycloakToken(token)
+	require.NoError(t, err)
+
+	_, err = s3Client.ListBuckets(&s3.ListBucketsInput{})
+	assert.NoError(t, err, "Fresh token should work for S3 operations")
+}
+
+// TestKeycloakRoleMapping tests role mapping from Keycloak to S3 policies
+func TestKeycloakRoleMapping(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	if !framework.useKeycloak {
+		t.Skip("Keycloak not available, skipping integration tests")
+	}
+
+	testCases := []struct {
+		username        string
+		expectedRole    string
+		canCreateBucket bool
+		canListBuckets  bool
+		description     string
+	}{
+		{
+			username:        "admin-user",
+			expectedRole:    "S3AdminRole",
+			canCreateBucket: true,
+			canListBuckets:  true,
+			description:     "Admin user should have full access",
+		},
+		{
+			username:        "read-user",
+			expectedRole:    "S3ReadOnlyRole",
+			canCreateBucket: false,
+			canListBuckets:  true,
+			description:     "Read-only user should have read-only access",
+		},
+		{
+			username:        "write-user",
+			expectedRole:    "S3ReadWriteRole",
+			canCreateBucket: true,
+			canListBuckets:  true,
+			description:     "Read-write user should have read-write access",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.username, func(t *testing.T) {
+			// Get Keycloak token for the user
+			token, err := framework.getKeycloakToken(tc.username)
+			require.NoError(t, err)
+
+			// Create S3 client with Keycloak token
+			s3Client, err := framework.CreateS3ClientWithKeycloakToken(token)
+			require.NoError(t, err, tc.description)
+
+			// Test list buckets permission
+			_, err = s3Client.ListBuckets(&s3.ListBucketsInput{})
+			if tc.canListBuckets {
+				assert.NoError(t, err, "%s should be able to list buckets", tc.username)
+			} else {
+				assert.Error(t, err, "%s should not be able to list buckets", tc.username)
+			}
+
+			// Test create bucket permission
+			testBucketName := testKeycloakBucket + "-" + tc.username
+			err = framework.CreateBucket(s3Client, testBucketName)
+			if tc.canCreateBucket {
+				assert.NoError(t, err, "%s should be able to create buckets", tc.username)
+			} else {
+				assert.Error(t, err, "%s should not be able to create buckets", tc.username)
+			}
+		})
+	}
+}
+
+// TestKeycloakS3Operations tests comprehensive S3 operations with Keycloak authentication
+func TestKeycloakS3Operations(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	if !framework.useKeycloak {
+		t.Skip("Keycloak not available, skipping integration tests")
+	}
+
+	// Use admin user for comprehensive testing
+	token, err := framework.getKeycloakToken("admin-user")
+	require.NoError(t, err)
+
+	s3Client, err := framework.CreateS3ClientWithKeycloakToken(token)
+	require.NoError(t, err)
+
+	bucketName := testKeycloakBucket + "-operations"
+
+	t.Run("bucket_lifecycle", func(t *testing.T) {
+		// Create bucket
+		err = framework.CreateBucket(s3Client, bucketName)
+		require.NoError(t, err, "Should be able to create bucket")
+
+		// Verify bucket exists
+		buckets, err := s3Client.ListBuckets(&s3.ListBucketsInput{})
+		require.NoError(t, err)
+
+		found := false
+		for _, bucket := range buckets.Buckets {
+			if *bucket.Name == bucketName {
+				found = true
+				break
+			}
+		}
+		assert.True(t, found, "Created bucket should be listed")
+	})
+
+	t.Run("object_operations", func(t *testing.T) {
+		objectKey := "test-object.txt"
+		objectContent := "Hello from Keycloak-authenticated SeaweedFS!"
+
+		// Put object
+		err = framework.PutTestObject(s3Client, bucketName, objectKey, objectContent)
+		require.NoError(t, err, "Should be able to put object")
+
+		// Get object
+		content, err := framework.GetTestObject(s3Client, bucketName, objectKey)
+		require.NoError(t, err, "Should be able to get object")
+		assert.Equal(t, objectContent, content, "Object content should match")
+
+		// List objects
+		objects, err := framework.ListTestObjects(s3Client, bucketName)
+		require.NoError(t, err, "Should be able to list objects")
+		assert.Contains(t, objects, objectKey, "Object should be listed")
+
+		// Delete object
+		err = framework.DeleteTestObject(s3Client, bucketName, objectKey)
+		assert.NoError(t, err, "Should be able to delete object")
+	})
+}
+
+// TestKeycloakFailover tests fallback to mock OIDC when Keycloak is unavailable
+func TestKeycloakFailover(t *testing.T) {
+	// Temporarily override Keycloak URL to simulate unavailability
+	originalURL := os.Getenv("KEYCLOAK_URL")
+	os.Setenv("KEYCLOAK_URL", "http://localhost:9999") // Non-existent service
+	defer func() {
+		if originalURL != "" {
+			os.Setenv("KEYCLOAK_URL", originalURL)
+		} else {
+			os.Unsetenv("KEYCLOAK_URL")
+		}
+	}()
+
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	// Should fall back to mock OIDC
+	assert.False(t, framework.useKeycloak, "Should fall back to mock OIDC when Keycloak is unavailable")
+	assert.Nil(t, framework.keycloakClient, "Keycloak client should not be initialized")
+	assert.NotNil(t, framework.mockOIDC, "Mock OIDC server should be initialized")
+
+	// Test that mock authentication still works
+	s3Client, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err, "Should be able to create S3 client with mock authentication")
+
+	// Basic operation should work
+	_, err = s3Client.ListBuckets(&s3.ListBucketsInput{})
+	// Note: This may still fail due to session store issues, but the client creation should work
+}

test/s3/iam/s3_policy_variables_test.go

@@ -0,0 +1,457 @@
+package iam
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestS3PolicyVariablesUsernameInResource tests ${aws:username} in resource paths
+func TestS3PolicyVariablesUsernameInResource(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	bucketName := framework.GenerateUniqueBucketName("test-policy-vars")
+	err = framework.CreateBucket(adminClient, bucketName)
+	require.NoError(t, err)
+	defer adminClient.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
+
+	// Policy with ${aws:username} in resource
+	bucketPolicy := fmt.Sprintf(`{
+		"Version": "2012-10-17",
+		"Statement": [{
+			"Effect": "Allow",
+			"Principal": "*",
+			"Action": ["s3:GetObject", "s3:PutObject"],
+			"Resource": ["arn:aws:s3:::%s/${aws:username}/*"]
+		}, {
+			"Sid": "DenyOthers",
+			"Effect": "Deny",
+			"Principal": "*",
+			"Action": ["s3:GetObject", "s3:PutObject"],
+			"NotResource": ["arn:aws:s3:::%s/${aws:username}/*"]
+		}]
+	}`, bucketName, bucketName)
+
+	_, err = adminClient.PutBucketPolicy(&s3.PutBucketPolicyInput{
+		Bucket: aws.String(bucketName),
+		Policy: aws.String(bucketPolicy),
+	})
+	require.NoError(t, err)
+
+	// Verify policy contains variable
+	policyResult, err := adminClient.GetBucketPolicy(&s3.GetBucketPolicyInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+	assert.Contains(t, *policyResult.Policy, "${aws:username}")
+
+	// Test Enforcement: Alice should be able to write to her own folder
+	aliceClient, err := framework.CreateS3ClientWithJWT("alice", "TestReadOnlyRole")
+	require.NoError(t, err)
+
+	_, err = aliceClient.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("alice/file.txt"),
+		Body:   nil, // Empty body is fine for this test
+	})
+	assert.NoError(t, err, "Alice should be allowed to put to alice/file.txt")
+
+	// Test Enforcement: Alice should NOT be able to write to bob's folder
+	_, err = aliceClient.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("bob/file.txt"),
+		Body:   nil,
+	})
+	assert.Error(t, err, "Alice should be denied put to bob/file.txt")
+}
+
+// TestS3PolicyVariablesUsernameInResourcePath tests ${aws:username} in Resource/NotResource
+// This validates that policy variables are correctly substituted in resource ARNs
+func TestS3PolicyVariablesUsernameInResourcePath(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	bucketName := framework.GenerateUniqueBucketName("test-policy-resource")
+	err = framework.CreateBucket(adminClient, bucketName)
+	require.NoError(t, err)
+	defer adminClient.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
+
+	// Policy with variable in resource ARN
+	bucketPolicy := fmt.Sprintf(`{
+		"Version": "2012-10-17",
+		"Statement": [{
+			"Effect": "Allow",
+			"Principal": "*",
+			"Action": ["s3:GetObject", "s3:PutObject"],
+			"Resource": ["arn:aws:s3:::%s/${aws:username}/*"]
+		}, {
+			"Sid": "DenyOthersFolders",
+			"Effect": "Deny",
+			"Principal": "*",
+			"Action": ["s3:GetObject", "s3:PutObject"],
+			"NotResource": ["arn:aws:s3:::%s/${aws:username}/*"]
+		}]
+	}`, bucketName, bucketName)
+
+	_, err = adminClient.PutBucketPolicy(&s3.PutBucketPolicyInput{
+		Bucket: aws.String(bucketName),
+		Policy: aws.String(bucketPolicy),
+	})
+	require.NoError(t, err)
+
+	policyResult, err := adminClient.GetBucketPolicy(&s3.GetBucketPolicyInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+	assert.Contains(t, *policyResult.Policy, "${aws:username}")
+
+	// Test Enforcement: Alice should be able to write to her own folder
+	aliceClient, err := framework.CreateS3ClientWithJWT("alice", "TestReadOnlyRole")
+	require.NoError(t, err)
+
+	_, err = aliceClient.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("alice/file.txt"),
+		Body:   nil, // Empty body is fine for this test
+	})
+	assert.NoError(t, err, "Alice should be allowed to put to alice/file.txt")
+
+	// Test Enforcement: Alice should NOT be able to write to bob's folder
+	_, err = aliceClient.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("bob/file.txt"),
+		Body:   nil,
+	})
+	assert.Error(t, err, "Alice should be denied put to bob/file.txt")
+}
+
+// TestS3PolicyVariablesJWTClaims tests ${jwt:*} variables
+func TestS3PolicyVariablesJWTClaims(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	bucketName := framework.GenerateUniqueBucketName("test-policy-jwt")
+	err = framework.CreateBucket(adminClient, bucketName)
+	require.NoError(t, err)
+	defer adminClient.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
+
+	// Policy with JWT claim variable
+	bucketPolicy := fmt.Sprintf(`{
+		"Version": "2012-10-17",
+		"Statement": [{
+			"Effect": "Allow",
+			"Principal": "*",
+			"Action": ["s3:GetObject"],
+			"Resource": ["arn:aws:s3:::%s/${jwt:preferred_username}/*"]
+		}]
+	}`, bucketName)
+
+	_, err = adminClient.PutBucketPolicy(&s3.PutBucketPolicyInput{
+		Bucket: aws.String(bucketName),
+		Policy: aws.String(bucketPolicy),
+	})
+	require.NoError(t, err)
+
+	policyResult, err := adminClient.GetBucketPolicy(&s3.GetBucketPolicyInput{
+		Bucket: aws.String(bucketName),
+	})
+	require.NoError(t, err)
+	assert.Contains(t, *policyResult.Policy, "jwt:preferred_username")
+}
+
+func TestS3PolicyVariablesUsernameIsolation(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	bucketName := framework.GenerateUniqueBucketName("test-isolation")
+	err = framework.CreateBucket(adminClient, bucketName)
+	require.NoError(t, err)
+	defer adminClient.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
+
+	bucketPolicy := fmt.Sprintf(`{
+		"Version": "2012-10-17",
+		"Statement": [{
+			"Sid": "AllowOwnFolder",
+			"Effect": "Allow",
+			"Principal": "*",
+			"Action": ["s3:GetObject", "s3:PutObject"],
+			"Resource": "arn:aws:s3:::%s/${aws:username}/*"
+		}, {
+			"Sid": "AllowListOwnPrefix",
+			"Effect": "Allow",
+			"Principal": "*",
+			"Action": "s3:ListBucket",
+			"Resource": "arn:aws:s3:::%s",
+			"Condition": {
+				"StringLike": {
+					"s3:prefix": ["${aws:username}/*", "${aws:username}"]
+				}
+			}
+		}, {
+			"Sid": "DenyOtherFolders",
+			"Effect": "Deny",
+			"Principal": "*",
+			"Action": ["s3:GetObject", "s3:PutObject"],
+			"NotResource": "arn:aws:s3:::%s/${aws:username}/*"
+		}, {
+			"Sid": "DenyListOtherPrefixes",
+			"Effect": "Deny",
+			"Principal": "*",
+			"Action": "s3:ListBucket",
+			"Resource": "arn:aws:s3:::%s",
+			"Condition": {
+				"StringNotLike": {
+					"s3:prefix": ["${aws:username}/*", "${aws:username}"]
+				}
+			}
+		}]
+	}`, bucketName, bucketName, bucketName, bucketName)
+
+	_, err = adminClient.PutBucketPolicy(&s3.PutBucketPolicyInput{
+		Bucket: aws.String(bucketName),
+		Policy: aws.String(bucketPolicy),
+	})
+	require.NoError(t, err)
+
+	// Wait for policy to propagate (fix race condition)
+	time.Sleep(2 * time.Second)
+
+	aliceClient, err := framework.CreateS3ClientWithJWT("alice", "TestReadOnlyRole")
+	require.NoError(t, err)
+
+	bobClient, err := framework.CreateS3ClientWithJWT("bob", "TestReadOnlyRole")
+	require.NoError(t, err)
+
+	_, err = aliceClient.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("alice/data.txt"),
+		Body:   strings.NewReader("Alice Private Data"),
+	})
+	assert.NoError(t, err, "Alice should be able to upload to her own folder")
+
+	_, err = aliceClient.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("bob/data.txt"),
+		Body:   strings.NewReader("Alice Intrusion"),
+	})
+	assert.Error(t, err, "Alice should be denied access to Bob's folder")
+
+	_, err = bobClient.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("bob/data.txt"),
+		Body:   strings.NewReader("Bob Private Data"),
+	})
+	assert.NoError(t, err, "Bob should be able to upload to his own folder")
+
+	_, err = bobClient.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("alice/data.txt"),
+	})
+	assert.Error(t, err, "Bob should be denied access to Alice's folder")
+
+	listAlice, err := aliceClient.ListObjects(&s3.ListObjectsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String("alice/"),
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(listAlice.Contents))
+
+	_, err = aliceClient.ListObjects(&s3.ListObjectsInput{
+		Bucket: aws.String(bucketName),
+		Prefix: aws.String("bob/"),
+	})
+	assert.Error(t, err, "Alice should be denied listing Bob's folder")
+}
+
+func TestS3PolicyVariablesAccountEnforcement(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	bucketName := framework.GenerateUniqueBucketName("test-account")
+	err = framework.CreateBucket(adminClient, bucketName)
+	require.NoError(t, err)
+	defer adminClient.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
+
+	bucketPolicy := fmt.Sprintf(`{
+		"Version": "2012-10-17",
+		"Statement": [{
+			"Effect": "Deny",
+			"Principal": "*",
+			"Action": ["s3:*"],
+			"Resource": ["arn:aws:s3:::%s/*"],
+			"Condition": {
+				"StringNotEquals": {
+					"aws:PrincipalAccount": ["999988887777"]
+				}
+			}
+		}, {
+			"Effect": "Allow",
+			"Principal": "*",
+			"Action": ["s3:*"],
+			"Resource": ["arn:aws:s3:::%s/*"]
+		}]
+	}`, bucketName, bucketName)
+
+	_, err = adminClient.PutBucketPolicy(&s3.PutBucketPolicyInput{
+		Bucket: aws.String(bucketName),
+		Policy: aws.String(bucketPolicy),
+	})
+	require.NoError(t, err)
+
+	authorizedClient, err := framework.CreateS3ClientWithCustomClaims("user1", "TestAdminRole", "999988887777", nil)
+	require.NoError(t, err)
+
+	unauthorizedClient, err := framework.CreateS3ClientWithCustomClaims("user2", "TestAdminRole", "111122223333", nil)
+	require.NoError(t, err)
+
+	_, err = authorizedClient.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("test.txt"),
+		Body:   strings.NewReader("Authorized Data"),
+	})
+	assert.NoError(t, err, "Authorized account should be able to upload")
+
+	_, err = unauthorizedClient.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("fail.txt"),
+		Body:   strings.NewReader("Unauthorized Data"),
+	})
+	assert.Error(t, err, "Unauthorized account should be denied")
+}
+
+func TestS3PolicyVariablesJWTPreferredUsername(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	bucketName := framework.GenerateUniqueBucketName("test-jwt-claim")
+	err = framework.CreateBucket(adminClient, bucketName)
+	require.NoError(t, err)
+	defer adminClient.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
+
+	bucketPolicy := fmt.Sprintf(`{
+		"Version": "2012-10-17",
+		"Statement": [{
+			"Sid": "AllowOwnFolder",
+			"Effect": "Allow",
+			"Principal": "*",
+			"Action": "s3:PutObject",
+			"Resource": "arn:aws:s3:::%s/${jwt:preferred_username}/*"
+		}, {
+			"Sid": "DenyOtherFolders",
+			"Effect": "Deny",
+			"Principal": "*",
+			"Action": "s3:PutObject",
+			"NotResource": "arn:aws:s3:::%s/${jwt:preferred_username}/*"
+		}]
+	}`, bucketName, bucketName)
+
+	_, err = adminClient.PutBucketPolicy(&s3.PutBucketPolicyInput{
+		Bucket: aws.String(bucketName),
+		Policy: aws.String(bucketPolicy),
+	})
+	require.NoError(t, err)
+
+	claims := map[string]interface{}{
+		"preferred_username": "jdoe",
+	}
+	client, err := framework.CreateS3ClientWithCustomClaims("jdoe", "TestReadOnlyRole", "", claims)
+	require.NoError(t, err)
+
+	_, err = client.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("jdoe/file.txt"),
+		Body:   strings.NewReader("JWT Claim Data"),
+	})
+	assert.NoError(t, err, "Should allow access based on jwt:preferred_username")
+
+	_, err = client.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("other/file.txt"),
+		Body:   strings.NewReader("JWT Claim Data"),
+	})
+	assert.Error(t, err, "Should deny access if prefix doesn't match jwt:preferred_username")
+}
+
+func TestS3PolicyVariablesLDAPClaims(t *testing.T) {
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
+	require.NoError(t, err)
+
+	bucketName := framework.GenerateUniqueBucketName("test-ldap-claim")
+	err = framework.CreateBucket(adminClient, bucketName)
+	require.NoError(t, err)
+	defer adminClient.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
+
+	bucketPolicy := fmt.Sprintf(`{
+		"Version": "2012-10-17",
+		"Statement": [{
+			"Effect": "Allow",
+			"Principal": "*",
+			"Action": ["s3:PutObject"],
+			"Resource": ["arn:aws:s3:::%s/${ldap:username}/*"]
+		}, {
+			"Effect": "Allow",
+			"Principal": "*",
+			"Action": ["s3:GetObject"],
+			"Resource": ["arn:aws:s3:::%s/*"],
+			"Condition": {
+				"StringEquals": {
+					"ldap:dn": ["cn=manager,dc=example,dc=org"]
+				}
+			}
+		}]
+	}`, bucketName, bucketName)
+
+	_, err = adminClient.PutBucketPolicy(&s3.PutBucketPolicyInput{
+		Bucket: aws.String(bucketName),
+		Policy: aws.String(bucketPolicy),
+	})
+	require.NoError(t, err)
+
+	claims := map[string]interface{}{
+		"ldap:username": "manager",
+		"ldap:dn":       "cn=manager,dc=example,dc=org",
+	}
+	client, err := framework.CreateS3ClientWithCustomClaims("manager", "TestReadOnlyRole", "", claims)
+	require.NoError(t, err)
+
+	_, err = client.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("manager/data.txt"),
+		Body:   strings.NewReader("LDAP Upload"),
+	})
+	assert.NoError(t, err)
+
+	_, err = client.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(bucketName),
+		Key:    aws.String("manager/data.txt"),
+	})
+	assert.NoError(t, err, "Should allow download based on ldap:dn condition")
+}

test/s3/iam/s3_service_account_security_test.go

@@ -0,0 +1,431 @@
+package iam
+
+// Integration tests for SeaweedFS service accounts.
+// These tests ensure comprehensive coverage of service account functionality
+// including security, access control, and expiration.
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestServiceAccountS3Access verifies that service accounts can actually
+// perform S3 operations using their credentials.
+func TestServiceAccountS3Access(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSeaweedFSRunning(t) {
+		t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
+	}
+
+	// Setup: Create a parent user
+	parentUserName := fmt.Sprintf("s3access-test-%d", time.Now().UnixNano())
+
+	// Create parent user
+	resp, err := callIAMAPI(t, "CreateUser", url.Values{
+		"UserName": {parentUserName},
+	})
+	require.NoError(t, err)
+	resp.Body.Close()
+	require.Equal(t, http.StatusOK, resp.StatusCode, "Failed to create parent user")
+
+	defer func() {
+		// Cleanup: delete parent user
+		callIAMAPI(t, "DeleteUser", url.Values{"UserName": {parentUserName}})
+	}()
+
+	// Create service account for the parent user
+	createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
+		"ParentUser":  {parentUserName},
+		"Description": {"S3 Access Test Service Account"},
+	})
+	require.NoError(t, err)
+	defer createResp.Body.Close()
+	require.Equal(t, http.StatusOK, createResp.StatusCode, "Failed to create service account")
+
+	body, err := io.ReadAll(createResp.Body)
+	require.NoError(t, err)
+
+	var saResp CreateServiceAccountResponse
+	err = xml.Unmarshal(body, &saResp)
+	require.NoError(t, err, "Failed to parse CreateServiceAccount response: %s", string(body))
+
+	accessKeyId := saResp.CreateServiceAccountResult.ServiceAccount.AccessKeyId
+	secretAccessKey := saResp.CreateServiceAccountResult.ServiceAccount.SecretAccessKey
+	saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId
+
+	require.NotEmpty(t, accessKeyId, "AccessKeyId should not be empty")
+	require.NotEmpty(t, secretAccessKey, "SecretAccessKey should not be empty")
+
+	defer func() {
+		// Cleanup: delete service account
+		callIAMAPI(t, "DeleteServiceAccount", url.Values{"ServiceAccountId": {saId}})
+	}()
+
+	t.Run("list_buckets_with_sa_credentials", func(t *testing.T) {
+		sess, err := session.NewSession(&aws.Config{
+			Region:   aws.String("us-east-1"),
+			Endpoint: aws.String(TestIAMEndpoint),
+			Credentials: credentials.NewStaticCredentials(
+				accessKeyId,
+				secretAccessKey,
+				"",
+			),
+			DisableSSL:       aws.Bool(true),
+			S3ForcePathStyle: aws.Bool(true),
+		})
+		require.NoError(t, err)
+
+		s3Client := s3.New(sess)
+		_, err = s3Client.ListBuckets(&s3.ListBucketsInput{})
+
+		// We don't necessarily expect success (depends on permissions),
+		// but we should NOT get InvalidAccessKeyId or SignatureDoesNotMatch
+		if err != nil {
+			if aerr, ok := err.(awserr.Error); ok {
+				assert.NotEqual(t, "InvalidAccessKeyId", aerr.Code(),
+					"Service account credentials should be recognized")
+				assert.NotEqual(t, "SignatureDoesNotMatch", aerr.Code(),
+					"Service account signature should be valid")
+			}
+		}
+	})
+
+	t.Run("create_bucket_with_sa_credentials", func(t *testing.T) {
+		sess, err := session.NewSession(&aws.Config{
+			Region:   aws.String("us-east-1"),
+			Endpoint: aws.String(TestIAMEndpoint),
+			Credentials: credentials.NewStaticCredentials(
+				accessKeyId,
+				secretAccessKey,
+				"",
+			),
+			DisableSSL:       aws.Bool(true),
+			S3ForcePathStyle: aws.Bool(true),
+		})
+		require.NoError(t, err)
+
+		s3Client := s3.New(sess)
+		bucketName := fmt.Sprintf("sa-test-bucket-%d", time.Now().UnixNano())
+
+		_, err = s3Client.CreateBucket(&s3.CreateBucketInput{
+			Bucket: aws.String(bucketName),
+		})
+
+		// Check that we get a proper response (success or AccessDenied based on policy)
+		// but NOT InvalidAccessKeyId
+		if err != nil {
+			if aerr, ok := err.(awserr.Error); ok {
+				assert.NotEqual(t, "InvalidAccessKeyId", aerr.Code(),
+					"Service account credentials should be recognized")
+				assert.NotEqual(t, "SignatureDoesNotMatch", aerr.Code(),
+					"Service account signature should be valid")
+			}
+		} else {
+			// Cleanup if bucket was created
+			defer s3Client.DeleteBucket(&s3.DeleteBucketInput{
+				Bucket: aws.String(bucketName),
+			})
+		}
+	})
+}
+
+// TestServiceAccountExpiration verifies that expired service accounts
+// are properly rejected.
+func TestServiceAccountExpiration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSeaweedFSRunning(t) {
+		t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
+	}
+
+	// Setup: Create a parent user
+	parentUserName := fmt.Sprintf("expiry-test-%d", time.Now().UnixNano())
+
+	resp, err := callIAMAPI(t, "CreateUser", url.Values{
+		"UserName": {parentUserName},
+	})
+	require.NoError(t, err)
+	resp.Body.Close()
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+
+	defer func() {
+		callIAMAPI(t, "DeleteUser", url.Values{"UserName": {parentUserName}})
+	}()
+
+	t.Run("reject_past_expiration", func(t *testing.T) {
+		// Try to create a service account with expiration in the past
+		pastExpiration := time.Now().Add(-1 * time.Hour).Unix()
+
+		createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
+			"ParentUser":  {parentUserName},
+			"Description": {"Should fail - past expiration"},
+			"Expiration":  {strconv.FormatInt(pastExpiration, 10)},
+		})
+		require.NoError(t, err)
+		defer createResp.Body.Close()
+
+		// Should fail because expiration is in the past
+		assert.NotEqual(t, http.StatusOK, createResp.StatusCode,
+			"Creating service account with past expiration should fail")
+	})
+
+	t.Run("accept_future_expiration", func(t *testing.T) {
+		// Create a service account with expiration in the future
+		futureExpiration := time.Now().Add(24 * time.Hour).Unix()
+
+		createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
+			"ParentUser":  {parentUserName},
+			"Description": {"Should succeed - future expiration"},
+			"Expiration":  {strconv.FormatInt(futureExpiration, 10)},
+		})
+		require.NoError(t, err)
+		defer createResp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, createResp.StatusCode,
+			"Creating service account with future expiration should succeed")
+
+		// Parse response to get service account ID for cleanup
+		if createResp.StatusCode == http.StatusOK {
+			body, err := io.ReadAll(createResp.Body)
+			require.NoError(t, err)
+
+			var saResp CreateServiceAccountResponse
+			require.NoError(t, xml.Unmarshal(body, &saResp))
+
+			saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId
+			if saId != "" {
+				t.Cleanup(func() {
+					callIAMAPI(t, "DeleteServiceAccount", url.Values{
+						"ServiceAccountId": {saId},
+					})
+				})
+			}
+		}
+	})
+
+	t.Run("reject_past_expiration_on_update", func(t *testing.T) {
+		// Create a valid service account first
+		futureExpiration := time.Now().Add(24 * time.Hour).Unix()
+
+		createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
+			"ParentUser":  {parentUserName},
+			"Description": {"For update test"},
+			"Expiration":  {strconv.FormatInt(futureExpiration, 10)},
+		})
+		require.NoError(t, err)
+		defer createResp.Body.Close()
+		require.Equal(t, http.StatusOK, createResp.StatusCode)
+
+		body, err := io.ReadAll(createResp.Body)
+		require.NoError(t, err)
+
+		var saResp CreateServiceAccountResponse
+		err = xml.Unmarshal(body, &saResp)
+		require.NoError(t, err)
+
+		saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId
+		require.NotEmpty(t, saId)
+
+		defer func() {
+			callIAMAPI(t, "DeleteServiceAccount", url.Values{"ServiceAccountId": {saId}})
+		}()
+
+		// Try to update with past expiration
+		pastExpiration := time.Now().Add(-1 * time.Hour).Unix()
+
+		updateResp, err := callIAMAPI(t, "UpdateServiceAccount", url.Values{
+			"ServiceAccountId": {saId},
+			"Expiration":       {strconv.FormatInt(pastExpiration, 10)},
+		})
+		require.NoError(t, err)
+		defer updateResp.Body.Close()
+
+		// Should fail because expiration is in the past
+		assert.NotEqual(t, http.StatusOK, updateResp.StatusCode,
+			"Updating service account with past expiration should fail")
+	})
+}
+
+// TestServiceAccountInheritedPermissions verifies that service accounts
+// inherit their parent user's permissions.
+// This is a key security test - SAs should not have MORE permissions than parent.
+func TestServiceAccountInheritedPermissions(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSeaweedFSRunning(t) {
+		t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
+	}
+
+	// Setup: Create a parent user
+	parentUserName := fmt.Sprintf("inherit-test-%d", time.Now().UnixNano())
+
+	resp, err := callIAMAPI(t, "CreateUser", url.Values{
+		"UserName": {parentUserName},
+	})
+	require.NoError(t, err)
+	resp.Body.Close()
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+
+	defer func() {
+		callIAMAPI(t, "DeleteUser", url.Values{"UserName": {parentUserName}})
+	}()
+
+	// Create service account
+	createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
+		"ParentUser":  {parentUserName},
+		"Description": {"Permissions inheritance test"},
+	})
+	require.NoError(t, err)
+	defer createResp.Body.Close()
+	require.Equal(t, http.StatusOK, createResp.StatusCode)
+
+	body, err := io.ReadAll(createResp.Body)
+	require.NoError(t, err)
+
+	var saResp CreateServiceAccountResponse
+	err = xml.Unmarshal(body, &saResp)
+	require.NoError(t, err)
+
+	saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId
+	require.NotEmpty(t, saId)
+
+	defer func() {
+		callIAMAPI(t, "DeleteServiceAccount", url.Values{"ServiceAccountId": {saId}})
+	}()
+
+	t.Run("service_account_linked_to_parent", func(t *testing.T) {
+		// Verify the service account is correctly linked to the parent
+		getResp, err := callIAMAPI(t, "GetServiceAccount", url.Values{
+			"ServiceAccountId": {saId},
+		})
+		require.NoError(t, err)
+		defer getResp.Body.Close()
+
+		body, err := io.ReadAll(getResp.Body)
+		require.NoError(t, err)
+
+		var result GetServiceAccountResponse
+		err = xml.Unmarshal(body, &result)
+		require.NoError(t, err, "Failed to parse response: %s", string(body))
+
+		assert.Equal(t, parentUserName, result.GetServiceAccountResult.ServiceAccount.ParentUser,
+			"Service account should be linked to correct parent user")
+	})
+
+	t.Run("list_shows_correct_parent", func(t *testing.T) {
+		// List service accounts filtered by parent
+		listResp, err := callIAMAPI(t, "ListServiceAccounts", url.Values{
+			"ParentUser": {parentUserName},
+		})
+		require.NoError(t, err)
+		defer listResp.Body.Close()
+
+		body, err := io.ReadAll(listResp.Body)
+		require.NoError(t, err)
+
+		var listResult ListServiceAccountsResponse
+		err = xml.Unmarshal(body, &listResult)
+		require.NoError(t, err)
+
+		// Should find at least one service account for this parent
+		found := false
+		for _, sa := range listResult.ListServiceAccountsResult.ServiceAccounts {
+			if sa.ServiceAccountId == saId {
+				found = true
+				assert.Equal(t, parentUserName, sa.ParentUser)
+				break
+			}
+		}
+		assert.True(t, found, "Service account should appear in list filtered by parent")
+	})
+}
+
+// TestServiceAccountAccessKeyFormat verifies that service account access keys
+// follow the correct AWS format.
+func TestServiceAccountAccessKeyFormat(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSeaweedFSRunning(t) {
+		t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
+	}
+
+	// Setup: Create a parent user
+	parentUserName := fmt.Sprintf("keyformat-test-%d", time.Now().UnixNano())
+
+	resp, err := callIAMAPI(t, "CreateUser", url.Values{
+		"UserName": {parentUserName},
+	})
+	require.NoError(t, err)
+	resp.Body.Close()
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+
+	defer func() {
+		callIAMAPI(t, "DeleteUser", url.Values{"UserName": {parentUserName}})
+	}()
+
+	createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
+		"ParentUser":  {parentUserName},
+		"Description": {"Key format test"},
+	})
+	require.NoError(t, err)
+	defer createResp.Body.Close()
+	require.Equal(t, http.StatusOK, createResp.StatusCode)
+
+	body, err := io.ReadAll(createResp.Body)
+	require.NoError(t, err)
+
+	var saResp CreateServiceAccountResponse
+	err = xml.Unmarshal(body, &saResp)
+	require.NoError(t, err)
+
+	accessKeyId := saResp.CreateServiceAccountResult.ServiceAccount.AccessKeyId
+	secretAccessKey := saResp.CreateServiceAccountResult.ServiceAccount.SecretAccessKey
+	saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId
+
+	defer func() {
+		callIAMAPI(t, "DeleteServiceAccount", url.Values{"ServiceAccountId": {saId}})
+	}()
+
+	t.Run("access_key_has_correct_prefix", func(t *testing.T) {
+		// Service account access keys should start with ABIA
+		assert.True(t, len(accessKeyId) >= 4,
+			"Access key should be at least 4 characters")
+		assert.Equal(t, "ABIA", accessKeyId[:4],
+			"Service account access key should start with ABIA prefix")
+	})
+
+	t.Run("access_key_has_correct_length", func(t *testing.T) {
+		// AWS access keys are 20 characters
+		assert.Equal(t, 20, len(accessKeyId),
+			"Access key should be exactly 20 characters (AWS standard)")
+	})
+
+	t.Run("secret_key_has_correct_length", func(t *testing.T) {
+		// AWS secret keys are 40 characters
+		assert.Equal(t, 40, len(secretAccessKey),
+			"Secret key should be exactly 40 characters (AWS standard)")
+	})
+}

test/s3/iam/s3_service_account_test.go

@@ -0,0 +1,379 @@
+package iam
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Service Account API test constants
+const (
+	TestIAMEndpoint = "http://localhost:8333"
+)
+
+// ServiceAccountInfo represents the response structure for service account operations
+type ServiceAccountInfo struct {
+	ServiceAccountId string `xml:"ServiceAccountId"`
+	ParentUser       string `xml:"ParentUser"`
+	Description      string `xml:"Description,omitempty"`
+	AccessKeyId      string `xml:"AccessKeyId"`
+	SecretAccessKey  string `xml:"SecretAccessKey,omitempty"`
+	Status           string `xml:"Status"`
+	Expiration       string `xml:"Expiration,omitempty"`
+	CreateDate       string `xml:"CreateDate"`
+}
+
+// CreateServiceAccountResponse represents the response for CreateServiceAccount
+type CreateServiceAccountResponse struct {
+	XMLName                    xml.Name `xml:"CreateServiceAccountResponse"`
+	CreateServiceAccountResult struct {
+		ServiceAccount ServiceAccountInfo `xml:"ServiceAccount"`
+	} `xml:"CreateServiceAccountResult"`
+}
+
+// ListServiceAccountsResponse represents the response for ListServiceAccounts
+type ListServiceAccountsResponse struct {
+	XMLName                   xml.Name `xml:"ListServiceAccountsResponse"`
+	ListServiceAccountsResult struct {
+		ServiceAccounts []ServiceAccountInfo `xml:"ServiceAccounts>member"`
+		IsTruncated     bool                 `xml:"IsTruncated"`
+	} `xml:"ListServiceAccountsResult"`
+}
+
+// GetServiceAccountResponse represents the response for GetServiceAccount
+type GetServiceAccountResponse struct {
+	XMLName                 xml.Name `xml:"GetServiceAccountResponse"`
+	GetServiceAccountResult struct {
+		ServiceAccount ServiceAccountInfo `xml:"ServiceAccount"`
+	} `xml:"GetServiceAccountResult"`
+}
+
+// TestServiceAccountLifecycle tests the complete lifecycle of service accounts
+// This is a high-value test covering Create, Get, List, Update, Delete operations
+func TestServiceAccountLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	// Check if SeaweedFS is running
+	if !isSeaweedFSRunning(t) {
+		t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
+	}
+
+	// First, ensure the parent user exists
+	parentUserName := fmt.Sprintf("testuser-%d", time.Now().UnixNano())
+
+	t.Run("create_parent_user", func(t *testing.T) {
+		resp, err := callIAMAPI(t, "CreateUser", url.Values{
+			"UserName": {parentUserName},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+		assert.Equal(t, http.StatusOK, resp.StatusCode, "CreateUser should succeed")
+	})
+
+	// Store service account IDs for cleanup
+	var createdServiceAccounts []string
+
+	defer func() {
+		// Cleanup: delete all service accounts first, then parent user
+		for _, saId := range createdServiceAccounts {
+			resp, _ := callIAMAPI(t, "DeleteServiceAccount", url.Values{
+				"ServiceAccountId": {saId},
+			})
+			if resp != nil {
+				resp.Body.Close()
+			}
+		}
+		// Now delete the parent user
+		resp, _ := callIAMAPI(t, "DeleteUser", url.Values{
+			"UserName": {parentUserName},
+		})
+		if resp != nil {
+			resp.Body.Close()
+		}
+	}()
+
+	var createdSAId string
+	var createdAccessKeyId string
+	var createdSecretAccessKey string
+
+	t.Run("create_service_account", func(t *testing.T) {
+		resp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
+			"ParentUser":  {parentUserName},
+			"Description": {"Test service account for CI/CD"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode, "CreateServiceAccount should succeed")
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		var createResp CreateServiceAccountResponse
+		err = xml.Unmarshal(body, &createResp)
+		require.NoError(t, err)
+
+		sa := createResp.CreateServiceAccountResult.ServiceAccount
+		createdSAId = sa.ServiceAccountId
+		createdAccessKeyId = sa.AccessKeyId
+		createdSecretAccessKey = sa.SecretAccessKey
+
+		// Add to cleanup list
+		createdServiceAccounts = append(createdServiceAccounts, createdSAId)
+
+		assert.NotEmpty(t, createdSAId, "ServiceAccountId should not be empty")
+		assert.Equal(t, parentUserName, sa.ParentUser, "ParentUser should match")
+		assert.Equal(t, "Test service account for CI/CD", sa.Description)
+		assert.Equal(t, "Active", sa.Status)
+		assert.NotEmpty(t, sa.AccessKeyId, "AccessKeyId should not be empty")
+		assert.NotEmpty(t, sa.SecretAccessKey, "SecretAccessKey should be returned on create")
+		assert.True(t, strings.HasPrefix(sa.AccessKeyId, "ABIA"),
+			"Service account AccessKeyId should have ABIA prefix")
+
+		t.Logf("Created service account: ID=%s, AccessKeyId=%s", createdSAId, createdAccessKeyId)
+	})
+
+	t.Run("get_service_account", func(t *testing.T) {
+		require.NotEmpty(t, createdSAId, "Service account should have been created")
+
+		resp, err := callIAMAPI(t, "GetServiceAccount", url.Values{
+			"ServiceAccountId": {createdSAId},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		var getResp GetServiceAccountResponse
+		err = xml.Unmarshal(body, &getResp)
+		require.NoError(t, err)
+
+		sa := getResp.GetServiceAccountResult.ServiceAccount
+		assert.Equal(t, createdSAId, sa.ServiceAccountId)
+		assert.Equal(t, parentUserName, sa.ParentUser)
+		assert.Empty(t, sa.SecretAccessKey, "SecretAccessKey should not be returned on Get")
+	})
+
+	t.Run("list_service_accounts", func(t *testing.T) {
+		resp, err := callIAMAPI(t, "ListServiceAccounts", url.Values{
+			"ParentUser": {parentUserName},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		var listResp ListServiceAccountsResponse
+		err = xml.Unmarshal(body, &listResp)
+		require.NoError(t, err)
+
+		assert.GreaterOrEqual(t, len(listResp.ListServiceAccountsResult.ServiceAccounts), 1,
+			"Should have at least one service account for the parent user")
+	})
+
+	t.Run("update_service_account_status", func(t *testing.T) {
+		require.NotEmpty(t, createdSAId)
+
+		// Disable the service account
+		resp, err := callIAMAPI(t, "UpdateServiceAccount", url.Values{
+			"ServiceAccountId": {createdSAId},
+			"Status":           {"Inactive"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+		// Verify it's now inactive
+		getResp, err := callIAMAPI(t, "GetServiceAccount", url.Values{
+			"ServiceAccountId": {createdSAId},
+		})
+		require.NoError(t, err)
+		defer getResp.Body.Close()
+
+		body, err := io.ReadAll(getResp.Body)
+		require.NoError(t, err)
+
+		var result GetServiceAccountResponse
+		err = xml.Unmarshal(body, &result)
+		require.NoError(t, err, "Failed to parse response: %s", string(body))
+
+		assert.Equal(t, "Inactive", result.GetServiceAccountResult.ServiceAccount.Status)
+	})
+
+	// Test that credentials could be used (verify they work with AWS SDK)
+	// This must run BEFORE delete_service_account to use valid credentials
+	t.Run("use_service_account_credentials", func(t *testing.T) {
+		require.NotEmpty(t, createdAccessKeyId)
+		require.NotEmpty(t, createdSecretAccessKey)
+
+		sess, err := session.NewSession(&aws.Config{
+			Region:   aws.String("us-east-1"),
+			Endpoint: aws.String(TestIAMEndpoint), // IAM and S3 usually on same port in mini-seaweed
+			Credentials: credentials.NewStaticCredentials(
+				createdAccessKeyId,
+				createdSecretAccessKey,
+				"",
+			),
+			DisableSSL:       aws.Bool(true),
+			S3ForcePathStyle: aws.Bool(true),
+		})
+		require.NoError(t, err)
+
+		s3Client := s3.New(sess)
+		_, err = s3Client.ListBuckets(&s3.ListBucketsInput{})
+
+		// Note: we don't necessarily expect success if no buckets/permissions
+		// but we expect it not to fail with "InvalidAccessKeyId" or "SignatureDoesNotMatch"
+		if err != nil {
+			if aerr, ok := err.(awserr.Error); ok {
+				assert.NotEqual(t, "InvalidAccessKeyId", aerr.Code(), "Credentials should be valid")
+				assert.NotEqual(t, "SignatureDoesNotMatch", aerr.Code(), "Signature should be valid")
+			}
+		}
+	})
+
+	t.Run("delete_service_account", func(t *testing.T) {
+		require.NotEmpty(t, createdSAId)
+
+		resp, err := callIAMAPI(t, "DeleteServiceAccount", url.Values{
+			"ServiceAccountId": {createdSAId},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+		// Verify it no longer exists
+		getResp, err := callIAMAPI(t, "GetServiceAccount", url.Values{
+			"ServiceAccountId": {createdSAId},
+		})
+		require.NoError(t, err)
+		defer getResp.Body.Close()
+
+		// Should return an error (not found)
+		assert.NotEqual(t, http.StatusOK, getResp.StatusCode,
+			"GetServiceAccount should fail after deletion")
+	})
+
+}
+
+// TestServiceAccountValidation tests validation of service account operations
+func TestServiceAccountValidation(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSeaweedFSRunning(t) {
+		t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
+	}
+
+	t.Run("create_without_parent_user", func(t *testing.T) {
+		resp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
+			"Description": {"Test without parent"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"CreateServiceAccount without ParentUser should fail")
+	})
+
+	t.Run("create_with_nonexistent_parent", func(t *testing.T) {
+		resp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
+			"ParentUser":  {"nonexistent-user-12345"},
+			"Description": {"Test with nonexistent parent"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"CreateServiceAccount with nonexistent parent should fail")
+	})
+
+	t.Run("get_nonexistent_service_account", func(t *testing.T) {
+		resp, err := callIAMAPI(t, "GetServiceAccount", url.Values{
+			"ServiceAccountId": {"sa-NONEXISTENT123"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"GetServiceAccount for nonexistent ID should fail")
+	})
+
+	t.Run("delete_nonexistent_service_account", func(t *testing.T) {
+		resp, err := callIAMAPI(t, "DeleteServiceAccount", url.Values{
+			"ServiceAccountId": {"sa-NONEXISTENT123"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"DeleteServiceAccount for nonexistent ID should fail")
+	})
+}
+
+// callIAMAPI is a helper to make IAM API calls
+func callIAMAPI(t *testing.T, action string, params url.Values) (*http.Response, error) {
+	params.Set("Action", action)
+	body := params.Encode()
+
+	req, err := http.NewRequest(http.MethodPost, TestIAMEndpoint+"/",
+		strings.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	// Sign with SigV4 when admin credentials are provided via env vars.
+	// IAM write APIs (CreateUser, CreateServiceAccount, etc.) require an
+	// authenticated admin caller; unsigned requests get routed to the STS
+	// fallback and rejected as an unknown action.
+	if accessKey := os.Getenv("STS_TEST_ACCESS_KEY"); accessKey != "" {
+		secretKey := os.Getenv("STS_TEST_SECRET_KEY")
+		creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
+		signer := v4.NewSigner(creds)
+		if _, err := signer.Sign(req, strings.NewReader(body), "iam", "us-east-1", time.Now()); err != nil {
+			return nil, fmt.Errorf("failed to sign IAM request: %w", err)
+		}
+	}
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	return client.Do(req)
+}
+
+// isSeaweedFSRunning checks if SeaweedFS S3 API is running
+func isSeaweedFSRunning(t *testing.T) bool {
+	client := &http.Client{Timeout: 2 * time.Second}
+	resp, err := client.Get(TestIAMEndpoint + "/status")
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+	return resp.StatusCode == http.StatusOK
+}

test/s3/iam/s3_sts_assume_role_test.go

@@ -0,0 +1,365 @@
+package iam
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// AssumeRoleResponse represents the STS AssumeRole response
+type AssumeRoleTestResponse struct {
+	XMLName xml.Name `xml:"AssumeRoleResponse"`
+	Result  struct {
+		Credentials struct {
+			AccessKeyId     string `xml:"AccessKeyId"`
+			SecretAccessKey string `xml:"SecretAccessKey"`
+			SessionToken    string `xml:"SessionToken"`
+			Expiration      string `xml:"Expiration"`
+		} `xml:"Credentials"`
+		AssumedRoleUser struct {
+			AssumedRoleId string `xml:"AssumedRoleId"`
+			Arn           string `xml:"Arn"`
+		} `xml:"AssumedRoleUser"`
+	} `xml:"AssumeRoleResult"`
+}
+
+// TestSTSAssumeRoleValidation tests input validation for AssumeRole endpoint
+func TestSTSAssumeRoleValidation(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Fatal("SeaweedFS STS endpoint is not running at", TestSTSEndpoint, "- please run 'make setup-all-tests' first")
+	}
+
+	// Check if AssumeRole is implemented by making a test call
+	if !isAssumeRoleImplemented(t) {
+		t.Fatal("AssumeRole action is not implemented in the running server - please rebuild weed binary with new code and restart the server")
+	}
+
+	t.Run("missing_role_arn_defaults_to_caller_identity", func(t *testing.T) {
+		// SeaweedFS intentionally allows AssumeRole without RoleArn to support
+		// S3-compatible clients that omit it. In that case the session is tied
+		// to the caller's own identity (User Context assumption). See
+		// handleAssumeRole in weed/s3api/s3api_sts.go.
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"AssumeRole"},
+			"Version":         {"2011-06-15"},
+			"RoleSessionName": {"test-session"},
+			// RoleArn is missing on purpose
+		}, "test-access-key", "test-secret-key")
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		assert.Equal(t, http.StatusOK, resp.StatusCode,
+			"Should succeed without RoleArn (user-context assumption): %s", string(body))
+
+		var assumeResp AssumeRoleTestResponse
+		require.NoError(t, xml.Unmarshal(body, &assumeResp),
+			"Failed to parse response: %s", string(body))
+		assert.NotEmpty(t, assumeResp.Result.Credentials.AccessKeyId,
+			"AccessKeyId should be issued for user-context assumption")
+	})
+
+	t.Run("missing_role_session_name", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"AssumeRole"},
+			"Version": {"2011-06-15"},
+			"RoleArn": {"arn:aws:iam::role/test-role"},
+			// RoleSessionName is missing
+		}, "test-access-key", "test-secret-key")
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"Should fail without RoleSessionName")
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		var errResp STSErrorTestResponse
+		err = xml.Unmarshal(body, &errResp)
+		require.NoError(t, err, "Failed to parse error response: %s", string(body))
+		assert.Equal(t, "MissingParameter", errResp.Error.Code)
+	})
+
+	t.Run("unsupported_action_for_anonymous", func(t *testing.T) {
+		// AssumeRole requires SigV4 authentication, anonymous requests should fail
+		resp, err := callSTSAPI(t, url.Values{
+			"Action":          {"AssumeRole"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/test-role"},
+			"RoleSessionName": {"test-session"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		// Should fail because AssumeRole requires AWS SigV4 authentication
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"AssumeRole should require authentication")
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Logf("Response for anonymous AssumeRole: status=%d, body=%s", resp.StatusCode, string(body))
+	})
+
+	t.Run("invalid_duration_too_short", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"AssumeRole"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/test-role"},
+			"RoleSessionName": {"test-session"},
+			"DurationSeconds": {"100"}, // Less than 900 seconds minimum
+		}, "test-access-key", "test-secret-key")
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"Should fail with DurationSeconds < 900")
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		var errResp STSErrorTestResponse
+		err = xml.Unmarshal(body, &errResp)
+		require.NoError(t, err, "Failed to parse error response: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("invalid_duration_too_long", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"AssumeRole"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/test-role"},
+			"RoleSessionName": {"test-session"},
+			"DurationSeconds": {"100000"}, // More than 43200 seconds maximum
+		}, "test-access-key", "test-secret-key")
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"Should fail with DurationSeconds > 43200")
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		var errResp STSErrorTestResponse
+		err = xml.Unmarshal(body, &errResp)
+		require.NoError(t, err, "Failed to parse error response: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+}
+
+// isAssumeRoleImplemented checks if the running server supports AssumeRole
+func isAssumeRoleImplemented(t *testing.T) bool {
+	resp, err := callSTSAPIWithSigV4(t, url.Values{
+		"Action":          {"AssumeRole"},
+		"Version":         {"2011-06-15"},
+		"RoleArn":         {"arn:aws:iam::role/test"},
+		"RoleSessionName": {"test"},
+	}, "test", "test")
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return false
+	}
+
+	// If we get "NotImplemented", the action isn't supported
+	var errResp STSErrorTestResponse
+	if xml.Unmarshal(body, &errResp) == nil && errResp.Error.Code == "NotImplemented" {
+		return false
+	}
+
+	// If we get InvalidAction, the action isn't routed
+	if errResp.Error.Code == "InvalidAction" {
+		return false
+	}
+
+	return true
+}
+
+// TestSTSAssumeRoleWithValidCredentials tests AssumeRole with valid IAM credentials
+// This test requires a configured IAM user in SeaweedFS
+func TestSTSAssumeRoleWithValidCredentials(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	// Use test credentials from environment or fall back to defaults
+	accessKey := os.Getenv("STS_TEST_ACCESS_KEY")
+	if accessKey == "" {
+		accessKey = "admin"
+	}
+	secretKey := os.Getenv("STS_TEST_SECRET_KEY")
+	if secretKey == "" {
+		secretKey = "admin"
+	}
+
+	t.Run("successful_assume_role", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"AssumeRole"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/admin"},
+			"RoleSessionName": {"integration-test-session"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
+
+		// If AssumeRole is not yet implemented, expect an error about unsupported action
+		if resp.StatusCode != http.StatusOK {
+			var errResp STSErrorTestResponse
+			err = xml.Unmarshal(body, &errResp)
+			require.NoError(t, err, "Failed to parse error response: %s", string(body))
+			t.Logf("Error response: code=%s, message=%s", errResp.Error.Code, errResp.Error.Message)
+
+			// This test will initially fail until AssumeRole is implemented
+			// Once implemented, uncomment the assertions below
+			// assert.Fail(t, "AssumeRole not yet implemented")
+		} else {
+			var stsResp AssumeRoleTestResponse
+			err = xml.Unmarshal(body, &stsResp)
+			require.NoError(t, err, "Failed to parse response: %s", string(body))
+
+			creds := stsResp.Result.Credentials
+			assert.NotEmpty(t, creds.AccessKeyId, "AccessKeyId should not be empty")
+			assert.NotEmpty(t, creds.SecretAccessKey, "SecretAccessKey should not be empty")
+			assert.NotEmpty(t, creds.SessionToken, "SessionToken should not be empty")
+			assert.NotEmpty(t, creds.Expiration, "Expiration should not be empty")
+
+			t.Logf("Successfully obtained temporary credentials: AccessKeyId=%s", creds.AccessKeyId)
+		}
+	})
+
+	t.Run("with_custom_duration", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"AssumeRole"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/admin"},
+			"RoleSessionName": {"duration-test-session"},
+			"DurationSeconds": {"3600"}, // 1 hour
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
+
+		// Verify DurationSeconds is accepted
+		if resp.StatusCode != http.StatusOK {
+			var errResp STSErrorTestResponse
+			err = xml.Unmarshal(body, &errResp)
+			require.NoError(t, err, "Failed to parse error response: %s", string(body))
+			// Should not fail due to DurationSeconds parameter
+			assert.NotContains(t, errResp.Error.Message, "DurationSeconds",
+				"DurationSeconds parameter should be accepted")
+		}
+	})
+}
+
+// TestSTSAssumeRoleWithInvalidCredentials tests AssumeRole rejection with bad credentials
+func TestSTSAssumeRoleWithInvalidCredentials(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	t.Run("invalid_access_key", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"AssumeRole"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/admin"},
+			"RoleSessionName": {"test-session"},
+		}, "invalid-access-key", "some-secret-key")
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		// Should fail with access denied or signature mismatch
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"Should fail with invalid access key")
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Logf("Response for invalid credentials: status=%d, body=%s", resp.StatusCode, string(body))
+	})
+
+	t.Run("invalid_secret_key", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"AssumeRole"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/admin"},
+			"RoleSessionName": {"test-session"},
+		}, "admin", "wrong-secret-key")
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		// Should fail with signature mismatch
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"Should fail with invalid secret key")
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Logf("Response for wrong secret: status=%d, body=%s", resp.StatusCode, string(body))
+	})
+}
+
+// callSTSAPIWithSigV4 makes an STS API call with AWS Signature V4 authentication
+func callSTSAPIWithSigV4(t *testing.T, params url.Values, accessKey, secretKey string) (*http.Response, error) {
+	// Prepare request body
+	body := params.Encode()
+
+	// Create request
+	req, err := http.NewRequest(http.MethodPost, TestSTSEndpoint+"/",
+		strings.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	// Do NOT set req.Header["Host"] — aws-sdk-go v1 v4.Signer reads the host
+	// from req.URL.Host/req.Host, and a manual Host header makes the signer
+	// emit `host;host` in SignedHeaders, producing a different signature than
+	// the server calculates from canonical headers.
+
+	// Sign request with AWS Signature V4 using official SDK
+	creds := credentials.NewStaticCredentials(accessKey, secretKey, "")
+	signer := v4.NewSigner(creds)
+
+	// Read body for signing
+	// Note: We need a ReadSeeker for the signer, or we can pass the body string/bytes to ComputeBodyHash if needed,
+	// but standard Sign method takes an io.ReadSeeker for the body.
+	bodyReader := strings.NewReader(body)
+	_, err = signer.Sign(req, bodyReader, "sts", "us-east-1", time.Now())
+	if err != nil {
+		return nil, fmt.Errorf("failed to sign request: %w", err)
+	}
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	return client.Do(req)
+}

test/s3/iam/s3_sts_credential_prefix_test.go

@@ -0,0 +1,82 @@
+package iam
+
+import (
+	"encoding/xml"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestSTSTemporaryCredentialPrefix verifies that STS temporary credentials use ASIA prefix
+// This test ensures AWS compatibility - temporary credentials should use ASIA, not AKIA
+func TestSTSTemporaryCredentialPrefix(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	// Use test credentials from environment or fall back to defaults
+	accessKey := os.Getenv("STS_TEST_ACCESS_KEY")
+	if accessKey == "" {
+		accessKey = "admin"
+	}
+	secretKey := os.Getenv("STS_TEST_SECRET_KEY")
+	if secretKey == "" {
+		secretKey = "admin"
+	}
+
+	t.Run("assume_role_returns_asia_prefix", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"AssumeRole"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/admin"},
+			"RoleSessionName": {"asia-prefix-test"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		if resp.StatusCode != http.StatusOK {
+			t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
+			t.Skip("AssumeRole not fully implemented yet")
+		}
+
+		var stsResp AssumeRoleTestResponse
+		err = xml.Unmarshal(body, &stsResp)
+		require.NoError(t, err, "Failed to parse response: %s", string(body))
+
+		creds := stsResp.Result.Credentials
+		require.NotEmpty(t, creds.AccessKeyId, "AccessKeyId should not be empty")
+
+		// Verify ASIA prefix for temporary credentials
+		assert.True(t, strings.HasPrefix(creds.AccessKeyId, "ASIA"),
+			"Temporary credentials must use ASIA prefix (not AKIA for permanent keys), got: %s", creds.AccessKeyId)
+
+		// Verify it's NOT using AKIA (permanent credentials)
+		assert.False(t, strings.HasPrefix(creds.AccessKeyId, "AKIA"),
+			"Temporary credentials must NOT use AKIA prefix (that's for permanent IAM keys), got: %s", creds.AccessKeyId)
+
+		// Verify format: ASIA + 16 hex characters = 20 chars total
+		assert.Equal(t, 20, len(creds.AccessKeyId),
+			"Access key ID should be 20 characters (ASIA + 16 hex chars), got: %s", creds.AccessKeyId)
+
+		t.Logf("✓ Temporary credentials correctly use ASIA prefix: %s", creds.AccessKeyId)
+	})
+
+	t.Run("assume_role_with_web_identity_returns_asia_prefix", func(t *testing.T) {
+		// This test would require OIDC setup, so we'll skip it for now
+		// but the same ASIA prefix validation should apply
+		t.Skip("AssumeRoleWithWebIdentity requires OIDC provider setup")
+	})
+}

test/s3/iam/s3_sts_get_federation_token_test.go

@@ -0,0 +1,511 @@
+package iam
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// GetFederationTokenTestResponse represents the STS GetFederationToken response
+type GetFederationTokenTestResponse struct {
+	XMLName xml.Name `xml:"GetFederationTokenResponse"`
+	Result  struct {
+		Credentials struct {
+			AccessKeyId     string `xml:"AccessKeyId"`
+			SecretAccessKey string `xml:"SecretAccessKey"`
+			SessionToken    string `xml:"SessionToken"`
+			Expiration      string `xml:"Expiration"`
+		} `xml:"Credentials"`
+		FederatedUser struct {
+			FederatedUserId string `xml:"FederatedUserId"`
+			Arn             string `xml:"Arn"`
+		} `xml:"FederatedUser"`
+	} `xml:"GetFederationTokenResult"`
+}
+
+func getTestCredentials() (string, string) {
+	accessKey := os.Getenv("STS_TEST_ACCESS_KEY")
+	if accessKey == "" {
+		accessKey = "admin"
+	}
+	secretKey := os.Getenv("STS_TEST_SECRET_KEY")
+	if secretKey == "" {
+		secretKey = "admin"
+	}
+	return accessKey, secretKey
+}
+
+// isGetFederationTokenImplemented checks if the running server supports GetFederationToken
+func isGetFederationTokenImplemented(t *testing.T) bool {
+	accessKey, secretKey := getTestCredentials()
+	resp, err := callSTSAPIWithSigV4(t, url.Values{
+		"Action":  {"GetFederationToken"},
+		"Version": {"2011-06-15"},
+		"Name":    {"probe"},
+	}, accessKey, secretKey)
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(resp.Body)
+	var errResp STSErrorTestResponse
+	if xml.Unmarshal(body, &errResp) == nil {
+		if errResp.Error.Code == "InvalidAction" || errResp.Error.Code == "NotImplemented" {
+			return false
+		}
+	}
+	return true
+}
+
+// TestSTSGetFederationTokenValidation tests input validation for the GetFederationToken endpoint
+func TestSTSGetFederationTokenValidation(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Fatal("SeaweedFS STS endpoint is not running at", TestSTSEndpoint, "- please run 'make setup-all-tests' first")
+	}
+
+	if !isGetFederationTokenImplemented(t) {
+		t.Fatal("GetFederationToken action is not implemented in the running server")
+	}
+
+	accessKey, secretKey := getTestCredentials()
+
+	t.Run("missing_name", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			// Name is missing
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "MissingParameter", errResp.Error.Code)
+	})
+
+	t.Run("name_too_short", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"A"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("name_too_long", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {strings.Repeat("A", 33)},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("name_invalid_characters", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"bad name"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("duration_too_short", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"GetFederationToken"},
+			"Version":         {"2011-06-15"},
+			"Name":            {"TestApp"},
+			"DurationSeconds": {"100"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("duration_too_long", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"GetFederationToken"},
+			"Version":         {"2011-06-15"},
+			"Name":            {"TestApp"},
+			"DurationSeconds": {"200000"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "InvalidParameterValue", errResp.Error.Code)
+	})
+
+	t.Run("malformed_policy", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"TestApp"},
+			"Policy":  {"not-valid-json"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		var errResp STSErrorTestResponse
+		require.NoError(t, xml.Unmarshal(body, &errResp), "Failed to parse: %s", string(body))
+		assert.Equal(t, "MalformedPolicyDocument", errResp.Error.Code)
+	})
+
+	t.Run("anonymous_rejected", func(t *testing.T) {
+		// GetFederationToken requires SigV4, anonymous should fail
+		resp, err := callSTSAPI(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"TestApp"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode)
+	})
+}
+
+// TestSTSGetFederationTokenRejectTemporaryCredentials tests that temporary
+// credentials (session tokens) are rejected by GetFederationToken
+func TestSTSGetFederationTokenRejectTemporaryCredentials(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	if !isGetFederationTokenImplemented(t) {
+		t.Skip("GetFederationToken not implemented")
+	}
+
+	accessKey, secretKey := getTestCredentials()
+
+	// First, obtain temporary credentials via AssumeRole
+	resp, err := callSTSAPIWithSigV4(t, url.Values{
+		"Action":          {"AssumeRole"},
+		"Version":         {"2011-06-15"},
+		"RoleArn":         {"arn:aws:iam::role/admin"},
+		"RoleSessionName": {"temp-session"},
+	}, accessKey, secretKey)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+
+	if resp.StatusCode != http.StatusOK {
+		t.Skipf("AssumeRole failed (may not be configured): status=%d body=%s", resp.StatusCode, string(body))
+	}
+
+	var assumeResp AssumeRoleTestResponse
+	require.NoError(t, xml.Unmarshal(body, &assumeResp), "Parse AssumeRole response: %s", string(body))
+
+	tempAccessKey := assumeResp.Result.Credentials.AccessKeyId
+	tempSecretKey := assumeResp.Result.Credentials.SecretAccessKey
+	tempSessionToken := assumeResp.Result.Credentials.SessionToken
+	require.NotEmpty(t, tempAccessKey)
+	require.NotEmpty(t, tempSessionToken)
+
+	// Now try GetFederationToken with the temporary credentials
+	// Include X-Amz-Security-Token header which marks this as a temp credential call
+	params := url.Values{
+		"Action":  {"GetFederationToken"},
+		"Version": {"2011-06-15"},
+		"Name":    {"ShouldFail"},
+	}
+
+	reqBody := params.Encode()
+	req, err := http.NewRequest(http.MethodPost, TestSTSEndpoint+"/", strings.NewReader(reqBody))
+	require.NoError(t, err)
+
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Set("X-Amz-Security-Token", tempSessionToken)
+
+	creds := credentials.NewStaticCredentials(tempAccessKey, tempSecretKey, tempSessionToken)
+	signer := v4.NewSigner(creds)
+	_, err = signer.Sign(req, strings.NewReader(reqBody), "sts", "us-east-1", time.Now())
+	require.NoError(t, err)
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	resp2, err := client.Do(req)
+	require.NoError(t, err)
+	defer resp2.Body.Close()
+
+	body2, _ := io.ReadAll(resp2.Body)
+	assert.Equal(t, http.StatusForbidden, resp2.StatusCode,
+		"GetFederationToken should reject temporary credentials: %s", string(body2))
+	assert.Contains(t, string(body2), "temporary credentials",
+		"Error should mention temporary credentials")
+}
+
+// TestSTSGetFederationTokenSuccess tests a successful GetFederationToken call
+// and verifies the returned credentials can be used to access S3
+func TestSTSGetFederationTokenSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	if !isGetFederationTokenImplemented(t) {
+		t.Skip("GetFederationToken not implemented")
+	}
+
+	accessKey, secretKey := getTestCredentials()
+
+	t.Run("basic_success", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":  {"GetFederationToken"},
+			"Version": {"2011-06-15"},
+			"Name":    {"AppClient"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
+
+		if resp.StatusCode != http.StatusOK {
+			var errResp STSErrorTestResponse
+			_ = xml.Unmarshal(body, &errResp)
+			t.Fatalf("GetFederationToken failed: code=%s message=%s", errResp.Error.Code, errResp.Error.Message)
+		}
+
+		var stsResp GetFederationTokenTestResponse
+		require.NoError(t, xml.Unmarshal(body, &stsResp), "Parse response: %s", string(body))
+
+		creds := stsResp.Result.Credentials
+		assert.NotEmpty(t, creds.AccessKeyId)
+		assert.NotEmpty(t, creds.SecretAccessKey)
+		assert.NotEmpty(t, creds.SessionToken)
+		assert.NotEmpty(t, creds.Expiration)
+
+		fedUser := stsResp.Result.FederatedUser
+		assert.Contains(t, fedUser.Arn, "federated-user/AppClient")
+		assert.Contains(t, fedUser.FederatedUserId, "AppClient")
+	})
+
+	t.Run("with_custom_duration", func(t *testing.T) {
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"GetFederationToken"},
+			"Version":         {"2011-06-15"},
+			"Name":            {"DurationTest"},
+			"DurationSeconds": {"3600"},
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
+
+		if resp.StatusCode == http.StatusOK {
+			var stsResp GetFederationTokenTestResponse
+			require.NoError(t, xml.Unmarshal(body, &stsResp))
+			assert.NotEmpty(t, stsResp.Result.Credentials.AccessKeyId)
+
+			// Verify expiration is roughly 1 hour from now
+			expTime, err := time.Parse(time.RFC3339, stsResp.Result.Credentials.Expiration)
+			require.NoError(t, err)
+			diff := time.Until(expTime)
+			assert.InDelta(t, 3600, diff.Seconds(), 60,
+				"Expiration should be ~1 hour from now")
+		}
+	})
+
+	t.Run("with_36_hour_duration", func(t *testing.T) {
+		// GetFederationToken allows up to 36 hours (unlike AssumeRole's 12h max)
+		resp, err := callSTSAPIWithSigV4(t, url.Values{
+			"Action":          {"GetFederationToken"},
+			"Version":         {"2011-06-15"},
+			"Name":            {"LongDuration"},
+			"DurationSeconds": {"129600"}, // 36 hours
+		}, accessKey, secretKey)
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, _ := io.ReadAll(resp.Body)
+		if resp.StatusCode == http.StatusOK {
+			var stsResp GetFederationTokenTestResponse
+			require.NoError(t, xml.Unmarshal(body, &stsResp))
+
+			expTime, err := time.Parse(time.RFC3339, stsResp.Result.Credentials.Expiration)
+			require.NoError(t, err)
+			diff := time.Until(expTime)
+			assert.InDelta(t, 129600, diff.Seconds(), 60,
+				"Expiration should be ~36 hours from now")
+		} else {
+			// Duration should not cause a rejection
+			var errResp STSErrorTestResponse
+			_ = xml.Unmarshal(body, &errResp)
+			assert.NotContains(t, errResp.Error.Message, "DurationSeconds",
+				"36-hour duration should be accepted by GetFederationToken")
+		}
+	})
+}
+
+// TestSTSGetFederationTokenWithSessionPolicy tests that vended credentials
+// are scoped down by an inline session policy
+func TestSTSGetFederationTokenWithSessionPolicy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	if !isGetFederationTokenImplemented(t) {
+		t.Skip("GetFederationToken not implemented")
+	}
+
+	accessKey, secretKey := getTestCredentials()
+
+	// Create a test bucket using admin credentials
+	adminSess, err := session.NewSession(&aws.Config{
+		Region:           aws.String("us-east-1"),
+		Endpoint:         aws.String(TestSTSEndpoint),
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, ""),
+	})
+	require.NoError(t, err)
+
+	adminS3 := s3.New(adminSess)
+	bucket := fmt.Sprintf("fed-token-test-%d", time.Now().UnixNano())
+
+	_, err = adminS3.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucket)})
+	require.NoError(t, err)
+	defer adminS3.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)})
+
+	_, err = adminS3.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String("test.txt"),
+		Body:   strings.NewReader("hello"),
+	})
+	require.NoError(t, err)
+	defer adminS3.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String("test.txt")})
+
+	// Get federated credentials with a session policy that only allows GetObject
+	sessionPolicy := fmt.Sprintf(`{
+		"Version": "2012-10-17",
+		"Statement": [{
+			"Effect": "Allow",
+			"Action": ["s3:GetObject"],
+			"Resource": ["arn:aws:s3:::%s/*"]
+		}]
+	}`, bucket)
+
+	resp, err := callSTSAPIWithSigV4(t, url.Values{
+		"Action":  {"GetFederationToken"},
+		"Version": {"2011-06-15"},
+		"Name":    {"ScopedClient"},
+		"Policy":  {sessionPolicy},
+	}, accessKey, secretKey)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	body, _ := io.ReadAll(resp.Body)
+	t.Logf("GetFederationToken response: status=%d body=%s", resp.StatusCode, string(body))
+
+	if resp.StatusCode != http.StatusOK {
+		t.Skipf("GetFederationToken failed (may need IAM policy config): %s", string(body))
+	}
+
+	var stsResp GetFederationTokenTestResponse
+	require.NoError(t, xml.Unmarshal(body, &stsResp))
+
+	fedCreds := stsResp.Result.Credentials
+	require.NotEmpty(t, fedCreds.AccessKeyId)
+	require.NotEmpty(t, fedCreds.SessionToken)
+
+	// Create S3 client with the federated credentials
+	fedSess, err := session.NewSession(&aws.Config{
+		Region:           aws.String("us-east-1"),
+		Endpoint:         aws.String(TestSTSEndpoint),
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+		Credentials: credentials.NewStaticCredentials(
+			fedCreds.AccessKeyId, fedCreds.SecretAccessKey, fedCreds.SessionToken),
+	})
+	require.NoError(t, err)
+
+	fedS3 := s3.New(fedSess)
+
+	// GetObject should succeed (allowed by session policy)
+	getResp, err := fedS3.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String("test.txt"),
+	})
+	if err == nil {
+		defer getResp.Body.Close()
+		t.Log("GetObject with federated credentials succeeded (as expected)")
+	} else {
+		t.Logf("GetObject with federated credentials: %v (session policy enforcement may vary)", err)
+	}
+
+	// PutObject should be denied (not allowed by session policy)
+	_, err = fedS3.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String("denied.txt"),
+		Body:   strings.NewReader("should fail"),
+	})
+	if err != nil {
+		t.Log("PutObject correctly denied with federated credentials")
+		assert.Contains(t, err.Error(), "AccessDenied",
+			"PutObject should be denied by session policy")
+	} else {
+		// Clean up if unexpectedly succeeded
+		adminS3.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String("denied.txt")})
+		t.Log("PutObject unexpectedly succeeded — session policy enforcement may not be active")
+	}
+}

test/s3/iam/s3_sts_ldap_test.go

@@ -0,0 +1,291 @@
+package iam
+
+import (
+	"encoding/xml"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// AssumeRoleWithLDAPIdentityResponse represents the STS response for LDAP identity
+type AssumeRoleWithLDAPIdentityTestResponse struct {
+	XMLName xml.Name `xml:"AssumeRoleWithLDAPIdentityResponse"`
+	Result  struct {
+		Credentials struct {
+			AccessKeyId     string `xml:"AccessKeyId"`
+			SecretAccessKey string `xml:"SecretAccessKey"`
+			SessionToken    string `xml:"SessionToken"`
+			Expiration      string `xml:"Expiration"`
+		} `xml:"Credentials"`
+	} `xml:"AssumeRoleWithLDAPIdentityResult"`
+}
+
+// TestSTSLDAPValidation tests input validation for AssumeRoleWithLDAPIdentity
+func TestSTSLDAPValidation(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Fatal("SeaweedFS STS endpoint is not running at", TestSTSEndpoint, "- please run 'make setup-all-tests' first")
+	}
+
+	// Check if AssumeRoleWithLDAPIdentity is implemented
+	if !isLDAPIdentityActionImplemented(t) {
+		t.Fatal("AssumeRoleWithLDAPIdentity action is not implemented in the running server - please rebuild weed binary with new code and restart the server")
+	}
+
+	t.Run("missing_ldap_username", func(t *testing.T) {
+		resp, err := callSTSAPIForLDAP(t, url.Values{
+			"Action":          {"AssumeRoleWithLDAPIdentity"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/test-role"},
+			"RoleSessionName": {"test-session"},
+			"LDAPPassword":    {"testpass"},
+			// LDAPUsername is missing
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"Should fail without LDAPUsername")
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		var errResp STSErrorTestResponse
+		err = xml.Unmarshal(body, &errResp)
+		require.NoError(t, err, "Failed to parse error response: %s", string(body))
+		// Expect either MissingParameter or InvalidAction (if not implemented)
+		assert.Contains(t, []string{"MissingParameter", "InvalidAction"}, errResp.Error.Code)
+	})
+
+	t.Run("missing_ldap_password", func(t *testing.T) {
+		resp, err := callSTSAPIForLDAP(t, url.Values{
+			"Action":          {"AssumeRoleWithLDAPIdentity"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/test-role"},
+			"RoleSessionName": {"test-session"},
+			"LDAPUsername":    {"testuser"},
+			// LDAPPassword is missing
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"Should fail without LDAPPassword")
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		var errResp STSErrorTestResponse
+		err = xml.Unmarshal(body, &errResp)
+		require.NoError(t, err, "Failed to parse error response: %s", string(body))
+		assert.Contains(t, []string{"MissingParameter", "InvalidAction"}, errResp.Error.Code)
+	})
+
+	t.Run("missing_role_arn", func(t *testing.T) {
+		resp, err := callSTSAPIForLDAP(t, url.Values{
+			"Action":          {"AssumeRoleWithLDAPIdentity"},
+			"Version":         {"2011-06-15"},
+			"RoleSessionName": {"test-session"},
+			"LDAPUsername":    {"testuser"},
+			"LDAPPassword":    {"testpass"},
+			// RoleArn is missing
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"Should fail without RoleArn")
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		var errResp STSErrorTestResponse
+		err = xml.Unmarshal(body, &errResp)
+		require.NoError(t, err, "Failed to parse error response: %s", string(body))
+		assert.Contains(t, []string{"MissingParameter", "InvalidAction"}, errResp.Error.Code)
+	})
+
+	t.Run("invalid_duration_too_short", func(t *testing.T) {
+		resp, err := callSTSAPIForLDAP(t, url.Values{
+			"Action":          {"AssumeRoleWithLDAPIdentity"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/test-role"},
+			"RoleSessionName": {"test-session"},
+			"LDAPUsername":    {"testuser"},
+			"LDAPPassword":    {"testpass"},
+			"DurationSeconds": {"100"}, // Less than 900 seconds minimum
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		// If the action is implemented, it should reject invalid duration
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Logf("Response for invalid duration: status=%d, body=%s", resp.StatusCode, string(body))
+	})
+}
+
+// TestSTSLDAPWithValidCredentials tests LDAP authentication
+// This test requires an LDAP server to be configured
+func TestSTSLDAPWithValidCredentials(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	// Check if LDAP is configured (skip if not)
+	if !isLDAPConfigured() {
+		t.Skip("LDAP is not configured - skipping LDAP integration tests")
+	}
+
+	t.Run("successful_ldap_auth", func(t *testing.T) {
+		resp, err := callSTSAPIForLDAP(t, url.Values{
+			"Action":          {"AssumeRoleWithLDAPIdentity"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/ldap-user"},
+			"RoleSessionName": {"ldap-test-session"},
+			"LDAPUsername":    {"testuser"},
+			"LDAPPassword":    {"testpass"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
+
+		if resp.StatusCode == http.StatusOK {
+			var stsResp AssumeRoleWithLDAPIdentityTestResponse
+			err = xml.Unmarshal(body, &stsResp)
+			require.NoError(t, err, "Failed to parse response: %s", string(body))
+
+			creds := stsResp.Result.Credentials
+			assert.NotEmpty(t, creds.AccessKeyId, "AccessKeyId should not be empty")
+			assert.NotEmpty(t, creds.SecretAccessKey, "SecretAccessKey should not be empty")
+			assert.NotEmpty(t, creds.SessionToken, "SessionToken should not be empty")
+			assert.NotEmpty(t, creds.Expiration, "Expiration should not be empty")
+		}
+	})
+}
+
+// TestSTSLDAPWithInvalidCredentials tests LDAP rejection with bad credentials
+func TestSTSLDAPWithInvalidCredentials(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	t.Run("invalid_ldap_password", func(t *testing.T) {
+		resp, err := callSTSAPIForLDAP(t, url.Values{
+			"Action":          {"AssumeRoleWithLDAPIdentity"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/ldap-user"},
+			"RoleSessionName": {"ldap-test-session"},
+			"LDAPUsername":    {"testuser"},
+			"LDAPPassword":    {"wrong-password"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Logf("Response for invalid LDAP credentials: status=%d, body=%s", resp.StatusCode, string(body))
+
+		// Should fail (either AccessDenied or InvalidAction if not implemented)
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"Should fail with invalid LDAP password")
+	})
+
+	t.Run("nonexistent_ldap_user", func(t *testing.T) {
+		resp, err := callSTSAPIForLDAP(t, url.Values{
+			"Action":          {"AssumeRoleWithLDAPIdentity"},
+			"Version":         {"2011-06-15"},
+			"RoleArn":         {"arn:aws:iam::role/ldap-user"},
+			"RoleSessionName": {"ldap-test-session"},
+			"LDAPUsername":    {"nonexistent-user-12345"},
+			"LDAPPassword":    {"somepassword"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Logf("Response for nonexistent user: status=%d, body=%s", resp.StatusCode, string(body))
+
+		// Should fail
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"Should fail with nonexistent LDAP user")
+	})
+}
+
+// callSTSAPIForLDAP makes an STS API call for LDAP operation
+func callSTSAPIForLDAP(t *testing.T, params url.Values) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodPost, TestSTSEndpoint+"/",
+		strings.NewReader(params.Encode()))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	return client.Do(req)
+}
+
+// isLDAPConfigured checks if LDAP server is configured and available
+func isLDAPConfigured() bool {
+	// Check environment variable for LDAP URL
+	ldapURL := os.Getenv("LDAP_URL")
+	return ldapURL != ""
+}
+
+// isLDAPIdentityActionImplemented checks if the running server supports AssumeRoleWithLDAPIdentity
+func isLDAPIdentityActionImplemented(t *testing.T) bool {
+	resp, err := callSTSAPIForLDAP(t, url.Values{
+		"Action":          {"AssumeRoleWithLDAPIdentity"},
+		"Version":         {"2011-06-15"},
+		"RoleArn":         {"arn:aws:iam::role/test"},
+		"RoleSessionName": {"test"},
+		"LDAPUsername":    {"test"},
+		"LDAPPassword":    {"test"},
+	})
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return false
+	}
+
+	// If we get "NotImplemented" or empty response, the action isn't supported
+	if len(body) == 0 {
+		return false
+	}
+
+	var errResp STSErrorTestResponse
+	if xml.Unmarshal(body, &errResp) == nil && errResp.Error.Code == "NotImplemented" {
+		return false
+	}
+
+	// If we get InvalidAction, the action isn't routed
+	if errResp.Error.Code == "InvalidAction" {
+		return false
+	}
+
+	return true
+}

test/s3/iam/s3_sts_test.go

@@ -0,0 +1,268 @@
+package iam
+
+import (
+	"encoding/xml"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// STS API test constants
+const (
+	TestSTSEndpoint = "http://localhost:8333"
+)
+
+// AssumeRoleWithWebIdentityResponse represents the STS response
+type AssumeRoleWithWebIdentityTestResponse struct {
+	XMLName xml.Name `xml:"AssumeRoleWithWebIdentityResponse"`
+	Result  struct {
+		Credentials struct {
+			AccessKeyId     string `xml:"AccessKeyId"`
+			SecretAccessKey string `xml:"SecretAccessKey"`
+			SessionToken    string `xml:"SessionToken"`
+			Expiration      string `xml:"Expiration"`
+		} `xml:"Credentials"`
+		SubjectFromWebIdentityToken string `xml:"SubjectFromWebIdentityToken,omitempty"`
+	} `xml:"AssumeRoleWithWebIdentityResult"`
+}
+
+// STSErrorResponse represents an STS error response
+type STSErrorTestResponse struct {
+	XMLName xml.Name `xml:"ErrorResponse"`
+	Error   struct {
+		Type    string `xml:"Type"`
+		Code    string `xml:"Code"`
+		Message string `xml:"Message"`
+	} `xml:"Error"`
+	RequestId string `xml:"RequestId"`
+}
+
+// TestAssumeRoleWithWebIdentityValidation tests input validation for the STS endpoint
+func TestAssumeRoleWithWebIdentityValidation(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	t.Run("missing_web_identity_token", func(t *testing.T) {
+		resp, err := callSTSAPI(t, url.Values{
+			"Action":          {"AssumeRoleWithWebIdentity"},
+			"RoleArn":         {"arn:aws:iam::role/test-role"},
+			"RoleSessionName": {"test-session"},
+			// WebIdentityToken is missing
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"Should fail without WebIdentityToken")
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		var errResp STSErrorTestResponse
+		err = xml.Unmarshal(body, &errResp)
+		require.NoError(t, err, "Failed to parse error response: %s", string(body))
+		assert.Equal(t, "MissingParameter", errResp.Error.Code)
+	})
+
+	t.Run("missing_role_arn_invalid_jwt_still_rejected", func(t *testing.T) {
+		// Missing RoleArn is no longer a fast-fail at the HTTP layer:
+		// claim-based policy mode (Phase 3b) advertises RoleArn as
+		// optional so the STS service can derive the assumed-role ARN
+		// from the configured policy claim. Validation now happens at
+		// the STS layer once the JWT is parsed. With a bogus token the
+		// JWT parse fails first, so the request is still rejected —
+		// just with the JWT-parse error code instead of MissingParameter.
+		resp, err := callSTSAPI(t, url.Values{
+			"Action":           {"AssumeRoleWithWebIdentity"},
+			"WebIdentityToken": {"fake-jwt-token"},
+			"RoleSessionName":  {"test-session"},
+			// RoleArn omitted on purpose.
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"Should still fail when RoleArn is missing and the JWT is invalid")
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		var errResp STSErrorTestResponse
+		err = xml.Unmarshal(body, &errResp)
+		require.NoError(t, err, "Failed to parse error response: %s", string(body))
+		assert.NotEqual(t, "MissingParameter", errResp.Error.Code,
+			"missing RoleArn alone must no longer surface as MissingParameter")
+	})
+
+	t.Run("missing_role_session_name", func(t *testing.T) {
+		resp, err := callSTSAPI(t, url.Values{
+			"Action":           {"AssumeRoleWithWebIdentity"},
+			"WebIdentityToken": {"fake-jwt-token"},
+			"RoleArn":          {"arn:aws:iam::role/test-role"},
+			// RoleSessionName is missing
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"Should fail without RoleSessionName")
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		var errResp STSErrorTestResponse
+		err = xml.Unmarshal(body, &errResp)
+		require.NoError(t, err, "Failed to parse error response: %s", string(body))
+		assert.Equal(t, "MissingParameter", errResp.Error.Code)
+	})
+
+	t.Run("invalid_jwt_token", func(t *testing.T) {
+		resp, err := callSTSAPI(t, url.Values{
+			"Action":           {"AssumeRoleWithWebIdentity"},
+			"WebIdentityToken": {"not-a-valid-jwt-token"},
+			"RoleArn":          {"arn:aws:iam::role/test-role"},
+			"RoleSessionName":  {"test-session"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		// Should fail with AccessDenied since the JWT is invalid
+		assert.NotEqual(t, http.StatusOK, resp.StatusCode,
+			"Should fail with invalid JWT token")
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		var errResp STSErrorTestResponse
+		err = xml.Unmarshal(body, &errResp)
+		require.NoError(t, err, "Failed to parse error response: %s", string(body))
+		assert.Contains(t, []string{"AccessDenied", "InvalidParameterValue"}, errResp.Error.Code)
+	})
+}
+
+// TestAssumeRoleWithWebIdentityWithMockJWT tests the STS endpoint with mock JWTs
+// This test requires the mock OIDC provider to be configured
+func TestAssumeRoleWithWebIdentityWithMockJWT(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping integration test in short mode")
+	}
+
+	if !isSTSEndpointRunning(t) {
+		t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
+	}
+
+	// Create a test framework to get valid JWT tokens
+	framework := NewS3IAMTestFramework(t)
+	defer framework.Cleanup()
+
+	// Generate a test JWT using the framework
+	testUsername := "sts-test-user"
+	testRole := "readonly"
+
+	// Try to get a token - use Keycloak if available, otherwise generate a mock JWT
+	var token string
+	var err error
+	if framework.useKeycloak {
+		token, err = framework.getKeycloakToken(testUsername)
+	} else {
+		// Generate a mock JWT token with 1 hour validity
+		token, err = framework.generateJWTToken(testUsername, testRole, time.Hour)
+	}
+	if err != nil {
+		t.Skipf("Unable to generate test JWT (requires mock OIDC or Keycloak): %v", err)
+	}
+
+	t.Run("valid_jwt_token", func(t *testing.T) {
+		resp, err := callSTSAPI(t, url.Values{
+			"Action":           {"AssumeRoleWithWebIdentity"},
+			"WebIdentityToken": {token},
+			"RoleArn":          {"arn:aws:iam::role/" + testRole},
+			"RoleSessionName":  {"integration-test-session"},
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
+
+		// Note: This may still fail if the role/trust policy is not configured
+		// In that case, we just verify the error is about trust policy, not token validation
+		if resp.StatusCode != http.StatusOK {
+			var errResp STSErrorTestResponse
+			err = xml.Unmarshal(body, &errResp)
+			require.NoError(t, err, "Failed to parse error response: %s", string(body))
+			assert.NotEqual(t, "InvalidParameterValue", errResp.Error.Code,
+				"Token validation should not fail - error should be about trust policy")
+		} else {
+			var stsResp AssumeRoleWithWebIdentityTestResponse
+			err = xml.Unmarshal(body, &stsResp)
+			require.NoError(t, err, "Failed to parse response: %s", string(body))
+
+			creds := stsResp.Result.Credentials
+			assert.NotEmpty(t, creds.AccessKeyId, "AccessKeyId should not be empty")
+			assert.NotEmpty(t, creds.SecretAccessKey, "SecretAccessKey should not be empty")
+			assert.NotEmpty(t, creds.SessionToken, "SessionToken should not be empty")
+			assert.NotEmpty(t, creds.Expiration, "Expiration should not be empty")
+
+			t.Logf("Successfully obtained temporary credentials: AccessKeyId=%s", creds.AccessKeyId)
+		}
+	})
+
+	t.Run("with_duration_seconds", func(t *testing.T) {
+		resp, err := callSTSAPI(t, url.Values{
+			"Action":           {"AssumeRoleWithWebIdentity"},
+			"WebIdentityToken": {token},
+			"RoleArn":          {"arn:aws:iam::role/" + testRole},
+			"RoleSessionName":  {"integration-test-session"},
+			"DurationSeconds":  {"3600"}, // 1 hour
+		})
+		require.NoError(t, err)
+		defer resp.Body.Close()
+
+		// Verify the request is accepted (even if trust policy causes rejection)
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		// Should not fail with InvalidParameterValue for DurationSeconds
+		if resp.StatusCode != http.StatusOK {
+			var errResp STSErrorTestResponse
+			err = xml.Unmarshal(body, &errResp)
+			require.NoError(t, err, "Failed to parse error response: %s", string(body))
+			assert.NotContains(t, errResp.Error.Message, "DurationSeconds",
+				"DurationSeconds parameter should be accepted")
+		}
+	})
+}
+
+// callSTSAPI is a helper to make STS API calls
+func callSTSAPI(t *testing.T, params url.Values) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodPost, TestSTSEndpoint+"/",
+		strings.NewReader(params.Encode()))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	return client.Do(req)
+}
+
+// isSTSEndpointRunning checks if SeaweedFS STS endpoint is running
+func isSTSEndpointRunning(t *testing.T) bool {
+	client := &http.Client{Timeout: 2 * time.Second}
+	resp, err := client.Get(TestSTSEndpoint + "/status")
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+	return resp.StatusCode == http.StatusOK
+}

test/s3/iam/seaweedfs-test-realm.json

@@ -0,0 +1,107 @@
+{
+  "realm": "seaweedfs-test",
+  "enabled": true,
+  "roles": {
+    "realm": [
+      { "name": "s3-admin" },
+      { "name": "s3-read-only" },
+      { "name": "s3-write-only" },
+      { "name": "s3-read-write" }
+    ]
+  },
+  "clients": [
+    {
+      "clientId": "seaweedfs-s3",
+      "enabled": true,
+      "secret": "seaweedfs-s3-secret",
+      "publicClient": false,
+      "serviceAccountsEnabled": true,
+      "standardFlowEnabled": true,
+      "directAccessGrantsEnabled": true,
+      "fullScopeAllowed": true,
+      "redirectUris": ["*"],
+      "webOrigins": ["*"],
+      "protocol": "openid-connect",
+      "protocolMappers": [
+        {
+          "name": "realm-roles",
+          "protocol": "openid-connect",
+          "protocolMapper": "oidc-usermodel-realm-role-mapper",
+          "config": {
+            "claim.name": "roles",
+            "jsonType.label": "String",
+            "multivalued": "true",
+            "usermodel.realmRoleMapping.rolePrefix": "",
+            "access.token.claim": "true",
+            "id.token.claim": "true",
+            "userinfo.token.claim": "true"
+          }
+        },
+        {
+          "name": "audience-mapper",
+          "protocol": "openid-connect",
+          "protocolMapper": "oidc-audience-mapper",
+          "config": {
+            "included.client.audience": "seaweedfs-s3",
+            "id.token.claim": "false",
+            "access.token.claim": "true"
+          }
+        }
+      ]
+    }
+  ],
+  "users": [
+    {
+      "username": "admin-user",
+      "enabled": true,
+      "email": "admin-user@test.com",
+      "firstName": "Test",
+      "lastName": "User",
+      "emailVerified": true,
+      "requiredActions": [],
+      "realmRoles": ["s3-admin"],
+      "credentials": [
+        { "type": "password", "value": "adminuser123", "temporary": false }
+      ]
+    },
+    {
+      "username": "read-user",
+      "enabled": true,
+      "email": "read-user@test.com",
+      "firstName": "Test",
+      "lastName": "User",
+      "emailVerified": true,
+      "requiredActions": [],
+      "realmRoles": ["s3-read-only"],
+      "credentials": [
+        { "type": "password", "value": "readuser123", "temporary": false }
+      ]
+    },
+    {
+      "username": "write-user",
+      "enabled": true,
+      "email": "write-user@test.com",
+      "firstName": "Test",
+      "lastName": "User",
+      "emailVerified": true,
+      "requiredActions": [],
+      "realmRoles": ["s3-read-write"],
+      "credentials": [
+        { "type": "password", "value": "writeuser123", "temporary": false }
+      ]
+    },
+    {
+      "username": "write-only-user",
+      "enabled": true,
+      "email": "write-only-user@test.com",
+      "firstName": "Test",
+      "lastName": "User",
+      "emailVerified": true,
+      "requiredActions": [],
+      "realmRoles": ["s3-write-only"],
+      "credentials": [
+        { "type": "password", "value": "writeonlyuser123", "temporary": false }
+      ]
+    }
+  ]
+}

test/s3/iam/setup_all_tests.sh

@@ -0,0 +1,294 @@
+#!/bin/bash
+
+# Complete Test Environment Setup Script
+# This script sets up all required services and configurations for S3 IAM integration tests
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+echo -e "${BLUE}🚀 Setting up complete test environment for SeaweedFS S3 IAM...${NC}"
+echo -e "${BLUE}==========================================================${NC}"
+
+# Check prerequisites
+check_prerequisites() {
+    echo -e "${YELLOW}🔍 Checking prerequisites...${NC}"
+    
+    local missing_tools=()
+    
+    for tool in docker jq curl; do
+        if ! command -v "$tool" >/dev/null 2>&1; then
+            missing_tools+=("$tool")
+        fi
+    done
+    
+    if [ ${#missing_tools[@]} -gt 0 ]; then
+        echo -e "${RED}[FAIL] Missing required tools: ${missing_tools[*]}${NC}"
+        echo -e "${YELLOW}Please install the missing tools and try again${NC}"
+        exit 1
+    fi
+    
+    echo -e "${GREEN}[OK] All prerequisites met${NC}"
+}
+
+# Set up Keycloak for OIDC testing
+setup_keycloak() {
+    echo -e "\n${BLUE}1. Setting up Keycloak for OIDC testing...${NC}"
+    
+    if ! "${SCRIPT_DIR}/setup_keycloak.sh"; then
+        echo -e "${RED}[FAIL] Failed to set up Keycloak${NC}"
+        return 1
+    fi
+    
+    echo -e "${GREEN}[OK] Keycloak setup completed${NC}"
+}
+
+# Set up OpenLDAP for LDAP-based STS testing
+setup_ldap() {
+    echo -e "\n${BLUE}1a. Setting up OpenLDAP for STS LDAP testing...${NC}"
+    
+    # Check if LDAP container is already running
+    if docker ps --format '{{.Names}}' | grep -q '^openldap-iam-test$'; then
+        echo -e "${YELLOW}OpenLDAP container already running${NC}"
+        echo -e "${GREEN}[OK] LDAP setup completed (using existing container)${NC}"
+        return 0
+    fi
+    
+    # Remove any stopped container with the same name
+    docker rm -f openldap-iam-test 2>/dev/null || true
+    
+    # Start OpenLDAP container
+    echo -e "${YELLOW}🔧 Starting OpenLDAP container...${NC}"
+    docker run -d \
+        --name openldap-iam-test \
+        -p 389:389 \
+        -p 636:636 \
+        -e LDAP_ADMIN_PASSWORD=adminpassword \
+        -e LDAP_ORGANISATION="SeaweedFS" \
+        -e LDAP_DOMAIN="seaweedfs.test" \
+        osixia/openldap:latest || {
+            echo -e "${YELLOW}⚠️  OpenLDAP setup failed (optional for basic STS tests)${NC}"
+            return 0  # Don't fail - LDAP is optional
+        }
+    
+    # Wait for LDAP to be ready
+    echo -e "${YELLOW}⏳ Waiting for OpenLDAP to be ready...${NC}"
+    for i in $(seq 1 30); do
+        if docker exec openldap-iam-test ldapsearch -x -H ldap://localhost -b "dc=seaweedfs,dc=test" -D "cn=admin,dc=seaweedfs,dc=test" -w adminpassword "(objectClass=*)" >/dev/null 2>&1; then
+            break
+        fi
+        sleep 1
+    done
+    
+    # Add test users for LDAP STS testing
+    echo -e "${YELLOW}📝 Adding test users for LDAP STS...${NC}"
+    docker exec -i openldap-iam-test ldapadd -x -D "cn=admin,dc=seaweedfs,dc=test" -w adminpassword <<EOF 2>/dev/null || true
+dn: ou=users,dc=seaweedfs,dc=test
+objectClass: organizationalUnit
+ou: users
+
+dn: cn=testuser,ou=users,dc=seaweedfs,dc=test
+objectClass: inetOrgPerson
+cn: testuser
+sn: Test User
+uid: testuser
+userPassword: testpass
+
+dn: cn=ldapadmin,ou=users,dc=seaweedfs,dc=test
+objectClass: inetOrgPerson
+cn: ldapadmin
+sn: LDAP Admin
+uid: ldapadmin
+userPassword: ldapadminpass
+EOF
+    
+    # Verify test users were created successfully
+    echo -e "${YELLOW}🔍 Verifying LDAP test users...${NC}"
+    if docker exec openldap-iam-test ldapsearch -x -D "cn=admin,dc=seaweedfs,dc=test" -w adminpassword -b "ou=users,dc=seaweedfs,dc=test" "(cn=testuser)" cn 2>/dev/null | grep -q "cn: testuser"; then
+        echo -e "${GREEN}[OK] Test user 'testuser' verified${NC}"
+    else
+        echo -e "${RED}[WARN] Could not verify test user 'testuser' - LDAP tests may fail${NC}"
+    fi
+    
+    # Set environment for LDAP tests
+    export LDAP_URL="ldap://localhost:389"
+    export LDAP_BASE_DN="dc=seaweedfs,dc=test"
+    export LDAP_BIND_DN="cn=admin,dc=seaweedfs,dc=test"
+    export LDAP_BIND_PASSWORD="adminpassword"
+    
+    echo -e "${GREEN}[OK] LDAP setup completed${NC}"
+}
+
+# Set up SeaweedFS test cluster
+setup_seaweedfs_cluster() {
+    echo -e "\n${BLUE}2. Setting up SeaweedFS test cluster...${NC}"
+    
+    # Build SeaweedFS binary if needed
+    echo -e "${YELLOW}🔧 Building SeaweedFS binary...${NC}"
+    cd "${SCRIPT_DIR}/../../../"  # Go to seaweedfs root
+    if ! make > /dev/null 2>&1; then
+        echo -e "${RED}[FAIL] Failed to build SeaweedFS binary${NC}"
+        return 1
+    fi
+    
+    cd "${SCRIPT_DIR}"  # Return to test directory
+    
+    # Clean up any existing test data
+    echo -e "${YELLOW}🧹 Cleaning up existing test data...${NC}"
+    rm -rf test-volume-data/* 2>/dev/null || true
+    
+    echo -e "${GREEN}[OK] SeaweedFS cluster setup completed${NC}"
+}
+
+# Set up test data and configurations
+setup_test_configurations() {
+    echo -e "\n${BLUE}3. Setting up test configurations...${NC}"
+    
+    # Ensure IAM configuration is properly set up
+    if [ ! -f "${SCRIPT_DIR}/iam_config.json" ]; then
+        echo -e "${YELLOW}⚠️  IAM configuration not found, using default config${NC}"
+        cp "${SCRIPT_DIR}/iam_config.local.json" "${SCRIPT_DIR}/iam_config.json" 2>/dev/null || {
+            echo -e "${RED}[FAIL] No IAM configuration files found${NC}"
+            return 1
+        }
+    fi
+    
+    # Validate configuration
+    if ! jq . "${SCRIPT_DIR}/iam_config.json" >/dev/null; then
+        echo -e "${RED}[FAIL] Invalid IAM configuration JSON${NC}"
+        return 1
+    fi
+    
+    echo -e "${GREEN}[OK] Test configurations set up${NC}"
+}
+
+# Verify services are ready
+verify_services() {
+    echo -e "\n${BLUE}4. Verifying services are ready...${NC}"
+    
+    # Check if Keycloak is responding
+    echo -e "${YELLOW}🔍 Checking Keycloak availability...${NC}"
+    local keycloak_ready=false
+    for i in $(seq 1 30); do
+        if curl -sf "http://localhost:8080/health/ready" >/dev/null 2>&1; then
+            keycloak_ready=true
+            break
+        fi
+        if curl -sf "http://localhost:8080/realms/master" >/dev/null 2>&1; then
+            keycloak_ready=true
+            break
+        fi
+        sleep 2
+    done
+    
+    if [ "$keycloak_ready" = true ]; then
+        echo -e "${GREEN}[OK] Keycloak is ready${NC}"
+    else
+        echo -e "${YELLOW}⚠️  Keycloak may not be fully ready yet${NC}"
+        echo -e "${YELLOW}This is okay - tests will wait for Keycloak when needed${NC}"
+    fi
+    
+    echo -e "${GREEN}[OK] Service verification completed${NC}"
+}
+
+# Set up environment variables
+setup_environment() {
+    echo -e "\n${BLUE}5. Setting up environment variables...${NC}"
+    
+    export ENABLE_DISTRIBUTED_TESTS=true
+    export ENABLE_PERFORMANCE_TESTS=true
+    export ENABLE_STRESS_TESTS=true
+    export KEYCLOAK_URL="http://localhost:8080"
+    export S3_ENDPOINT="http://localhost:8333"
+    export TEST_TIMEOUT=60m
+    export CGO_ENABLED=0
+    
+    # Write environment to a file for other scripts to source
+    cat > "${SCRIPT_DIR}/.test_env" << EOF
+export ENABLE_DISTRIBUTED_TESTS=true
+export ENABLE_PERFORMANCE_TESTS=true
+export ENABLE_STRESS_TESTS=true
+export KEYCLOAK_URL="http://localhost:8080"
+export S3_ENDPOINT="http://localhost:8333"
+export TEST_TIMEOUT=60m
+export CGO_ENABLED=0
+EOF
+    
+    echo -e "${GREEN}[OK] Environment variables set${NC}"
+}
+
+# Display setup summary
+display_summary() {
+    echo -e "\n${BLUE}📊 Setup Summary${NC}"
+    echo -e "${BLUE}=================${NC}"
+    echo -e "Keycloak URL: ${KEYCLOAK_URL:-http://localhost:8080}"
+    echo -e "LDAP URL: ${LDAP_URL:-ldap://localhost:389}"
+    echo -e "S3 Endpoint: ${S3_ENDPOINT:-http://localhost:8333}"
+    echo -e "Test Timeout: ${TEST_TIMEOUT:-60m}"
+    echo -e "IAM Config: ${SCRIPT_DIR}/iam_config.json"
+    echo -e ""
+    echo -e "${GREEN}[OK] Complete test environment setup finished!${NC}"
+    echo -e "${YELLOW}💡 You can now run tests with: make run-all-tests${NC}"
+    echo -e "${YELLOW}💡 Or run specific tests with: go test -v -timeout=60m -run TestName${NC}"
+    echo -e "${YELLOW}💡 To stop Keycloak: docker stop keycloak-iam-test${NC}"
+    echo -e "${YELLOW}💡 To stop LDAP: docker stop openldap-iam-test${NC}"
+}
+
+# Main execution
+main() {
+    check_prerequisites
+    
+    # Track what was set up for cleanup on failure
+    local setup_steps=()
+    
+    if setup_keycloak; then
+        setup_steps+=("keycloak")
+    else
+        echo -e "${RED}[FAIL] Failed to set up Keycloak${NC}"
+        exit 1
+    fi
+    
+    # LDAP is optional but we try to set it up
+    setup_ldap
+    setup_steps+=("ldap")
+    
+    if setup_seaweedfs_cluster; then
+        setup_steps+=("seaweedfs")
+    else
+        echo -e "${RED}[FAIL] Failed to set up SeaweedFS cluster${NC}"
+        exit 1
+    fi
+    
+    if setup_test_configurations; then
+        setup_steps+=("config")
+    else
+        echo -e "${RED}[FAIL] Failed to set up test configurations${NC}"
+        exit 1
+    fi
+    
+    setup_environment
+    verify_services
+    display_summary
+    
+    echo -e "${GREEN}🎉 All setup completed successfully!${NC}"
+}
+
+# Cleanup on script interruption
+cleanup() {
+    echo -e "\n${YELLOW}🧹 Cleaning up on script interruption...${NC}"
+    # Note: We don't automatically stop Keycloak as it might be shared
+    echo -e "${YELLOW}💡 If you want to stop Keycloak: docker stop keycloak-iam-test${NC}"
+    exit 1
+}
+
+trap cleanup INT TERM
+
+# Execute main function
+main "$@"

test/s3/iam/setup_keycloak.sh

@@ -0,0 +1,416 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+KEYCLOAK_IMAGE="quay.io/keycloak/keycloak:26.0.7"
+CONTAINER_NAME="keycloak-iam-test"
+KEYCLOAK_PORT="8080"  # Default external port
+KEYCLOAK_INTERNAL_PORT="8080"  # Internal container port (always 8080)
+KEYCLOAK_URL="http://localhost:${KEYCLOAK_PORT}"
+
+# Realm and test fixtures expected by tests
+REALM_NAME="seaweedfs-test"
+CLIENT_ID="seaweedfs-s3"
+CLIENT_SECRET="seaweedfs-s3-secret"
+ROLE_ADMIN="s3-admin"
+ROLE_READONLY="s3-read-only"
+ROLE_WRITEONLY="s3-write-only"
+ROLE_READWRITE="s3-read-write"
+
+# User credentials (matches Docker setup script logic: removes non-alphabetic chars + "123")
+get_user_password() {
+  case "$1" in
+    "admin-user") echo "adminuser123" ;;        # "admin-user" -> "adminuser123"
+    "read-user") echo "readuser123" ;;          # "read-user" -> "readuser123"
+    "write-user") echo "writeuser123" ;;        # "write-user" -> "writeuser123"
+    "write-only-user") echo "writeonlyuser123" ;;  # "write-only-user" -> "writeonlyuser123"
+    *) echo "" ;;
+  esac
+}
+
+# List of users to create
+USERS="admin-user read-user write-user write-only-user"
+
+echo -e "${BLUE}🔧 Setting up Keycloak realm and users for SeaweedFS S3 IAM testing...${NC}"
+
+ensure_container() {
+  # Check for any existing Keycloak container and detect its port
+  local keycloak_containers=$(docker ps --format '{{.Names}}\t{{.Ports}}' | grep -E "(keycloak|quay.io/keycloak)")
+  
+  if [[ -n "$keycloak_containers" ]]; then
+    # Parse the first available Keycloak container
+    CONTAINER_NAME=$(echo "$keycloak_containers" | head -1 | awk '{print $1}')
+    
+    # Extract the external port from the port mapping using sed (compatible with older bash)
+    local port_mapping=$(echo "$keycloak_containers" | head -1 | awk '{print $2}')
+    local extracted_port=$(echo "$port_mapping" | sed -n 's/.*:\([0-9]*\)->8080.*/\1/p')
+    if [[ -n "$extracted_port" ]]; then
+      KEYCLOAK_PORT="$extracted_port"
+      KEYCLOAK_URL="http://localhost:${KEYCLOAK_PORT}"
+      echo -e "${GREEN}[OK] Using existing container '${CONTAINER_NAME}' on port ${KEYCLOAK_PORT}${NC}"
+      return 0
+    fi
+  fi
+  
+  # Fallback: check for specific container names  
+  if docker ps --format '{{.Names}}' | grep -q '^keycloak$'; then
+    CONTAINER_NAME="keycloak"
+    # Try to detect port for 'keycloak' container using docker port command
+    local ports=$(docker port keycloak 8080 2>/dev/null | head -1)
+    if [[ -n "$ports" ]]; then
+      local extracted_port=$(echo "$ports" | sed -n 's/.*:\([0-9]*\)$/\1/p')
+      if [[ -n "$extracted_port" ]]; then
+        KEYCLOAK_PORT="$extracted_port"
+        KEYCLOAK_URL="http://localhost:${KEYCLOAK_PORT}"
+      fi
+    fi
+    echo -e "${GREEN}[OK] Using existing container '${CONTAINER_NAME}' on port ${KEYCLOAK_PORT}${NC}"
+    return 0
+  fi
+  if docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
+    echo -e "${GREEN}[OK] Using existing container '${CONTAINER_NAME}'${NC}"
+    return 0
+  fi
+  echo -e "${YELLOW}🐳 Starting Keycloak container (${KEYCLOAK_IMAGE})...${NC}"
+  docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true
+  docker run -d --name "${CONTAINER_NAME}" -p "${KEYCLOAK_PORT}:8080" \
+    -e KEYCLOAK_ADMIN=admin \
+    -e KEYCLOAK_ADMIN_PASSWORD=admin \
+    -e KC_HTTP_ENABLED=true \
+    -e KC_HOSTNAME_STRICT=false \
+    -e KC_HOSTNAME_STRICT_HTTPS=false \
+    -e KC_HEALTH_ENABLED=true \
+    "${KEYCLOAK_IMAGE}" start-dev >/dev/null
+}
+
+wait_ready() {
+  echo -e "${YELLOW}⏳ Waiting for Keycloak to be ready...${NC}"
+  for i in $(seq 1 120); do
+    if curl -sf "${KEYCLOAK_URL}/health/ready" >/dev/null; then
+      echo -e "${GREEN}[OK] Keycloak health check passed${NC}"
+      return 0
+    fi
+    if curl -sf "${KEYCLOAK_URL}/realms/master" >/dev/null; then
+      echo -e "${GREEN}[OK] Keycloak master realm accessible${NC}"
+      return 0
+    fi
+    sleep 2
+  done
+  echo -e "${RED}[FAIL] Keycloak did not become ready in time${NC}"
+  exit 1
+}
+
+kcadm() {
+  # Always authenticate before each command to ensure context
+  # Try different admin passwords that might be used in different environments
+  # GitHub Actions uses "admin", local testing might use "admin123"
+  local admin_passwords=("admin" "admin123" "password")
+  local auth_success=false
+  
+  for pwd in "${admin_passwords[@]}"; do
+    if docker exec -i "${CONTAINER_NAME}" /opt/keycloak/bin/kcadm.sh config credentials --server "http://localhost:${KEYCLOAK_INTERNAL_PORT}" --realm master --user admin --password "$pwd" >/dev/null 2>&1; then
+      auth_success=true
+      break
+    fi
+  done
+  
+  if [[ "$auth_success" == false ]]; then
+    echo -e "${RED}[FAIL] Failed to authenticate with any known admin password${NC}"
+    return 1
+  fi
+  
+  docker exec -i "${CONTAINER_NAME}" /opt/keycloak/bin/kcadm.sh "$@"
+}
+
+admin_login() {
+  # This is now handled by each kcadm() call  
+  echo "Logging into http://localhost:${KEYCLOAK_INTERNAL_PORT} as user admin of realm master"
+}
+
+ensure_realm() {
+  if kcadm get realms | grep -q "${REALM_NAME}"; then
+    echo -e "${GREEN}[OK] Realm '${REALM_NAME}' already exists${NC}"
+  else
+    echo -e "${YELLOW}📝 Creating realm '${REALM_NAME}'...${NC}"
+    if kcadm create realms -s realm="${REALM_NAME}" -s enabled=true; then
+    echo -e "${GREEN}[OK] Realm created${NC}"
+    else
+      # Check if it exists now (might have been created by another process)
+      if kcadm get realms | grep -q "${REALM_NAME}"; then
+        echo -e "${GREEN}[OK] Realm '${REALM_NAME}' already exists (created concurrently)${NC}"
+      else
+        echo -e "${RED}[FAIL] Failed to create realm '${REALM_NAME}'${NC}"
+        return 1
+      fi
+    fi
+  fi
+}
+
+ensure_client() {
+  local id
+  id=$(kcadm get clients -r "${REALM_NAME}" -q clientId="${CLIENT_ID}" | jq -r '.[0].id // empty')
+  if [[ -n "${id}" ]]; then
+    echo -e "${GREEN}[OK] Client '${CLIENT_ID}' already exists${NC}"
+  else
+    echo -e "${YELLOW}📝 Creating client '${CLIENT_ID}'...${NC}"
+    kcadm create clients -r "${REALM_NAME}" \
+      -s clientId="${CLIENT_ID}" \
+      -s protocol=openid-connect \
+      -s publicClient=false \
+      -s serviceAccountsEnabled=true \
+      -s directAccessGrantsEnabled=true \
+      -s standardFlowEnabled=true \
+      -s implicitFlowEnabled=false \
+      -s secret="${CLIENT_SECRET}" >/dev/null
+    echo -e "${GREEN}[OK] Client created${NC}"
+  fi
+  
+  # Create and configure role mapper for the client
+  configure_role_mapper "${CLIENT_ID}"
+}
+
+ensure_role() {
+  local role="$1"
+  if kcadm get roles -r "${REALM_NAME}" | jq -r '.[].name' | grep -qx "${role}"; then
+    echo -e "${GREEN}[OK] Role '${role}' exists${NC}"
+  else
+    echo -e "${YELLOW}📝 Creating role '${role}'...${NC}"
+    kcadm create roles -r "${REALM_NAME}" -s name="${role}" >/dev/null
+  fi
+}
+
+ensure_user() {
+  local username="$1" password="$2"
+  local uid
+  uid=$(kcadm get users -r "${REALM_NAME}" -q username="${username}" | jq -r '.[0].id // empty')
+  if [[ -z "${uid}" ]]; then
+    echo -e "${YELLOW}📝 Creating user '${username}'...${NC}"
+    uid=$(kcadm create users -r "${REALM_NAME}" \
+      -s username="${username}" \
+      -s enabled=true \
+      -s email="${username}@seaweedfs.test" \
+      -s emailVerified=true \
+      -s firstName="${username}" \
+      -s lastName="User" \
+      -i)
+  else
+    echo -e "${GREEN}[OK] User '${username}' exists${NC}"
+  fi
+  echo -e "${YELLOW}🔑 Setting password for '${username}'...${NC}"
+  kcadm set-password -r "${REALM_NAME}" --userid "${uid}" --new-password "${password}" --temporary=false >/dev/null
+}
+
+assign_role() {
+  local username="$1" role="$2"
+  local uid rid
+  uid=$(kcadm get users -r "${REALM_NAME}" -q username="${username}" | jq -r '.[0].id')
+  rid=$(kcadm get roles -r "${REALM_NAME}" | jq -r ".[] | select(.name==\"${role}\") | .id")
+  # Check if role already assigned
+  if kcadm get "users/${uid}/role-mappings/realm" -r "${REALM_NAME}" | jq -r '.[].name' | grep -qx "${role}"; then
+    echo -e "${GREEN}[OK] User '${username}' already has role '${role}'${NC}"
+    return 0
+  fi
+  echo -e "${YELLOW}➕ Assigning role '${role}' to '${username}'...${NC}"
+  kcadm add-roles -r "${REALM_NAME}" --uid "${uid}" --rolename "${role}" >/dev/null
+}
+
+configure_role_mapper() {
+  echo -e "${YELLOW}🔧 Configuring role mapper for client '${CLIENT_ID}'...${NC}"
+  
+  # Get client's internal ID
+  local internal_id
+  internal_id=$(kcadm get clients -r "${REALM_NAME}" -q clientId="${CLIENT_ID}" | jq -r '.[0].id // empty')
+  
+  if [[ -z "${internal_id}" ]]; then
+    echo -e "${RED}[FAIL] Could not find client ${client_id} to configure role mapper${NC}"
+    return 1
+  fi
+  
+  # Check if a realm roles mapper already exists for this client
+  local existing_mapper
+  existing_mapper=$(kcadm get "clients/${internal_id}/protocol-mappers/models" -r "${REALM_NAME}" | jq -r '.[] | select(.name=="realm roles" and .protocolMapper=="oidc-usermodel-realm-role-mapper") | .id // empty')
+  
+  if [[ -n "${existing_mapper}" ]]; then
+    echo -e "${GREEN}[OK] Realm roles mapper already exists${NC}"
+  else
+    echo -e "${YELLOW}📝 Creating realm roles mapper...${NC}"
+    
+    # Create protocol mapper for realm roles
+    kcadm create "clients/${internal_id}/protocol-mappers/models" -r "${REALM_NAME}" \
+      -s name="realm roles" \
+      -s protocol="openid-connect" \
+      -s protocolMapper="oidc-usermodel-realm-role-mapper" \
+      -s consentRequired=false \
+      -s 'config."multivalued"=true' \
+      -s 'config."userinfo.token.claim"=true' \
+      -s 'config."id.token.claim"=true' \
+      -s 'config."access.token.claim"=true' \
+      -s 'config."claim.name"=roles' \
+      -s 'config."jsonType.label"=String' >/dev/null || {
+        echo -e "${RED}[FAIL] Failed to create realm roles mapper${NC}"
+        return 1
+      }
+    
+    echo -e "${GREEN}[OK] Realm roles mapper created${NC}"
+  fi
+}
+
+configure_audience_mapper() {
+  echo -e "${YELLOW}🔧 Configuring audience mapper for client '${CLIENT_ID}'...${NC}"
+  
+  # Get client's internal ID
+  local internal_id
+  internal_id=$(kcadm get clients -r "${REALM_NAME}" -q clientId="${CLIENT_ID}" | jq -r '.[0].id // empty')
+  
+  if [[ -z "${internal_id}" ]]; then
+    echo -e "${RED}[FAIL] Could not find client ${CLIENT_ID} to configure audience mapper${NC}"
+    return 1
+  fi
+  
+  # Check if an audience mapper already exists for this client
+  local existing_mapper
+  existing_mapper=$(kcadm get "clients/${internal_id}/protocol-mappers/models" -r "${REALM_NAME}" | jq -r '.[] | select(.name=="audience-mapper" and .protocolMapper=="oidc-audience-mapper") | .id // empty')
+  
+  if [[ -n "${existing_mapper}" ]]; then
+    echo -e "${GREEN}[OK] Audience mapper already exists${NC}"
+  else
+    echo -e "${YELLOW}📝 Creating audience mapper...${NC}"
+    
+    # Create protocol mapper for audience
+    kcadm create "clients/${internal_id}/protocol-mappers/models" -r "${REALM_NAME}" \
+      -s name="audience-mapper" \
+      -s protocol="openid-connect" \
+      -s protocolMapper="oidc-audience-mapper" \
+      -s consentRequired=false \
+      -s 'config."included.client.audience"='"${CLIENT_ID}" \
+      -s 'config."id.token.claim"=false' \
+      -s 'config."access.token.claim"=true' >/dev/null || {
+        echo -e "${RED}[FAIL] Failed to create audience mapper${NC}"
+        return 1
+      }
+    
+    echo -e "${GREEN}[OK] Audience mapper created${NC}"
+  fi
+}
+
+main() {
+  command -v docker >/dev/null || { echo -e "${RED}[FAIL] Docker is required${NC}"; exit 1; }
+  command -v jq >/dev/null || { echo -e "${RED}[FAIL] jq is required${NC}"; exit 1; }
+
+  ensure_container
+  echo "Keycloak URL: ${KEYCLOAK_URL}"
+  wait_ready
+  admin_login
+  ensure_realm
+  ensure_client
+  configure_role_mapper
+  configure_audience_mapper
+  ensure_role "${ROLE_ADMIN}"
+  ensure_role "${ROLE_READONLY}"
+  ensure_role "${ROLE_WRITEONLY}"
+  ensure_role "${ROLE_READWRITE}"
+
+  for u in $USERS; do
+    ensure_user "$u" "$(get_user_password "$u")"
+  done
+
+  assign_role admin-user  "${ROLE_ADMIN}"
+  assign_role read-user   "${ROLE_READONLY}"
+  assign_role write-user  "${ROLE_READWRITE}"
+
+  # Also create a dedicated write-only user for testing  
+  ensure_user write-only-user "$(get_user_password write-only-user)"
+  assign_role write-only-user "${ROLE_WRITEONLY}"
+  
+  # Copy the appropriate IAM configuration for this environment
+  setup_iam_config
+
+  # Validate the setup by testing authentication and role inclusion
+  echo -e "${YELLOW}🔍 Validating setup by testing admin-user authentication and role mapping...${NC}"
+  sleep 2
+  
+  local validation_result=$(curl -s -w "%{http_code}" -X POST "http://localhost:${KEYCLOAK_PORT}/realms/${REALM_NAME}/protocol/openid-connect/token" \
+            -H "Content-Type: application/x-www-form-urlencoded" \
+            -d "grant_type=password" \
+    -d "client_id=${CLIENT_ID}" \
+    -d "client_secret=${CLIENT_SECRET}" \
+    -d "username=admin-user" \
+    -d "password=adminuser123" \
+    -d "scope=openid profile email" \
+    -o /tmp/auth_test_response.json)
+  
+  if [[ "${validation_result: -3}" == "200" ]]; then
+    echo -e "${GREEN}[OK] Authentication validation successful${NC}"
+    
+    # Extract and decode JWT token to check for roles
+    local access_token=$(cat /tmp/auth_test_response.json | jq -r '.access_token // empty')
+    if [[ -n "${access_token}" ]]; then
+      # Decode JWT payload (second part) and check for roles
+      local payload=$(echo "${access_token}" | cut -d'.' -f2)
+      # Add padding if needed for base64 decode
+      while [[ $((${#payload} % 4)) -ne 0 ]]; do
+        payload="${payload}="
+      done
+      
+      local decoded=$(echo "${payload}" | base64 -d 2>/dev/null || echo "{}")
+      local roles=$(echo "${decoded}" | jq -r '.roles // empty' 2>/dev/null || echo "")
+      
+      if [[ -n "${roles}" && "${roles}" != "null" ]]; then
+        echo -e "${GREEN}[OK] JWT token includes roles: ${roles}${NC}"
+      else
+        echo -e "${YELLOW}⚠️  JWT token does not include 'roles' claim${NC}"
+        echo -e "${YELLOW}Decoded payload sample:${NC}"
+        echo "${decoded}" | jq '.' 2>/dev/null || echo "${decoded}"
+      fi
+    fi
+  else
+    echo -e "${RED}[FAIL] Authentication validation failed with HTTP ${validation_result: -3}${NC}"
+    echo -e "${YELLOW}Response body:${NC}"
+    cat /tmp/auth_test_response.json 2>/dev/null || echo "No response body"
+    echo -e "${YELLOW}This may indicate a setup issue that needs to be resolved${NC}"
+  fi
+  rm -f /tmp/auth_test_response.json
+  
+  echo -e "${GREEN}[OK] Keycloak test realm '${REALM_NAME}' configured${NC}"
+}
+
+setup_iam_config() {
+  echo -e "${BLUE}🔧 Setting up IAM configuration for detected environment${NC}"
+  
+  # Change to script directory to ensure config files are found
+  local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+  cd "$script_dir"
+  
+  # Choose the appropriate config based on detected port
+  local config_source
+  if [[ "${KEYCLOAK_PORT}" == "8080" ]]; then
+    config_source="iam_config.github.json"
+    echo "   Using GitHub Actions configuration (port 8080)"
+  else
+    config_source="iam_config.local.json" 
+    echo "   Using local development configuration (port ${KEYCLOAK_PORT})"
+  fi
+  
+  # Verify source config exists
+  if [[ ! -f "$config_source" ]]; then
+    echo -e "${RED}[FAIL] Config file $config_source not found in $script_dir${NC}"
+    exit 1
+  fi
+  
+  # Copy the appropriate config
+  cp "$config_source" "iam_config.json"
+  
+  local detected_issuer=$(cat iam_config.json | jq -r '.providers[] | select(.name=="keycloak") | .config.issuer')
+  echo -e "${GREEN}[OK] IAM configuration set successfully${NC}"
+  echo "   - Using config: $config_source"
+  echo "   - Keycloak issuer: $detected_issuer"
+}
+
+main "$@"

test/s3/iam/setup_keycloak_docker.sh

@@ -0,0 +1,419 @@
+#!/bin/bash
+set -e
+
+# Keycloak configuration for Docker environment
+KEYCLOAK_URL="http://keycloak:8080"
+KEYCLOAK_ADMIN_USER="admin"
+KEYCLOAK_ADMIN_PASSWORD="admin"
+REALM_NAME="seaweedfs-test"
+CLIENT_ID="seaweedfs-s3"
+CLIENT_SECRET="seaweedfs-s3-secret"
+
+echo "🔧 Setting up Keycloak realm and users for SeaweedFS S3 IAM testing..."
+echo "Keycloak URL: $KEYCLOAK_URL"
+
+# Wait for Keycloak to be ready
+echo "⏳ Waiting for Keycloak to be ready..."
+timeout 120 bash -c '
+    until curl -f "$0/health/ready" > /dev/null 2>&1; do 
+        echo "Waiting for Keycloak..."
+        sleep 5
+    done
+    echo "[OK] Keycloak health check passed"
+' "$KEYCLOAK_URL"
+
+# Download kcadm.sh if not available
+if ! command -v kcadm.sh &> /dev/null; then
+    echo "📥 Downloading Keycloak admin CLI..."
+    wget -q https://github.com/keycloak/keycloak/releases/download/26.0.7/keycloak-26.0.7.tar.gz
+    tar -xzf keycloak-26.0.7.tar.gz
+    export PATH="$PWD/keycloak-26.0.7/bin:$PATH"
+fi
+
+# Wait a bit more for admin user initialization
+echo "⏳ Waiting for admin user to be fully initialized..."
+sleep 10
+
+# Function to execute kcadm commands with retry and multiple password attempts
+kcadm() {
+    local max_retries=3
+    local retry_count=0
+    local passwords=("admin" "admin123" "password")
+    
+    while [ $retry_count -lt $max_retries ]; do
+        for password in "${passwords[@]}"; do
+            if kcadm.sh "$@" --server "$KEYCLOAK_URL" --realm master --user "$KEYCLOAK_ADMIN_USER" --password "$password" 2>/dev/null; then
+                return 0
+            fi
+        done
+        retry_count=$((retry_count + 1))
+        echo "🔄 Retry $retry_count of $max_retries..."
+        sleep 5
+    done
+    
+    echo "[FAIL] Failed to execute kcadm command after $max_retries retries"
+    return 1
+}
+
+# Create realm
+echo "📝 Creating realm '$REALM_NAME'..."
+kcadm create realms -s realm="$REALM_NAME" -s enabled=true || echo "Realm may already exist"
+echo "[OK] Realm created"
+
+# Create OIDC client
+echo "📝 Creating client '$CLIENT_ID'..."
+CLIENT_UUID=$(kcadm create clients -r "$REALM_NAME" \
+    -s clientId="$CLIENT_ID" \
+    -s secret="$CLIENT_SECRET" \
+    -s enabled=true \
+    -s serviceAccountsEnabled=true \
+    -s standardFlowEnabled=true \
+    -s directAccessGrantsEnabled=true \
+    -s 'redirectUris=["*"]' \
+    -s 'webOrigins=["*"]' \
+    -i 2>/dev/null || echo "existing-client")
+
+if [ "$CLIENT_UUID" != "existing-client" ]; then
+    echo "[OK] Client created with ID: $CLIENT_UUID"
+else
+    echo "[OK] Using existing client"
+    CLIENT_UUID=$(kcadm get clients -r "$REALM_NAME" -q clientId="$CLIENT_ID" --fields id --format csv --noquotes | tail -n +2)
+fi
+
+# Configure protocol mapper for roles
+echo "🔧 Configuring role mapper for client '$CLIENT_ID'..."
+MAPPER_CONFIG='{
+  "protocol": "openid-connect",
+  "protocolMapper": "oidc-usermodel-realm-role-mapper",
+  "name": "realm-roles",
+  "config": {
+    "claim.name": "roles",
+    "jsonType.label": "String",
+    "multivalued": "true",
+    "usermodel.realmRoleMapping.rolePrefix": ""
+  }
+}'
+
+kcadm create clients/"$CLIENT_UUID"/protocol-mappers/models -r "$REALM_NAME" -b "$MAPPER_CONFIG" 2>/dev/null || echo "[OK] Role mapper already exists"
+echo "[OK] Realm roles mapper configured"
+
+# Configure audience mapper to ensure JWT tokens have correct audience claim
+echo "🔧 Configuring audience mapper for client '$CLIENT_ID'..."
+AUDIENCE_MAPPER_CONFIG='{
+  "protocol": "openid-connect",
+  "protocolMapper": "oidc-audience-mapper",
+  "name": "audience-mapper",
+  "config": {
+    "included.client.audience": "'$CLIENT_ID'",
+    "id.token.claim": "false",
+    "access.token.claim": "true"
+  }
+}'
+
+kcadm create clients/"$CLIENT_UUID"/protocol-mappers/models -r "$REALM_NAME" -b "$AUDIENCE_MAPPER_CONFIG" 2>/dev/null || echo "[OK] Audience mapper already exists"
+echo "[OK] Audience mapper configured"
+
+# Create realm roles
+echo "📝 Creating realm roles..."
+for role in "s3-admin" "s3-read-only" "s3-write-only" "s3-read-write"; do
+    kcadm create roles -r "$REALM_NAME" -s name="$role" 2>/dev/null || echo "Role $role may already exist"
+done
+
+# Create users with roles
+declare -A USERS=(
+    ["admin-user"]="s3-admin"
+    ["read-user"]="s3-read-only"
+    ["write-user"]="s3-read-write"
+    ["write-only-user"]="s3-write-only"
+)
+
+for username in "${!USERS[@]}"; do
+    role="${USERS[$username]}"
+    password="${username//[^a-zA-Z]/}123"  # e.g., "admin-user" -> "adminuser123"
+    
+    echo "📝 Creating user '$username'..."
+    kcadm create users -r "$REALM_NAME" \
+        -s username="$username" \
+        -s enabled=true \
+        -s firstName="Test" \
+        -s lastName="User" \
+        -s email="$username@test.com" 2>/dev/null || echo "User $username may already exist"
+    
+    echo "🔑 Setting password for '$username'..."
+    kcadm set-password -r "$REALM_NAME" --username "$username" --new-password "$password"
+    
+    echo "➕ Assigning role '$role' to '$username'..."
+    kcadm add-roles -r "$REALM_NAME" --uusername "$username" --rolename "$role"
+done
+
+# Create IAM configuration for Docker environment
+echo "🔧 Setting up IAM configuration for Docker environment..."
+cat > iam_config.json << 'EOF'
+{
+  "sts": {
+    "tokenDuration": "1h",
+    "maxSessionLength": "12h",
+    "issuer": "seaweedfs-sts",
+    "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc="
+  },
+  "providers": [
+    {
+      "name": "keycloak",
+      "type": "oidc",
+      "enabled": true,
+      "config": {
+        "issuer": "http://keycloak:8080/realms/seaweedfs-test",
+        "clientId": "seaweedfs-s3",
+        "clientSecret": "seaweedfs-s3-secret",
+        "jwksUri": "http://keycloak:8080/realms/seaweedfs-test/protocol/openid-connect/certs",
+        "userInfoUri": "http://keycloak:8080/realms/seaweedfs-test/protocol/openid-connect/userinfo",
+        "scopes": ["openid", "profile", "email"],
+        "claimsMapping": {
+          "username": "preferred_username",
+          "email": "email",
+          "name": "name"
+        },
+        "roleMapping": {
+          "rules": [
+            {
+              "claim": "roles",
+              "value": "s3-admin",
+              "role": "arn:aws:iam::role/KeycloakAdminRole"
+            },
+            {
+              "claim": "roles",
+              "value": "s3-read-only",
+              "role": "arn:aws:iam::role/KeycloakReadOnlyRole"
+            },
+            {
+              "claim": "roles",
+              "value": "s3-write-only",
+              "role": "arn:aws:iam::role/KeycloakWriteOnlyRole"
+            },
+            {
+              "claim": "roles",
+              "value": "s3-read-write",
+              "role": "arn:aws:iam::role/KeycloakReadWriteRole"
+            }
+          ],
+          "defaultRole": "arn:aws:iam::role/KeycloakReadOnlyRole"
+        }
+      }
+    }
+  ],
+  "policy": {
+    "defaultEffect": "Deny"
+  },
+  "roles": [
+    {
+      "roleName": "KeycloakAdminRole",
+      "roleArn": "arn:aws:iam::role/KeycloakAdminRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"]
+          }
+        ]
+      },
+      "attachedPolicies": ["S3AdminPolicy"],
+      "description": "Admin role for Keycloak users"
+    },
+    {
+      "roleName": "KeycloakReadOnlyRole",
+      "roleArn": "arn:aws:iam::role/KeycloakReadOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"]
+          }
+        ]
+      },
+      "attachedPolicies": ["S3ReadOnlyPolicy"],
+      "description": "Read-only role for Keycloak users"
+    },
+    {
+      "roleName": "KeycloakWriteOnlyRole",
+      "roleArn": "arn:aws:iam::role/KeycloakWriteOnlyRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"]
+          }
+        ]
+      },
+      "attachedPolicies": ["S3WriteOnlyPolicy"],
+      "description": "Write-only role for Keycloak users"
+    },
+    {
+      "roleName": "KeycloakReadWriteRole",
+      "roleArn": "arn:aws:iam::role/KeycloakReadWriteRole",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": {
+              "Federated": "keycloak"
+            },
+            "Action": ["sts:AssumeRoleWithWebIdentity"]
+          }
+        ]
+      },
+      "attachedPolicies": ["S3ReadWritePolicy"],
+      "description": "Read-write role for Keycloak users"
+    }
+  ],
+  "policies": [
+    {
+      "name": "S3AdminPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": ["*"]
+          },
+          {
+            "Effect": "Allow",
+            "Action": ["sts:ValidateSession"],
+            "Resource": ["*"]
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3ReadOnlyPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "s3:GetObject",
+              "s3:ListBucket"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Allow",
+            "Action": ["sts:ValidateSession"],
+            "Resource": ["*"]
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3WriteOnlyPolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Deny",
+            "Action": [
+              "s3:GetObject",
+              "s3:ListBucket"
+            ],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Allow",
+            "Action": ["sts:ValidateSession"],
+            "Resource": ["*"]
+          }
+        ]
+      }
+    },
+    {
+      "name": "S3ReadWritePolicy",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": ["s3:*"],
+            "Resource": [
+              "arn:aws:s3:::*",
+              "arn:aws:s3:::*/*"
+            ]
+          },
+          {
+            "Effect": "Allow",
+            "Action": ["sts:ValidateSession"],
+            "Resource": ["*"]
+          }
+        ]
+      }
+    }
+  ]
+}
+EOF
+
+# Validate setup by testing authentication
+echo "🔍 Validating setup by testing admin-user authentication and role mapping..."
+KEYCLOAK_TOKEN_URL="http://keycloak:8080/realms/$REALM_NAME/protocol/openid-connect/token"
+
+# Get access token for admin-user
+ACCESS_TOKEN=$(curl -s -X POST "$KEYCLOAK_TOKEN_URL" \
+    -H "Content-Type: application/x-www-form-urlencoded" \
+    -d "grant_type=password" \
+    -d "client_id=$CLIENT_ID" \
+    -d "client_secret=$CLIENT_SECRET" \
+    -d "username=admin-user" \
+    -d "password=adminuser123" \
+    -d "scope=openid profile email" | jq -r '.access_token')
+
+if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then
+    echo "[FAIL] Failed to obtain access token"
+    exit 1
+fi
+
+echo "[OK] Authentication validation successful"
+
+# Decode and check JWT claims
+PAYLOAD=$(echo "$ACCESS_TOKEN" | cut -d'.' -f2)
+# Add padding for base64 decode
+while [ $((${#PAYLOAD} % 4)) -ne 0 ]; do
+    PAYLOAD="${PAYLOAD}="
+done
+
+CLAIMS=$(echo "$PAYLOAD" | base64 -d 2>/dev/null | jq .)
+ROLES=$(echo "$CLAIMS" | jq -r '.roles[]?')
+
+if [ -n "$ROLES" ]; then
+    echo "[OK] JWT token includes roles: [$(echo "$ROLES" | tr '\n' ',' | sed 's/,$//' | sed 's/,/, /g')]"
+else
+    echo "⚠️  No roles found in JWT token"
+fi
+
+echo "[OK] Keycloak test realm '$REALM_NAME' configured for Docker environment"
+echo "🐳 Setup complete! You can now run: docker-compose up -d"

test/s3/iam/test_config.json

@@ -0,0 +1,94 @@
+{
+  "identities": [
+    {
+      "name": "testuser",
+      "credentials": [
+        {
+          "accessKey": "test-access-key",
+          "secretKey": "test-secret-key"
+        }
+      ],
+      "actions": [
+        "Admin"
+      ]
+    },
+    {
+      "name": "readonlyuser",
+      "credentials": [
+        {
+          "accessKey": "readonly-access-key",
+          "secretKey": "readonly-secret-key"
+        }
+      ],
+      "actions": [
+        "Read"
+      ]
+    },
+    {
+      "name": "writeonlyuser",
+      "credentials": [
+        {
+          "accessKey": "writeonly-access-key",
+          "secretKey": "writeonly-secret-key"
+        }
+      ],
+      "actions": [
+        "Write"
+      ]
+    }
+  ],
+  "bucketPolicyExamples": {
+    "PublicReadPolicy": {
+      "Version": "2012-10-17",
+      "Statement": [
+        {
+          "Sid": "PublicReadGetObject",
+          "Effect": "Allow",
+          "Principal": "*",
+          "Action": "s3:GetObject",
+          "Resource": "arn:aws:s3:::example-bucket/*"
+        }
+      ]
+    },
+    "DenyDeletePolicy": {
+      "Version": "2012-10-17",
+      "Statement": [
+        {
+          "Sid": "DenyDeleteOperations",
+          "Effect": "Deny",
+          "Principal": "*",
+          "Action": [
+            "s3:DeleteObject",
+            "s3:DeleteBucket"
+          ],
+          "Resource": [
+            "arn:aws:s3:::example-bucket",
+            "arn:aws:s3:::example-bucket/*"
+          ]
+        }
+      ]
+    },
+    "IPRestrictedAccessPolicy": {
+      "Version": "2012-10-17",
+      "Statement": [
+        {
+          "Sid": "IPRestrictedAccess",
+          "Effect": "Allow",
+          "Principal": "*",
+          "Action": [
+            "s3:GetObject",
+            "s3:PutObject"
+          ],
+          "Resource": "arn:aws:s3:::example-bucket/*",
+          "Condition": {
+            "IpAddress": {
+              "aws:SourceIp": [
+                "203.0.113.0/24"
+              ]
+            }
+          }
+        }
+      ]
+    }
+  }
+}

test/s3/iam/test_iam_config.json

@@ -0,0 +1,76 @@
+{
+    "sts": {
+        "issuer": "seaweedfs-sts",
+        "signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc=",
+        "tokenDuration": "1h",
+        "maxSessionLength": "12h"
+    },
+    "policy": {
+        "defaultEffect": "Deny",
+        "storeType": "memory"
+    },
+    "roles": [
+        {
+            "roleName": "TestAdminRole",
+            "roleArn": "arn:aws:iam::123456789012:role/TestAdminRole",
+            "trustPolicy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Principal": {
+                            "Federated": "*"
+                        },
+                        "Action": [
+                            "sts:AssumeRoleWithWebIdentity"
+                        ]
+                    }
+                ]
+            },
+            "attachedPolicies": [
+                "AllowAll"
+            ]
+        },
+        {
+            "roleName": "TestReadOnlyRole",
+            "roleArn": "arn:aws:iam::123456789012:role/TestReadOnlyRole",
+            "trustPolicy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Principal": {
+                            "Federated": "*"
+                        },
+                        "Action": [
+                            "sts:AssumeRoleWithWebIdentity"
+                        ]
+                    }
+                ]
+            },
+            "attachedPolicies": [
+                "AllowAll"
+            ]
+        }
+    ],
+    "policies": [
+        {
+            "name": "AllowAll",
+            "document": {
+                "version": "2012-10-17",
+                "statement": [
+                    {
+                        "effect": "Allow",
+                        "action": [
+                            "s3:*"
+                        ],
+                        "resource": [
+                            "*"
+                        ]
+                    }
+                ]
+            }
+        }
+    ],
+    "providers": []
+}

test/s3/lifecycle/Makefile

@@ -0,0 +1,86 @@
+# S3 Lifecycle Test Makefile
+# End-to-end test of the event-driven lifecycle worker, driven by
+# the s3.lifecycle.run-shard shell command.
+
+.PHONY: help build-weed start-server stop-server test test-with-server clean health-check
+
+WEED_BINARY := ../../../weed/weed_binary
+S3_PORT := 8333
+MASTER_PORT := 9333
+VOLUME_PORT := 8080
+FILER_PORT := 8888
+# Pin the filer gRPC port off the FILER_PORT+10000 convention so any
+# code path that assumes the offset breaks here, not in production.
+FILER_GRPC_PORT := 18890
+ADMIN_PORT := 23646
+ACCESS_KEY ?= some_access_key1
+SECRET_KEY ?= some_secret_key1
+TEST_TIMEOUT := 10m
+TEST_PATTERN := TestLifecycle
+SERVER_DIR := ./test-volume-data/server-data
+
+help:
+	@echo "S3 Lifecycle Test Makefile"
+	@echo ""
+	@echo "Targets:"
+	@echo "  build-weed       - build the SeaweedFS binary"
+	@echo "  start-server     - start a local 'weed mini' cluster"
+	@echo "  stop-server      - stop the cluster"
+	@echo "  test             - run tests against an already-running cluster"
+	@echo "  test-with-server - start, run tests, stop"
+	@echo "  health-check     - is the S3 endpoint up?"
+	@echo "  clean            - remove test artifacts"
+
+build-weed:
+	@echo "Building SeaweedFS binary..."
+	@cd ../../../weed && go build -o weed_binary .
+	@chmod +x $(WEED_BINARY)
+
+start-server: build-weed
+	@echo "Starting weed mini..."
+	@rm -f weed-server.pid
+	@mkdir -p $(SERVER_DIR)
+	@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \
+		-dir=$(SERVER_DIR) \
+		-s3.port=$(S3_PORT) \
+		-filer.port.grpc=$(FILER_GRPC_PORT) \
+		> weed-test.log 2>&1 & \
+		echo $$! > weed-server.pid
+	@for i in $$(seq 1 90); do \
+		if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
+			echo "server up after $$i s"; exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "server did not start within 90s"; \
+	tail -50 weed-test.log; \
+	exit 1
+
+stop-server:
+	@if [ -f weed-server.pid ]; then \
+		PID=$$(cat weed-server.pid); \
+		kill -TERM $$PID 2>/dev/null || true; \
+		sleep 1; \
+		kill -KILL $$PID 2>/dev/null || true; \
+		rm -f weed-server.pid; \
+	fi
+
+health-check:
+	@curl -s http://localhost:$(S3_PORT) >/dev/null && echo "S3 up on $(S3_PORT)" || (echo "S3 not reachable"; exit 1)
+
+test:
+	@WEED_BINARY=$$(cd ../../../weed && pwd)/weed_binary \
+	S3_ENDPOINT=http://localhost:$(S3_PORT) \
+	S3_GRPC_ENDPOINT=localhost:$$(($(S3_PORT) + 10000)) \
+	MASTER_ENDPOINT=http://localhost:$(MASTER_PORT) \
+	FILER_GRPC_ADDRESS=localhost:$(FILER_GRPC_PORT) \
+	ADMIN_ENDPOINT=http://localhost:$(ADMIN_PORT) \
+	go test -v -timeout $(TEST_TIMEOUT) -run $(TEST_PATTERN)
+
+test-with-server: start-server
+	@$(MAKE) test || (RC=$$?; $(MAKE) stop-server; exit $$RC)
+	@$(MAKE) stop-server
+
+clean: stop-server
+	@rm -rf $(SERVER_DIR) test-volume-data
+	@rm -f weed-test.log weed-server.pid

test/s3/lifecycle/README.md

@@ -0,0 +1,38 @@
+# S3 Lifecycle Integration Tests
+
+End-to-end test of the event-driven S3 lifecycle worker, exercised through
+the `s3.lifecycle.run-shard` shell command.
+
+## Why backdate mtimes?
+
+The S3 API rejects `Expiration.Days < 1`, so a literal "wait one day"
+integration test isn't workable. Each test sets up a 1-day expiration rule,
+puts the target object, then rewrites its filer entry's `Mtime` to ~30 days
+ago via `filer_pb.UpdateEntry`. From the engine's perspective the object is
+past its expiration window the moment the shell command starts.
+
+## Running
+
+```sh
+# build the binary, start a local mini cluster, run tests, stop it
+make test-with-server
+
+# or, if a cluster is already running on the default ports
+make test
+```
+
+The test runs the shell command once with `-shards 0-15` (one filer
+subscription covering all 16 shards) rather than computing the target
+object's shard up front. This keeps the test independent of the
+`ShardID(bucket, key)` hash function — only that some shard reaches the
+deletion within the polling window.
+
+## Environment
+
+| variable             | default                  | description                |
+|----------------------|--------------------------|----------------------------|
+| `WEED_BINARY`        | _required_               | path to `weed_binary`      |
+| `S3_ENDPOINT`        | `http://localhost:8333`  | S3 API URL                 |
+| `S3_GRPC_ENDPOINT`   | `localhost:18333`        | S3 gRPC for lifecycle dispatch |
+| `MASTER_ENDPOINT`    | `http://localhost:9333`  | master HTTP                |
+| `FILER_GRPC_ADDRESS` | `localhost:18888`        | filer gRPC for `UpdateEntry` |

test/s3/lifecycle/s3_lifecycle_admin_dispatch_test.go

@@ -0,0 +1,155 @@
+package lifecycle
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/require"
+)
+
+// Drives the real admin->worker dispatch path via the run endpoint and
+// asserts the dispatch reaches the filer (no dial error) and deletes
+// the backdated object. The Makefile pins filer.port.grpc off the
+// FILER_PORT+10000 convention so a raw-address forwarding regression
+// resurfaces here.
+func TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort(t *testing.T) {
+	adminEndpoint := envOr("ADMIN_ENDPOINT", defaultAdminEndpoint)
+
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("admin-dispatch")
+	mustCreateBucket(t, c, bucket)
+	putExpirationLifecycle(t, c, bucket, "expire/", 1)
+	const oldKey = "expire/old.txt"
+	putObject(t, c, bucket, oldKey, "old")
+	backdateMtime(t, fc, bucket, oldKey, 30)
+
+	waitForLifecycleWorkerReady(t, adminEndpoint)
+
+	payload := runLifecycleJob(t, adminEndpoint, 30)
+
+	require.GreaterOrEqual(t, jsonNumber(t, payload, "detected_count"), 1)
+	require.Equal(t, 0, jsonNumber(t, payload, "error_count"),
+		"dispatched job errored — likely filer_grpc_address was raw host:httpPort.grpcPort")
+
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(oldKey),
+		})
+		return err != nil
+	}, 30*time.Second, 500*time.Millisecond,
+		"expected %s/%s to be deleted after admin-dispatched lifecycle run", bucket, oldKey)
+}
+
+// A worker-dispatched pass with rules in place but nothing due: the
+// invariant is that a pass returns on its own, so the admin never has to
+// cancel it and the executor slot is free for the next one.
+//
+// Not a regression test for the wedge this scenario comes from. Before
+// the subscription was bounded a pass ended only when some write landed
+// past its boundary, and on a shared test cluster something usually
+// does — verified: the whole suite passes on the unfixed build. The
+// deterministic guards are the dailyrun unit tests. This covers the
+// worker path end-to-end and would catch a pass that hangs
+// unconditionally.
+func TestLifecycleAdminDispatchCompletesWithNothingDue(t *testing.T) {
+	adminEndpoint := envOr("ADMIN_ENDPOINT", defaultAdminEndpoint)
+
+	c := s3Client(t)
+	bucket := uniqueBucket("admin-idle")
+	mustCreateBucket(t, c, bucket)
+	putExpirationLifecycle(t, c, bucket, "expire/", 1)
+	// Deliberately not backdated: the rule matches the prefix but nothing
+	// is due, so the pass dispatches nothing.
+	putObject(t, c, bucket, "expire/fresh.txt", "fresh")
+
+	waitForLifecycleWorkerReady(t, adminEndpoint)
+
+	// Let the writes above settle below the pass boundary, or one of them
+	// ends the pass and the wedge is masked.
+	time.Sleep(2 * time.Second)
+
+	payload := runLifecycleJob(t, adminEndpoint, 30)
+
+	require.Equal(t, 0, jsonNumber(t, payload, "canceled_count"),
+		"pass did not return on its own; the admin cancelled it at the run timeout")
+	require.GreaterOrEqual(t, jsonNumber(t, payload, "success_count"), 1,
+		"expected the dispatched pass to complete")
+	require.Equal(t, 0, jsonNumber(t, payload, "skipped_active_count"),
+		"a previous pass still held the executor slot")
+}
+
+// runLifecycleJob drives the admin's run endpoint and returns its decoded
+// response. timeoutSeconds bounds the run admin-side: a pass that never
+// returns comes back as canceled_count=1 after that long.
+func runLifecycleJob(t *testing.T, adminEndpoint string, timeoutSeconds int) map[string]any {
+	t.Helper()
+	body, err := json.Marshal(map[string]any{"timeout_seconds": timeoutSeconds})
+	require.NoError(t, err)
+	req, err := http.NewRequestWithContext(
+		context.Background(), http.MethodPost,
+		adminEndpoint+"/api/plugin/job-types/s3_lifecycle/run",
+		bytes.NewReader(body),
+	)
+	require.NoError(t, err)
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	var payload map[string]any
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&payload))
+	t.Logf("admin /api/plugin/job-types/s3_lifecycle/run response: %v", payload)
+	require.Equal(t, http.StatusOK, resp.StatusCode, "admin run endpoint failed: %v", payload)
+	return payload
+}
+
+func jsonNumber(t *testing.T, payload map[string]any, key string) int {
+	t.Helper()
+	raw, ok := payload[key]
+	require.Truef(t, ok, "response missing key %q: %v", key, payload)
+	switch v := raw.(type) {
+	case float64:
+		return int(v)
+	case json.Number:
+		n, err := v.Int64()
+		require.NoErrorf(t, err, "key %q is not an int: %v", key, raw)
+		return int(n)
+	default:
+		t.Fatalf("key %q has unexpected type %T (%v)", key, raw, raw)
+		return 0
+	}
+}
+
+func waitForLifecycleWorkerReady(t *testing.T, adminEndpoint string) {
+	t.Helper()
+	deadline := time.Now().Add(30 * time.Second)
+	for time.Now().Before(deadline) {
+		req, err := http.NewRequest(http.MethodGet, adminEndpoint+"/api/plugin/scheduler-states", nil)
+		require.NoError(t, err)
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			time.Sleep(250 * time.Millisecond)
+			continue
+		}
+		var payload any
+		_ = json.NewDecoder(resp.Body).Decode(&payload)
+		resp.Body.Close()
+		if resp.StatusCode == http.StatusOK && strings.Contains(fmt.Sprintf("%v", payload), "s3_lifecycle") {
+			return
+		}
+		time.Sleep(250 * time.Millisecond)
+	}
+	t.Fatalf("admin never reported an s3_lifecycle-capable worker")
+}

test/s3/lifecycle/s3_lifecycle_bootstrap_walk_test.go

@@ -0,0 +1,79 @@
+// Bootstrap-walk integration scenario: lifecycle config added AFTER the
+// objects exist must still expire them on the first sweep.
+package lifecycle
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleBootstrapWalkOnExistingObjects: PUT a batch of objects
+// FIRST, backdate them, THEN configure the lifecycle rule, THEN run
+// the worker. The worker must discover the pre-existing objects via
+// the bootstrap walk (BucketBootstrapper) — there were no meta-log
+// events the reader could have observed because the objects predate
+// the rule.
+//
+// Production scenario: an operator enables lifecycle on a bucket that
+// already holds a million objects from before the policy. Without a
+// bootstrap walk, only NEW writes would ever match; the existing
+// content would never expire.
+func TestLifecycleBootstrapWalkOnExistingObjects(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("bootstrap")
+	mustCreateBucket(t, c, bucket)
+
+	// 5 objects so the test exercises the walker iterating multiple
+	// entries in the same bucket directory. PUT FIRST, before any
+	// lifecycle config is set, so the meta-log doesn't carry events
+	// that match any rule.
+	const total = 5
+	keys := make([]string, total)
+	for i := 0; i < total; i++ {
+		keys[i] = fmt.Sprintf("preexisting/obj-%02d.txt", i)
+		putObject(t, c, bucket, keys[i], "content")
+		backdateMtime(t, fc, bucket, keys[i], 30)
+	}
+
+	// One out-of-prefix object that must NOT be expired. Same
+	// pre-existing PUT timing.
+	const survivor = "other/keep.txt"
+	putObject(t, c, bucket, survivor, "keep")
+	backdateMtime(t, fc, bucket, survivor, 30)
+
+	// NOW set the rule. Up to this point the worker (if it were
+	// running event-driven only) would have nothing matching to do.
+	putExpirationLifecycle(t, c, bucket, "preexisting/", 1)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// Every pre-existing in-prefix object must be expired by the
+	// bootstrap walk's discovery + dispatch.
+	for _, k := range keys {
+		k := k
+		require.Eventuallyf(t, func() bool {
+			_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+				Bucket: aws.String(bucket), Key: aws.String(k),
+			})
+			return isS3NotFound(err)
+		}, 30*time.Second, 500*time.Millisecond,
+			"pre-existing %s/%s must be discovered + expired by bootstrap walk", bucket, k)
+	}
+
+	// Out-of-prefix object stays — pin that the bootstrap walk
+	// doesn't ignore the rule's prefix filter.
+	_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(survivor),
+	})
+	require.NoError(t, err, "out-of-prefix object must survive the bootstrap walk")
+}

test/s3/lifecycle/s3_lifecycle_config_update_test.go

@@ -0,0 +1,79 @@
+// Lifecycle config update across sweeps.
+package lifecycle
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleConfigUpdateBetweenSweeps: an operator changes the
+// lifecycle rule between two shell-driven sweeps. The second sweep
+// must respect the NEW rule, not a cached version of the old one.
+//
+// Each `runLifecycleShard` invocation spawns a fresh `weed shell`
+// subprocess, so cached engine state from a previous sweep doesn't
+// persist across runs. This test pins that the freshly-loaded config
+// actually changes routing — under the new prefix only matching
+// objects expire, even if there are still backdated objects sitting
+// under the old prefix.
+func TestLifecycleConfigUpdateBetweenSweeps(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("config-update")
+	mustCreateBucket(t, c, bucket)
+
+	// Sweep 1: rule expires anything under "first/".
+	putExpirationLifecycle(t, c, bucket, "first/", 1)
+
+	const firstKey = "first/initial.txt"
+	putObject(t, c, bucket, firstKey, "first")
+	backdateMtime(t, fc, bucket, firstKey, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("sweep 1 output:\n%s", out)
+
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(firstKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "sweep 1 must expire %s", firstKey)
+
+	// Update the rule to a different prefix. The old "first/" prefix
+	// is no longer covered by any rule; objects under it must NOT be
+	// expired by sweep 2 even when backdated.
+	putExpirationLifecycle(t, c, bucket, "second/", 1)
+
+	const secondKey = "second/new.txt"
+	const oldPrefixKey = "first/post-update.txt"
+	putObject(t, c, bucket, secondKey, "second")
+	putObject(t, c, bucket, oldPrefixKey, "stale rule")
+	backdateMtime(t, fc, bucket, secondKey, 30)
+	backdateMtime(t, fc, bucket, oldPrefixKey, 30)
+
+	out = runLifecycleShard(t)
+	t.Logf("sweep 2 output:\n%s", out)
+
+	// Sweep 2 expires the new-prefix object.
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(secondKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "sweep 2 must expire %s under the new rule", secondKey)
+
+	// Sweep 2 must NOT expire the old-prefix object — the rule was
+	// replaced, not merged. A regression that caches old rules across
+	// PutBucketLifecycleConfiguration calls would fail here.
+	_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(oldPrefixKey),
+	})
+	require.NoError(t, err, "old-prefix object must survive after rule update — config replacement, not merge")
+}

test/s3/lifecycle/s3_lifecycle_delete_config_test.go

@@ -0,0 +1,67 @@
+// Operator-removes-lifecycle-config integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleDeleteBucketLifecycleStopsDispatching: when an operator
+// runs DeleteBucketLifecycle, subsequent sweeps must NOT continue
+// expiring objects under the now-removed rule. The worker reads each
+// bucket's XML config from filer state on every run; if it cached the
+// previous config across PutBucketLifecycleConfiguration → DeleteBucket
+// Lifecycle the bucket would keep losing objects.
+//
+// Setup: positive control (rule active, backdated obj expires), then
+// DeleteBucketLifecycle, then PUT + backdate a fresh object, run the
+// worker again. The fresh object must remain — no rule, no dispatch.
+func TestLifecycleDeleteBucketLifecycleStopsDispatching(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("delete-config")
+	mustCreateBucket(t, c, bucket)
+	putExpirationLifecycle(t, c, bucket, "x/", 1)
+
+	const firstKey = "x/initial.txt"
+	putObject(t, c, bucket, firstKey, "first")
+	backdateMtime(t, fc, bucket, firstKey, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("sweep 1 output (rule active):\n%s", out)
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(firstKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "positive control: %s should expire while rule is active", firstKey)
+
+	// Now remove the rule. The worker must observe the removal on the
+	// next sweep.
+	_, err := c.DeleteBucketLifecycle(context.Background(), &s3.DeleteBucketLifecycleInput{
+		Bucket: aws.String(bucket),
+	})
+	require.NoError(t, err)
+
+	const survivorKey = "x/post-delete.txt"
+	putObject(t, c, bucket, survivorKey, "should-stay")
+	backdateMtime(t, fc, bucket, survivorKey, 30)
+
+	out = runLifecycleShard(t)
+	t.Logf("sweep 2 output (rule deleted):\n%s", out)
+
+	// Wait the same window; survivor MUST still exist after the worker
+	// has had every opportunity to (incorrectly) act on it.
+	time.Sleep(2 * time.Second)
+	_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(survivorKey),
+	})
+	require.NoError(t, err, "after DeleteBucketLifecycle, sweeps must not expire %s", survivorKey)
+}

test/s3/lifecycle/s3_lifecycle_empty_bucket_test.go

@@ -0,0 +1,46 @@
+// Empty bucket sweep — no-op safety integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleEmptyBucketSweepIsNoOp: a bucket carrying lifecycle
+// config but no objects must produce a successful sweep — no hangs,
+// no errors, no dispatches. Pinned because the bootstrap walker
+// iterates bucket directories, and an empty directory is a corner of
+// that traversal that's easy to break (e.g. a slice-bounds bug on
+// the first listing returning zero entries).
+func TestLifecycleEmptyBucketSweepIsNoOp(t *testing.T) {
+	c := s3Client(t)
+	_, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("empty")
+	mustCreateBucket(t, c, bucket)
+	putExpirationLifecycle(t, c, bucket, "anywhere/", 1)
+
+	// No PUTs. The bucket is empty.
+
+	out := runLifecycleShard(t)
+	t.Logf("sweep output:\n%s", out)
+
+	// Sanity: the worker must report it loaded the bucket's config and
+	// completed without errors. The runLifecycleShard helper already
+	// fails on FATAL; pin the success markers here too so a regression
+	// that produces a half-shaped output (no completion) is caught.
+	require.Contains(t, out, "loaded lifecycle for", "worker must report config load")
+	require.Contains(t, out, "shards 0-15 complete", "worker must complete all shards")
+	require.False(t, strings.Contains(out, "FATAL"), "worker must not produce FATAL output")
+
+	// Bucket still exists after the sweep — the worker doesn't drop
+	// empty buckets as a side effect.
+	_, err := c.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: aws.String(bucket)})
+	require.NoError(t, err, "empty bucket must still exist after sweep")
+}

test/s3/lifecycle/s3_lifecycle_expiration_date_test.go

@@ -0,0 +1,75 @@
+// ExpirationDate (date-based, not Days) integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleExpirationDateInThePast: an Expiration{Date: <past>} rule
+// routes through the engine's ScanAtDate mode rather than the
+// EventDriven delay-group path. The worker's dispatcher must still fire
+// when the date has already passed at the time the worker runs. Pinning
+// this path because most tests use Days-based rules; ScanAtDate is a
+// separate compile + dispatch branch (engine.decideMode case
+// ActionKindExpirationDate) that wouldn't be exercised otherwise.
+func TestLifecycleExpirationDateInThePast(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("expdate")
+	mustCreateBucket(t, c, bucket)
+
+	// Date in the past: AWS rejects ExpirationDate in the future from
+	// being processed early but a past date is the natural integration
+	// test — every object hit by the rule is immediately eligible.
+	pastDate := time.Now().Add(-7 * 24 * time.Hour).UTC().Truncate(24 * time.Hour)
+	_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
+		Bucket: aws.String(bucket),
+		LifecycleConfiguration: &types.BucketLifecycleConfiguration{
+			Rules: []types.LifecycleRule{{
+				ID:     aws.String("expire-by-date"),
+				Status: types.ExpirationStatusEnabled,
+				Filter: &types.LifecycleRuleFilter{Prefix: aws.String("d/")},
+				Expiration: &types.LifecycleExpiration{
+					Date: aws.Time(pastDate),
+				},
+			}},
+		},
+	})
+	require.NoError(t, err)
+
+	const oldKey = "d/in-prefix.txt"
+	const otherKey = "other/skip.txt"
+	putObject(t, c, bucket, oldKey, "old")
+	putObject(t, c, bucket, otherKey, "other")
+
+	// Backdate oldKey too — defense-in-depth so the test doesn't
+	// depend on whether the worker also considers MinTriggerAge for
+	// date kinds; a fresh object with a past date should still expire,
+	// but aging it locks the assertion either way.
+	backdateMtime(t, fc, bucket, oldKey, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(oldKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "%s/%s must be expired by past-date rule", bucket, oldKey)
+
+	// Out-of-prefix object stays — the rule's Filter.Prefix gates this.
+	_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(otherKey),
+	})
+	require.NoError(t, err, "object outside the rule's prefix must remain")
+}

test/s3/lifecycle/s3_lifecycle_mpu_test.go

@@ -0,0 +1,136 @@
+// AbortIncompleteMultipartUpload integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/require"
+)
+
+// putAbortMPULifecycle wires AbortIncompleteMultipartUpload on a prefix.
+// AWS rejects DaysAfterInitiation < 1, so the test backdates the upload
+// directory to age it past the threshold.
+func putAbortMPULifecycle(t *testing.T, c *s3.Client, bucket, prefix string, days int32) {
+	t.Helper()
+	_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
+		Bucket: aws.String(bucket),
+		LifecycleConfiguration: &types.BucketLifecycleConfiguration{
+			Rules: []types.LifecycleRule{
+				{
+					ID:     aws.String("abort-stale-mpu"),
+					Status: types.ExpirationStatusEnabled,
+					Filter: &types.LifecycleRuleFilter{Prefix: aws.String(prefix)},
+					AbortIncompleteMultipartUpload: &types.AbortIncompleteMultipartUpload{
+						DaysAfterInitiation: aws.Int32(days),
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err)
+}
+
+// backdateUploadDir ages the .uploads/<uploadID>/ directory entry, which is
+// what the lifecycle worker keys ABORT_MPU off of (the upload's init-time
+// directory carries the destination key in Extended).
+func backdateUploadDir(t *testing.T, fc filer_pb.SeaweedFilerClient, bucket, uploadID string, daysOld int) {
+	t.Helper()
+	dir := bucketsPath + "/" + bucket + "/.uploads"
+	resp, err := fc.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
+		Directory: dir, Name: uploadID,
+	})
+	require.NoError(t, err, "lookup .uploads/%s", uploadID)
+	require.NotNil(t, resp.Entry)
+	require.NotNil(t, resp.Entry.Attributes)
+	require.True(t, resp.Entry.IsDirectory, ".uploads/%s should be a directory", uploadID)
+
+	resp.Entry.Attributes.Mtime = time.Now().Add(-time.Duration(daysOld) * 24 * time.Hour).Unix()
+	resp.Entry.Attributes.MtimeNs = 0
+	_, err = fc.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
+		Directory: dir, Entry: resp.Entry,
+	})
+	require.NoError(t, err)
+}
+
+// TestLifecycleAbortIncompleteMultipartUpload: an MPU left incomplete past
+// the AbortIncompleteMultipartUpload.DaysAfterInitiation threshold must be
+// aborted by the lifecycle worker. Exercises the lifecycleAbortMPU handler
+// path which is otherwise unreachable from the prefix-based expiration
+// tests (the routing keys off of `.uploads/<id>/` directory events, not
+// regular object events).
+func TestLifecycleAbortIncompleteMultipartUpload(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("abort-mpu")
+	mustCreateBucket(t, c, bucket)
+	putAbortMPULifecycle(t, c, bucket, "uploads/", 1)
+
+	// Initiate an MPU, upload a single part, then leave it. CompleteMultipart
+	// would consume the upload; AbortMultipart would explicitly abort. We do
+	// neither so the lifecycle worker is the path that ends it.
+	const key = "uploads/big.bin"
+	createOut, err := c.CreateMultipartUpload(context.Background(), &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(bucket), Key: aws.String(key),
+	})
+	require.NoError(t, err)
+	uploadID := aws.ToString(createOut.UploadId)
+	require.NotEmpty(t, uploadID)
+
+	// Upload one part so the directory carries the right shape.
+	_, err = c.UploadPart(context.Background(), &s3.UploadPartInput{
+		Bucket:     aws.String(bucket),
+		Key:        aws.String(key),
+		UploadId:   aws.String(uploadID),
+		PartNumber: aws.Int32(1),
+		Body:       strings.NewReader("part 1 contents"),
+	})
+	require.NoError(t, err)
+
+	// Sanity: ListMultipartUploads sees our pending upload before the
+	// worker runs.
+	listOut, err := c.ListMultipartUploads(context.Background(), &s3.ListMultipartUploadsInput{
+		Bucket: aws.String(bucket),
+	})
+	require.NoError(t, err)
+	var foundBefore bool
+	for _, u := range listOut.Uploads {
+		if aws.ToString(u.UploadId) == uploadID {
+			foundBefore = true
+			break
+		}
+	}
+	require.True(t, foundBefore, "upload %s should be visible before the worker runs", uploadID)
+
+	// Backdate the upload directory entry so the rule's DaysAfterInitiation
+	// threshold is satisfied. The router walks .uploads/ and emits an
+	// MPU-init event for each entry.
+	backdateUploadDir(t, fc, bucket, uploadID, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// The upload must no longer appear in ListMultipartUploads.
+	require.Eventuallyf(t, func() bool {
+		listOut, err := c.ListMultipartUploads(context.Background(), &s3.ListMultipartUploadsInput{
+			Bucket: aws.String(bucket),
+		})
+		if err != nil {
+			return false
+		}
+		for _, u := range listOut.Uploads {
+			if aws.ToString(u.UploadId) == uploadID {
+				return false
+			}
+		}
+		return true
+	}, 30*time.Second, 500*time.Millisecond, "upload %s must be aborted by the lifecycle worker", uploadID)
+}

test/s3/lifecycle/s3_lifecycle_multi_bucket_test.go

@@ -0,0 +1,90 @@
+// Multiple-bucket integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
+	"github.com/stretchr/testify/require"
+)
+
+// isS3NotFound recognizes a NotFound (NoSuchKey/404) response from the
+// AWS SDK. Treating any HeadObject error as "deleted" lets a transport
+// failure or dead endpoint mask a real bug, so callers that need to
+// prove deletion specifically should use this.
+func isS3NotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	var nsk *types.NoSuchKey
+	if errors.As(err, &nsk) {
+		return true
+	}
+	var notFound *types.NotFound
+	if errors.As(err, &notFound) {
+		return true
+	}
+	var apiErr *smithyhttp.ResponseError
+	if errors.As(err, &apiErr) {
+		return apiErr.HTTPStatusCode() == 404
+	}
+	return false
+}
+
+// TestLifecycleMultipleBucketsInOneSweep: a single shell-driven shard
+// sweep must process every bucket carrying lifecycle config, not just
+// the first one alphabetically. Pinned because the scheduler iterates
+// the buckets directory and a regression that returns early after the
+// first match would silently disable lifecycle for every later bucket.
+//
+// Two buckets, each with its own 1-day prefix-expiration rule and one
+// backdated object. After the worker runs, both objects must be gone.
+func TestLifecycleMultipleBucketsInOneSweep(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucketA := uniqueBucket("multi-a")
+	bucketB := uniqueBucket("multi-b")
+	mustCreateBucket(t, c, bucketA)
+	mustCreateBucket(t, c, bucketB)
+
+	putExpirationLifecycle(t, c, bucketA, "exp/", 1)
+	putExpirationLifecycle(t, c, bucketB, "exp/", 1)
+
+	const keyA = "exp/a.txt"
+	const keyB = "exp/b.txt"
+	putObject(t, c, bucketA, keyA, "a")
+	putObject(t, c, bucketB, keyB, "b")
+	backdateMtime(t, fc, bucketA, keyA, 30)
+	backdateMtime(t, fc, bucketB, keyB, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// Both buckets must have their objects expired in this single sweep.
+	for _, c2 := range []struct {
+		bucket, key string
+	}{
+		{bucketA, keyA},
+		{bucketB, keyB},
+	} {
+		c2 := c2
+		require.Eventuallyf(t, func() bool {
+			_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+				Bucket: aws.String(c2.bucket), Key: aws.String(c2.key),
+			})
+			// Pin: only count 404 / NoSuchKey as deletion. Any other
+			// error (transport failure, dead endpoint, auth) would
+			// otherwise mask a real bug as a "passed" test.
+			return isS3NotFound(err)
+		}, 30*time.Second, 500*time.Millisecond,
+			"%s/%s must be expired by the multi-bucket sweep", c2.bucket, c2.key)
+	}
+}

test/s3/lifecycle/s3_lifecycle_multi_rule_test.go

@@ -0,0 +1,94 @@
+// Multiple-rules-per-bucket integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleMultipleRulesInOneBucket: a single bucket carries two
+// independent Expiration rules with disjoint prefix filters and
+// different Days thresholds. Each rule must fire only on its prefix
+// and ignore the other; objects outside both prefixes must survive.
+//
+// Pinned because compile.Compile builds one CompiledAction per rule
+// per kind, all sharing the same bucket index. A bug that lets one
+// rule's prefix or threshold leak into the other (e.g. last-write-
+// wins on a shared map) would silently expire wrong objects.
+func TestLifecycleMultipleRulesInOneBucket(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("multi-rule")
+	mustCreateBucket(t, c, bucket)
+
+	_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
+		Bucket: aws.String(bucket),
+		LifecycleConfiguration: &types.BucketLifecycleConfiguration{
+			Rules: []types.LifecycleRule{
+				{
+					ID:         aws.String("logs-1d"),
+					Status:     types.ExpirationStatusEnabled,
+					Filter:     &types.LifecycleRuleFilter{Prefix: aws.String("logs/")},
+					Expiration: &types.LifecycleExpiration{Days: aws.Int32(1)},
+				},
+				{
+					ID:         aws.String("tmp-7d"),
+					Status:     types.ExpirationStatusEnabled,
+					Filter:     &types.LifecycleRuleFilter{Prefix: aws.String("tmp/")},
+					Expiration: &types.LifecycleExpiration{Days: aws.Int32(7)},
+				},
+			},
+		},
+	})
+	require.NoError(t, err)
+
+	const logsKey = "logs/access.log"
+	const tmpKey = "tmp/scratch.bin"
+	const otherKey = "data/keep.bin"
+
+	putObject(t, c, bucket, logsKey, "log entry")
+	putObject(t, c, bucket, tmpKey, "scratch")
+	putObject(t, c, bucket, otherKey, "important")
+
+	// Backdate logs/ past the 1d rule, tmp/ past the 7d rule, data/
+	// past both thresholds — but data/ matches no rule, so it must
+	// survive regardless of age.
+	backdateMtime(t, fc, bucket, logsKey, 30)
+	backdateMtime(t, fc, bucket, tmpKey, 30)
+	backdateMtime(t, fc, bucket, otherKey, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// logs/ rule fires.
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(logsKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "logs/ rule must expire %s", logsKey)
+
+	// tmp/ rule fires.
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(tmpKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "tmp/ rule must expire %s", tmpKey)
+
+	// data/ matches NEITHER rule and must survive — pinning that the
+	// per-rule prefix gate is independent and additive, not a global
+	// "any rule's prefix" check.
+	_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(otherKey),
+	})
+	require.NoError(t, err, "object outside both rule prefixes must survive")
+}

test/s3/lifecycle/s3_lifecycle_newer_noncurrent_test.go

@@ -0,0 +1,110 @@
+// NewerNoncurrentVersions integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/require"
+)
+
+// putNewerNoncurrentLifecycle keeps the N most recent noncurrent versions
+// and expires the rest. NoncurrentDays is required by the AWS XML schema
+// alongside NewerNoncurrentVersions; keeping it at 1 means the rule fires
+// as soon as a version moves past the keep-count and is older than 1 day.
+func putNewerNoncurrentLifecycle(t *testing.T, c *s3.Client, bucket, prefix string, keep int32) {
+	t.Helper()
+	_, err := c.PutBucketLifecycleConfiguration(context.Background(), &s3.PutBucketLifecycleConfigurationInput{
+		Bucket: aws.String(bucket),
+		LifecycleConfiguration: &types.BucketLifecycleConfiguration{
+			Rules: []types.LifecycleRule{
+				{
+					ID:     aws.String("keep-newest-noncurrent"),
+					Status: types.ExpirationStatusEnabled,
+					Filter: &types.LifecycleRuleFilter{Prefix: aws.String(prefix)},
+					NoncurrentVersionExpiration: &types.NoncurrentVersionExpiration{
+						NewerNoncurrentVersions: aws.Int32(keep),
+						NoncurrentDays:          aws.Int32(1),
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err)
+}
+
+// TestLifecycleNewerNoncurrentVersions: NewerNoncurrentVersions=1 keeps
+// only the most recent noncurrent version; older noncurrent versions
+// must be deleted. PUT v1, v2, v3, v4 (v4 current; v1-v3 noncurrent),
+// backdate v1, v2, v3, run the worker, expect v3 (newest noncurrent)
+// and v4 (current) to remain, v1 and v2 to be removed.
+//
+// This routes through routePointerTransition's "needs full expansion"
+// path — distinct from the per-version NoncurrentDays path because
+// the rule depends on per-version rank, not just per-version age.
+func TestLifecycleNewerNoncurrentVersions(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("newer-nc")
+	mustCreateBucket(t, c, bucket)
+	putVersioningEnabled(t, c, bucket)
+	putNewerNoncurrentLifecycle(t, c, bucket, "n/", 1)
+
+	const key = "n/obj.txt"
+	puts := make([]string, 4) // versionIds in PUT order
+	for i := range puts {
+		out, err := c.PutObject(context.Background(), &s3.PutObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(key),
+			Body: strings.NewReader(time.Now().Format(time.RFC3339Nano) + "-" + string(rune('a'+i))),
+		})
+		require.NoError(t, err)
+		puts[i] = aws.ToString(out.VersionId)
+		require.NotEmpty(t, puts[i])
+	}
+	v1, v2, v3, v4 := puts[0], puts[1], puts[2], puts[3]
+	require.NotEqual(t, v1, v2)
+	require.NotEqual(t, v3, v4)
+
+	// Backdate every noncurrent (v1, v2, v3) so all three satisfy the
+	// NoncurrentDays>=1 floor; the keep-newest-1 logic then chooses
+	// which to drop.
+	for _, vid := range []string{v1, v2, v3} {
+		backdateVersionedMtime(t, fc, bucket, key, vid, 30)
+	}
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// v1 and v2 (older noncurrent) must be expired.
+	for _, vid := range []string{v1, v2} {
+		vid := vid
+		require.Eventuallyf(t, func() bool {
+			_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+				Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(vid),
+			})
+			return err != nil
+		}, 30*time.Second, 500*time.Millisecond, "older noncurrent version %s must be expired", vid)
+	}
+
+	// v3 (newest noncurrent within the keep=1 window) must remain.
+	_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(v3),
+	})
+	require.NoError(t, err, "v3 (newest noncurrent within keep=1) must survive")
+
+	// v4 (current) must remain — current versions are immune to
+	// NoncurrentVersionExpiration regardless of how the keep-count
+	// arithmetic plays out.
+	currentHead, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(key),
+	})
+	require.NoError(t, err, "current version v4 must remain")
+	require.Equal(t, v4, aws.ToString(currentHead.VersionId))
+}

test/s3/lifecycle/s3_lifecycle_object_lock_test.go

@@ -0,0 +1,119 @@
+// Object Lock + lifecycle integration scenario.
+package lifecycle
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/require"
+)
+
+// TestLifecycleSkipsObjectLockedObjects: an object under Object Lock
+// retention must NOT be expired by the lifecycle worker. The handler's
+// enforceObjectLockProtections check (s3api_internal_lifecycle.go:47)
+// returns an error which the dispatcher classifies as
+// SKIPPED_OBJECT_LOCK. Pinning the integration end-to-end so a
+// regression in either the lock state lookup or the dispatcher's
+// outcome handling is caught.
+//
+// Companion positive control: a second object in the same bucket
+// without retention is expired by the same sweep — proves the rule
+// is firing AND that the lock check is selective, not a blanket skip.
+func TestLifecycleSkipsObjectLockedObjects(t *testing.T) {
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("objlock")
+	// Object Lock requires the bucket to be created with the flag set;
+	// it can't be enabled retroactively on an existing bucket. Use the
+	// raw API rather than mustCreateBucket here so we can pass the
+	// ObjectLockEnabledForBucket option.
+	_, err := c.CreateBucket(context.Background(), &s3.CreateBucketInput{
+		Bucket:                     aws.String(bucket),
+		ObjectLockEnabledForBucket: aws.Bool(true),
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		// Best-effort cleanup mirrors mustCreateBucket; locked objects
+		// may resist deletion but the test process continues regardless.
+		c.DeleteBucketLifecycle(context.Background(), &s3.DeleteBucketLifecycleInput{Bucket: aws.String(bucket)})
+		listOut, _ := c.ListObjectVersions(context.Background(), &s3.ListObjectVersionsInput{Bucket: aws.String(bucket)})
+		if listOut != nil {
+			for _, v := range listOut.Versions {
+				// Bypass governance retention to free the bucket for delete.
+				c.DeleteObject(context.Background(), &s3.DeleteObjectInput{
+					Bucket:                    aws.String(bucket),
+					Key:                       v.Key,
+					VersionId:                 v.VersionId,
+					BypassGovernanceRetention: aws.Bool(true),
+				})
+			}
+			for _, m := range listOut.DeleteMarkers {
+				c.DeleteObject(context.Background(), &s3.DeleteObjectInput{
+					Bucket: aws.String(bucket), Key: m.Key, VersionId: m.VersionId,
+				})
+			}
+		}
+		c.DeleteBucket(context.Background(), &s3.DeleteBucketInput{Bucket: aws.String(bucket)})
+	})
+
+	putExpirationLifecycle(t, c, bucket, "lock/", 1)
+
+	const lockedKey = "lock/protected.txt"
+	const freeKey = "lock/free.txt"
+
+	// Locked object: PUT with GOVERNANCE retention until 1h from now.
+	lockedPut, err := c.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket:                    aws.String(bucket),
+		Key:                       aws.String(lockedKey),
+		Body:                      strings.NewReader("locked"),
+		ObjectLockMode:            types.ObjectLockModeGovernance,
+		ObjectLockRetainUntilDate: aws.Time(time.Now().Add(time.Hour)),
+	})
+	require.NoError(t, err, "PUT with retention must succeed on a lock-enabled bucket")
+	require.NotEmpty(t, aws.ToString(lockedPut.VersionId))
+
+	// Free object: PUT without retention. Object Lock requires
+	// versioning, so the bucket is implicitly versioned and every PUT
+	// produces a versionId. Capture both for the version-aware
+	// backdate path.
+	freePut, err := c.PutObject(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(freeKey), Body: strings.NewReader("free"),
+	})
+	require.NoError(t, err)
+	freeVersionID := aws.ToString(freePut.VersionId)
+	require.NotEmpty(t, freeVersionID)
+
+	// Backdate both so they would otherwise both expire under the
+	// 1-day rule. The lock check is what distinguishes them.
+	// Versioning-enabled buckets store entries under .versions/v_<id>,
+	// not at the bare key path, so use backdateVersionedMtime.
+	backdateVersionedMtime(t, fc, bucket, lockedKey, aws.ToString(lockedPut.VersionId), 30)
+	backdateVersionedMtime(t, fc, bucket, freeKey, freeVersionID, 30)
+
+	out := runLifecycleShard(t)
+	t.Logf("shell output:\n%s", out)
+
+	// Free object expires (positive control).
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(freeKey),
+		})
+		return isS3NotFound(err)
+	}, 30*time.Second, 500*time.Millisecond, "unlocked object must expire (control)")
+
+	// Locked object survives — Object Lock retention overrides the
+	// lifecycle rule. HEAD without versionId returns the still-current
+	// object data, not a 404.
+	_, err = c.HeadObject(context.Background(), &s3.HeadObjectInput{
+		Bucket: aws.String(bucket), Key: aws.String(lockedKey),
+	})
+	require.NoError(t, err, "object under GOVERNANCE retention must survive the sweep")
+}
+