snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/s3/compatibility/fix_s3_tests_bucket_conflicts.py

@@ -0,0 +1,290 @@
+#!/usr/bin/env python3
+"""
+Patch Ceph s3-tests helpers to avoid bucket name mismatches and make bucket
+creation idempotent when a fixed bucket name is provided.
+
+Why:
+- Some tests call get_new_bucket() to get a name, then call
+  get_new_bucket_resource(name=<that name>) which unconditionally calls
+  CreateBucket again. If the bucket already exists, boto3 raises a
+  ClientError. We want to treat that as idempotent and reuse the bucket.
+- We must NOT silently generate a different bucket name when a name is
+  explicitly provided, otherwise subsequent test steps still reference the
+  original string and read from the wrong (empty) bucket.
+
+What this does:
+- get_new_bucket_resource(name=...):
+  - Try to create the exact bucket name.
+  - If error code is BucketAlreadyOwnedByYou OR BucketAlreadyExists, simply
+    reuse and return the bucket object for that SAME name.
+  - Only when name is None, generate a new unique name with retries.
+- get_new_bucket(client=None, name=None):
+  - If name is None, generate unique names with retries until creation
+    succeeds, and return the actual name string to the caller.
+
+This keeps bucket names consistent across the test helper calls and prevents
+404s or KeyErrors later in the tests that depend on that bucket name.
+"""
+
+import os
+import sys
+
+
+def patch_s3_tests_init_file(file_path: str) -> bool:
+    if not os.path.exists(file_path):
+        print(f"Error: File {file_path} not found")
+        return False
+
+    print(f"Patching {file_path}...")
+    with open(file_path, "r", encoding="utf-8") as f:
+        content = f.read()
+
+    # If already patched, skip
+    if "max_retries = 10" in content and "BucketAlreadyOwnedByYou" in content and "BucketAlreadyExists" in content:
+        print("Already patched. Skipping.")
+        return True
+
+    old_resource_func = '''def get_new_bucket_resource(name=None):
+    """
+    Get a bucket that exists and is empty.
+
+    Always recreates a bucket from scratch. This is useful to also
+    reset ACLs and such.
+    """
+    s3 = boto3.resource('s3',
+                        aws_access_key_id=config.main_access_key,
+                        aws_secret_access_key=config.main_secret_key,
+                        endpoint_url=config.default_endpoint,
+                        use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify)
+    if name is None:
+        name = get_new_bucket_name()
+    bucket = s3.Bucket(name)
+    bucket_location = bucket.create()
+    return bucket'''
+
+    new_resource_func = '''def get_new_bucket_resource(name=None):
+    """
+    Get a bucket that exists and is empty.
+
+    Always recreates a bucket from scratch. This is useful to also
+    reset ACLs and such.
+    """
+    s3 = boto3.resource('s3',
+                        aws_access_key_id=config.main_access_key,
+                        aws_secret_access_key=config.main_secret_key,
+                        endpoint_url=config.default_endpoint,
+                        use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify)
+
+    from botocore.exceptions import ClientError
+
+    # If a name is provided, do not change it. Reuse that exact bucket name.
+    if name is not None:
+        bucket = s3.Bucket(name)
+        try:
+            bucket.create()
+        except ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'):
+                # Treat as idempotent create for an explicitly provided name.
+                # We must not change the name or tests will read from the wrong bucket.
+                return bucket
+            # Other errors should surface
+            raise
+        else:
+            return bucket
+
+    # Only generate unique names when no name was provided
+    max_retries = 10
+    for attempt in range(max_retries):
+        gen_name = get_new_bucket_name()
+        bucket = s3.Bucket(gen_name)
+        try:
+            bucket.create()
+            return bucket
+        except ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyExists', 'BucketAlreadyOwnedByYou'):
+                if attempt == max_retries - 1:
+                    raise Exception(f"Failed to create unique bucket after {max_retries} attempts")
+                continue
+            else:
+                raise'''
+
+    old_client_func = '''def get_new_bucket(client=None, name=None):
+    """
+    Get a bucket that exists and is empty.
+
+    Always recreates a bucket from scratch. This is useful to also
+    reset ACLs and such.
+    """
+    if client is None:
+        client = get_client()
+    if name is None:
+        name = get_new_bucket_name()
+
+    client.create_bucket(Bucket=name)
+    return name'''
+
+    new_client_func = '''def get_new_bucket(client=None, name=None):
+    """
+    Get a bucket that exists and is empty.
+
+    Always recreates a bucket from scratch. This is useful to also
+    reset ACLs and such.
+    """
+    if client is None:
+        client = get_client()
+
+    from botocore.exceptions import ClientError
+
+    # If a name is provided, just try to create it once and fall back to idempotent reuse
+    if name is not None:
+        try:
+            client.create_bucket(Bucket=name)
+        except ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'):
+                return name
+            raise
+        else:
+            return name
+
+    # Otherwise, generate a unique name with retries and return the actual name string
+    max_retries = 10
+    for attempt in range(max_retries):
+        gen_name = get_new_bucket_name()
+        try:
+            client.create_bucket(Bucket=gen_name)
+            return gen_name
+        except ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyExists', 'BucketAlreadyOwnedByYou'):
+                if attempt == max_retries - 1:
+                    raise Exception(f"Failed to create unique bucket after {max_retries} attempts")
+                continue
+            else:
+                raise'''
+
+    updated = content
+    updated = updated.replace(old_resource_func, new_resource_func)
+    updated = updated.replace(old_client_func, new_client_func)
+
+    if updated == content:
+        print("Patterns not found; appending override implementations to end of file.")
+        append_patch = '''
+
+# --- SeaweedFS override start ---
+from botocore.exceptions import ClientError as _Sw_ClientError
+
+
+# Idempotent create for provided name; generate unique only when no name given
+# Keep the bucket name stable when provided by the caller
+
+def _sw_get_new_bucket_resource(name=None):
+    s3 = boto3.resource('s3',
+                        aws_access_key_id=config.main_access_key,
+                        aws_secret_access_key=config.main_secret_key,
+                        endpoint_url=config.default_endpoint,
+                        use_ssl=config.default_is_secure,
+                        verify=config.default_ssl_verify)
+    if name is not None:
+        bucket = s3.Bucket(name)
+        try:
+            bucket.create()
+        except _Sw_ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'):
+                return bucket
+            raise
+        else:
+            return bucket
+    # name not provided: generate unique
+    max_retries = 10
+    for attempt in range(max_retries):
+        gen_name = get_new_bucket_name()
+        bucket = s3.Bucket(gen_name)
+        try:
+            bucket.create()
+            return bucket
+        except _Sw_ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyExists', 'BucketAlreadyOwnedByYou'):
+                if attempt == max_retries - 1:
+                    raise Exception(f"Failed to create unique bucket after {max_retries} attempts")
+                continue
+            else:
+                raise
+
+
+from botocore.exceptions import ClientError as _Sw2_ClientError
+
+
+def _sw_get_new_bucket(client=None, name=None):
+    if client is None:
+        client = get_client()
+    if name is not None:
+        try:
+            client.create_bucket(Bucket=name)
+        except _Sw2_ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'):
+                return name
+            raise
+        else:
+            return name
+    max_retries = 10
+    for attempt in range(max_retries):
+        gen_name = get_new_bucket_name()
+        try:
+            client.create_bucket(Bucket=gen_name)
+            return gen_name
+        except _Sw2_ClientError as e:
+            code = e.response.get('Error', {}).get('Code')
+            if code in ('BucketAlreadyExists', 'BucketAlreadyOwnedByYou'):
+                if attempt == max_retries - 1:
+                    raise Exception(f"Failed to create unique bucket after {max_retries} attempts")
+                continue
+            else:
+                raise
+
+# Override original helper functions
+get_new_bucket_resource = _sw_get_new_bucket_resource
+get_new_bucket = _sw_get_new_bucket
+# --- SeaweedFS override end ---
+'''
+        with open(file_path, "a", encoding="utf-8") as f:
+            f.write(append_patch)
+        print("Appended override implementations.")
+        return True
+
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(updated)
+
+    print("Successfully patched s3-tests helpers.")
+    return True
+
+
+def main() -> int:
+    s3_tests_path = os.environ.get("S3_TESTS_PATH", "s3-tests")
+    init_file_path = os.path.join(s3_tests_path, "s3tests", "functional", "__init__.py")
+    print("Applying s3-tests patch for bucket creation idempotency...")
+    print(f"Target repo path: {s3_tests_path}")
+    if not os.path.exists(s3_tests_path):
+        print(f"Warning: s3-tests directory not found at {s3_tests_path}")
+        print("Skipping patch - directory structure may have changed in the upstream repository")
+        return 0  # Return success to not break CI
+    if not os.path.exists(init_file_path):
+        print(f"Warning: Target file {init_file_path} not found")
+        print("This may indicate the s3-tests repository structure has changed.")
+        print("Skipping patch - tests may still work without it")
+        return 0  # Return success to not break CI
+    ok = patch_s3_tests_init_file(init_file_path)
+    return 0 if ok else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+
+