snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,110 @@
# SeaweedFS S3 Java SDK Compatibility Tests
This project contains Java-based integration tests for SeaweedFS S3 API compatibility.
## Overview
Tests are provided for both AWS SDK v1 and v2 to ensure compatibility with the various SDK versions commonly used in production.
## SDK Versions
| SDK | Version | Notes |
|-----|---------|-------|
| AWS SDK v1 for Java | 1.12.600 | Legacy SDK, less strict ETag validation |
| AWS SDK v2 for Java | 2.20.127 | Modern SDK with strict checksum validation |
## Running Tests
### Prerequisites
1. SeaweedFS running with S3 API enabled:
```bash
weed server -s3
```
2. Java 18+ and Maven
### Run All Tests
```bash
mvn test
```
### Run Specific Tests
```bash
# Run only ETag validation tests (AWS SDK v2)
mvn test -Dtest=ETagValidationTest
# Run with custom endpoint
mvn test -Dtest=ETagValidationTest -DS3_ENDPOINT=http://localhost:8333
```
### Environment Variables
| Variable | Default | Description |
|----------|---------|-------------|
| `S3_ENDPOINT` | `http://127.0.0.1:8333` | S3 API endpoint URL |
| `S3_ACCESS_KEY` | `some_access_key1` | Access key ID |
| `S3_SECRET_KEY` | `some_secret_key1` | Secret access key |
| `S3_REGION` | `us-east-1` | AWS region |
## Test Coverage
### ETagValidationTest (AWS SDK v2)
Tests for [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768) - ETag format validation.
| Test | Description |
|------|-------------|
| `testSmallFilePutObject` | Verify small files return pure MD5 ETag |
| `testLargeFilePutObject_Issue7768` | **Critical**: Verify large files (>8MB) return pure MD5 ETag |
| `testExtraLargeFilePutObject` | Verify very large files (>24MB) return pure MD5 ETag |
| `testMultipartUploadETag` | Verify multipart uploads return composite ETag |
| `testETagConsistency` | Verify ETag consistency across PUT/HEAD/GET |
| `testMultipleLargeFileUploads` | Stress test multiple large uploads |
### Background: Issue #7768
AWS SDK v2 for Java includes checksum validation that decodes the ETag as hexadecimal. When SeaweedFS returned composite ETags (`<md5>-<count>`) for regular `PutObject` with internally auto-chunked files, the SDK failed with:
```
java.lang.IllegalArgumentException: Invalid base 16 character: '-'
```
**Per AWS S3 specification:**
- `PutObject`: ETag is always a pure MD5 hex string (32 chars)
- `CompleteMultipartUpload`: ETag is composite format (`<md5>-<partcount>`)
The fix ensures SeaweedFS follows this specification.
## Project Structure
```
src/
├── main/java/com/seaweedfs/s3/
│ ├── PutObject.java # Example PutObject with SDK v1
│ └── HighLevelMultipartUpload.java
└── test/java/com/seaweedfs/s3/
├── PutObjectTest.java # Basic SDK v1 test
└── ETagValidationTest.java # Comprehensive SDK v2 ETag tests
```
## Validated SDK Versions
This Java test project validates:
- ✅ AWS SDK v2 for Java 2.20.127+
- ✅ AWS SDK v1 for Java 1.12.600+
Go SDK validation is performed by separate test suites:
- See [Go ETag Tests](/test/s3/etag/) for AWS SDK v2 for Go tests
- See [test/s3/SDK_COMPATIBILITY.md](/test/s3/SDK_COMPATIBILITY.md) for full SDK compatibility matrix
## Related
- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
- [Go ETag Tests](/test/s3/etag/)
- [SDK Compatibility Matrix](/test/s3/SDK_COMPATIBILITY.md)

View File

@@ -0,0 +1,85 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.seaweedfs.s3</groupId>
<artifactId>copier</artifactId>
<packaging>jar</packaging>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<aws.sdk.v1.version>1.12.600</aws.sdk.v1.version>
<aws.sdk.v2.version>2.20.127</aws.sdk.v2.version>
</properties>
<name>copier</name>
<url>http://maven.apache.org</url>
<dependencyManagement>
<dependencies>
<!-- AWS SDK v1 BOM -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-bom</artifactId>
<version>${aws.sdk.v1.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<!-- AWS SDK v2 BOM -->
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>bom</artifactId>
<version>${aws.sdk.v2.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<!-- AWS SDK v1 (for backward compatibility with existing code) -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
</dependency>
<!-- AWS SDK v2 (for modern tests with checksum validation) -->
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3-transfer-manager</artifactId>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<version>5.10.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>3.27.7</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.2.2</version>
</plugin>
</plugins>
</build>
</project>

View File

@@ -0,0 +1,78 @@
package com.seaweedfs.s3;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.amazonaws.services.s3.model.GetBucketLocationRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.amazonaws.services.s3.transfer.Upload;
import java.io.File;
public class HighLevelMultipartUpload {
public static void main(String[] args) throws Exception {
String bucketName = "javabucket";
String filePath = args[0];
File file = new File(filePath);
String keyName = "path/to/" + file.getName();
try {
AWSCredentials credentials = new BasicAWSCredentials("ANY-ACCESSKEYID", "ANY-SECRETACCESSKEY");
ClientConfiguration clientConfiguration = new ClientConfiguration();
clientConfiguration.setSignerOverride("AWSS3V4SignerType");
AmazonS3 s3Client = AmazonS3ClientBuilder
.standard()
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
"http://localhost:8333", Regions.US_WEST_1.name()))
.withPathStyleAccessEnabled(true)
.withClientConfiguration(clientConfiguration)
.withCredentials(new AWSStaticCredentialsProvider(credentials))
.build();
TransferManager tm = TransferManagerBuilder.standard()
.withS3Client(s3Client)
.build();
if (!s3Client.doesBucketExistV2(bucketName)) {
// Because the CreateBucketRequest object doesn't specify a region, the
// bucket is created in the region specified in the client.
s3Client.createBucket(new CreateBucketRequest(bucketName));
// Verify that the bucket was created by retrieving it and checking its location.
String bucketLocation = s3Client.getBucketLocation(new GetBucketLocationRequest(bucketName));
System.out.println("Bucket location: " + bucketLocation);
} else {
System.out.println("Bucket already exists");
}
// TransferManager processes all transfers asynchronously,
// so this call returns immediately.
Upload upload = tm.upload(bucketName, keyName, file);
System.out.println("Object upload started");
// Optionally, wait for the upload to finish before continuing.
upload.waitForCompletion();
System.out.println("Object upload complete");
} catch (AmazonServiceException e) {
// The call was transmitted successfully, but Amazon S3 couldn't process
// it, so it returned an error response.
e.printStackTrace();
} catch (SdkClientException e) {
// Amazon S3 couldn't be contacted for a response, or the client
// couldn't parse the response from Amazon S3.
e.printStackTrace();
}
}
}

View File

@@ -0,0 +1,101 @@
package com.seaweedfs.s3;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.util.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import java.io.File;
import java.io.IOException;
/**
* Hello world!
*/
public class PutObject {
private static Log log = LogFactory.getLog(PutObject.class);
public static void main(String[] args) {
AWSCredentials credentials = new BasicAWSCredentials("ANY-ACCESSKEYID", "ANY-SECRETACCESSKEY");
ClientConfiguration clientConfiguration = new ClientConfiguration();
clientConfiguration.setSignerOverride("AWSS3V4SignerType");
AmazonS3 s3Client = AmazonS3ClientBuilder
.standard()
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
"http://localhost:8333", Regions.US_WEST_1.name()))
.withPathStyleAccessEnabled(true)
.withClientConfiguration(clientConfiguration)
.withCredentials(new AWSStaticCredentialsProvider(credentials))
.build();
String bucketName = "javabucket";
String stringObjKeyName = "strObject2";
String fileObjKeyName = "fileObject2";
String fileName = args[0];
String stringContent = "Uploaded String Object v3";
try {
// Upload a text string as a new object.
s3Client.putObject(bucketName, stringObjKeyName, stringContent);
// Upload a file as a new object with ContentType and title specified.
PutObjectRequest request = new PutObjectRequest(bucketName, fileObjKeyName, new File(fileName));
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentType("plain/text");
metadata.addUserMetadata("x-amz-meta-title", "someTitle");
request.setMetadata(metadata);
s3Client.putObject(request);
// test reads
S3Object written = s3Client.getObject(bucketName, stringObjKeyName);
try {
String expected = IOUtils.toString(written.getObjectContent());
if (!stringContent.equals(expected)){
System.out.println("Failed to put and get back the content!");
}
} catch (IOException e) {
throw new SdkClientException("Error streaming content from S3 during download");
} finally {
IOUtils.closeQuietly(written, log);
}
// test deletes
s3Client.deleteObject(bucketName, stringObjKeyName);
// delete bucket
String tmpBucket = "tmpbucket";
s3Client.createBucket(tmpBucket);
s3Client.putObject(tmpBucket, stringObjKeyName, stringContent);
s3Client.deleteBucket(tmpBucket);
} catch (AmazonServiceException e) {
// The call was transmitted successfully, but Amazon S3 couldn't process
// it, so it returned an error response.
e.printStackTrace();
} catch (SdkClientException e) {
// Amazon S3 couldn't be contacted for a response, or the client
// couldn't parse the response from Amazon S3.
e.printStackTrace();
}
System.out.println("Hello World!");
}
}

View File

@@ -0,0 +1,439 @@
package com.seaweedfs.s3;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.*;
import java.net.URI;
import java.security.MessageDigest;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.regex.Pattern;
import static org.assertj.core.api.Assertions.*;
/**
* AWS SDK v2 Integration Tests for S3 ETag Format Validation.
*
* These tests verify that SeaweedFS returns correct ETag formats that are
* compatible with AWS SDK v2's checksum validation.
*
* Background (GitHub Issue #7768):
* AWS SDK v2 for Java validates ETags as hexadecimal MD5 hashes for PutObject
* responses. If the ETag contains non-hex characters (like '-' in composite
* format), the SDK fails with "Invalid base 16 character: '-'".
*
* Per AWS S3 specification:
* - Regular PutObject: ETag is always a pure MD5 hex string (32 chars)
* - CompleteMultipartUpload: ETag is composite format "<md5>-<partcount>"
*
* To run these tests:
* mvn test -Dtest=ETagValidationTest -DS3_ENDPOINT=http://localhost:8333
*
* Or set environment variable:
* export S3_ENDPOINT=http://localhost:8333
* mvn test -Dtest=ETagValidationTest
*/
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
@DisplayName("S3 ETag Format Validation Tests (AWS SDK v2)")
class ETagValidationTest {
// Configuration - can be overridden via system properties or environment variables
private static final String DEFAULT_ENDPOINT = "http://127.0.0.1:8333";
private static final String DEFAULT_ACCESS_KEY = "some_access_key1";
private static final String DEFAULT_SECRET_KEY = "some_secret_key1";
private static final String DEFAULT_REGION = "us-east-1";
// Auto-chunking threshold in SeaweedFS (must match s3api_object_handlers_put.go)
private static final int AUTO_CHUNK_SIZE = 8 * 1024 * 1024; // 8MB
// Test sizes
private static final int SMALL_FILE_SIZE = 1024; // 1KB
private static final int LARGE_FILE_SIZE = 10 * 1024 * 1024; // 10MB (triggers auto-chunking)
private static final int XL_FILE_SIZE = 25 * 1024 * 1024; // 25MB (multiple chunks)
private static final int MULTIPART_PART_SIZE = 5 * 1024 * 1024; // 5MB per part
// ETag format patterns
private static final Pattern PURE_MD5_PATTERN = Pattern.compile("^\"?[a-f0-9]{32}\"?$");
private static final Pattern COMPOSITE_PATTERN = Pattern.compile("^\"?[a-f0-9]{32}-\\d+\"?$");
private S3Client s3Client;
private String testBucketName;
private final SecureRandom random = new SecureRandom();
@BeforeAll
void setUp() {
String endpoint = getConfig("S3_ENDPOINT", DEFAULT_ENDPOINT);
String accessKey = getConfig("S3_ACCESS_KEY", DEFAULT_ACCESS_KEY);
String secretKey = getConfig("S3_SECRET_KEY", DEFAULT_SECRET_KEY);
String region = getConfig("S3_REGION", DEFAULT_REGION);
System.out.println("Connecting to S3 endpoint: " + endpoint);
s3Client = S3Client.builder()
.endpointOverride(URI.create(endpoint))
.credentialsProvider(StaticCredentialsProvider.create(
AwsBasicCredentials.create(accessKey, secretKey)))
.region(Region.of(region))
.forcePathStyle(true) // Required for SeaweedFS
.build();
// Create test bucket
testBucketName = "test-etag-" + UUID.randomUUID().toString().substring(0, 8);
s3Client.createBucket(CreateBucketRequest.builder()
.bucket(testBucketName)
.build());
System.out.println("Created test bucket: " + testBucketName);
}
@AfterAll
void tearDown() {
if (s3Client != null && testBucketName != null) {
try {
// Delete all objects with pagination
String continuationToken = null;
do {
ListObjectsV2Response listResp = s3Client.listObjectsV2(
ListObjectsV2Request.builder()
.bucket(testBucketName)
.continuationToken(continuationToken)
.build());
for (S3Object obj : listResp.contents()) {
s3Client.deleteObject(DeleteObjectRequest.builder()
.bucket(testBucketName)
.key(obj.key())
.build());
}
continuationToken = listResp.nextContinuationToken();
} while (continuationToken != null);
// Abort any multipart uploads
ListMultipartUploadsResponse mpResp = s3Client.listMultipartUploads(
ListMultipartUploadsRequest.builder().bucket(testBucketName).build());
for (MultipartUpload upload : mpResp.uploads()) {
s3Client.abortMultipartUpload(AbortMultipartUploadRequest.builder()
.bucket(testBucketName)
.key(upload.key())
.uploadId(upload.uploadId())
.build());
}
// Delete bucket
s3Client.deleteBucket(DeleteBucketRequest.builder()
.bucket(testBucketName)
.build());
System.out.println("Cleaned up test bucket: " + testBucketName);
} catch (Exception e) {
System.err.println("Warning: Failed to cleanup test bucket: " + e.getMessage());
}
s3Client.close();
}
}
@Test
@DisplayName("Small file PutObject should return pure MD5 ETag")
void testSmallFilePutObject() throws Exception {
byte[] testData = generateRandomData(SMALL_FILE_SIZE);
String expectedMD5 = calculateMD5Hex(testData);
String objectKey = "small-file-" + UUID.randomUUID() + ".bin";
PutObjectResponse response = s3Client.putObject(
PutObjectRequest.builder()
.bucket(testBucketName)
.key(objectKey)
.build(),
RequestBody.fromBytes(testData));
String etag = response.eTag();
System.out.println("Small file ETag: " + etag + " (expected MD5: " + expectedMD5 + ")");
assertThat(etag)
.describedAs("Small file ETag should be pure MD5")
.matches(PURE_MD5_PATTERN);
assertThat(cleanETag(etag))
.describedAs("ETag should match calculated MD5")
.isEqualTo(expectedMD5);
assertThat(etag)
.describedAs("ETag should not contain hyphen")
.doesNotContain("-");
}
/**
* Critical test for GitHub Issue #7768.
*
* This test uploads a file larger than the auto-chunking threshold (8MB),
* which triggers SeaweedFS to split the file into multiple internal chunks.
*
* Previously, this caused SeaweedFS to return a composite ETag like
* "d41d8cd98f00b204e9800998ecf8427e-2", which AWS SDK v2 rejected because
* it validates the ETag as hexadecimal and '-' is not a valid hex character.
*
* The fix ensures that regular PutObject always returns a pure MD5 ETag,
* regardless of internal chunking.
*/
@Test
@DisplayName("Large file PutObject (>8MB) should return pure MD5 ETag - Issue #7768")
void testLargeFilePutObject_Issue7768() throws Exception {
byte[] testData = generateRandomData(LARGE_FILE_SIZE);
String expectedMD5 = calculateMD5Hex(testData);
String objectKey = "large-file-" + UUID.randomUUID() + ".bin";
System.out.println("Uploading large file (" + LARGE_FILE_SIZE + " bytes, " +
"> " + AUTO_CHUNK_SIZE + " byte auto-chunk threshold)...");
// This is where Issue #7768 would manifest - SDK v2 validates ETag
PutObjectResponse response = s3Client.putObject(
PutObjectRequest.builder()
.bucket(testBucketName)
.key(objectKey)
.build(),
RequestBody.fromBytes(testData));
String etag = response.eTag();
int expectedChunks = (LARGE_FILE_SIZE / AUTO_CHUNK_SIZE) + 1;
System.out.println("Large file ETag: " + etag +
" (expected MD5: " + expectedMD5 + ", internal chunks: ~" + expectedChunks + ")");
// These assertions would fail before the fix
assertThat(etag)
.describedAs("Large file PutObject ETag MUST be pure MD5 (not composite)")
.matches(PURE_MD5_PATTERN);
assertThat(etag)
.describedAs("Large file ETag should NOT be composite format")
.doesNotMatch(COMPOSITE_PATTERN);
assertThat(etag)
.describedAs("ETag should not contain hyphen for regular PutObject")
.doesNotContain("-");
assertThat(cleanETag(etag))
.describedAs("ETag should match calculated MD5")
.isEqualTo(expectedMD5);
// Verify hex decoding works (this is what fails in Issue #7768)
assertThatCode(() -> hexToBytes(cleanETag(etag)))
.describedAs("ETag should be valid hexadecimal (AWS SDK v2 validation)")
.doesNotThrowAnyException();
}
@Test
@DisplayName("Extra large file PutObject (>24MB) should return pure MD5 ETag")
void testExtraLargeFilePutObject() throws Exception {
byte[] testData = generateRandomData(XL_FILE_SIZE);
String expectedMD5 = calculateMD5Hex(testData);
String objectKey = "xl-file-" + UUID.randomUUID() + ".bin";
int expectedChunks = (XL_FILE_SIZE / AUTO_CHUNK_SIZE) + 1;
System.out.println("Uploading XL file (" + XL_FILE_SIZE + " bytes, ~" +
expectedChunks + " internal chunks)...");
PutObjectResponse response = s3Client.putObject(
PutObjectRequest.builder()
.bucket(testBucketName)
.key(objectKey)
.build(),
RequestBody.fromBytes(testData));
String etag = response.eTag();
System.out.println("XL file ETag: " + etag);
assertThat(etag)
.describedAs("XL file PutObject ETag MUST be pure MD5")
.matches(PURE_MD5_PATTERN);
assertThat(cleanETag(etag))
.describedAs("ETag should match calculated MD5")
.isEqualTo(expectedMD5);
}
@Test
@DisplayName("Multipart upload should return composite ETag")
void testMultipartUploadETag() throws Exception {
int totalSize = 15 * 1024 * 1024; // 15MB = 3 parts
byte[] testData = generateRandomData(totalSize);
String objectKey = "multipart-file-" + UUID.randomUUID() + ".bin";
System.out.println("Performing multipart upload (" + totalSize + " bytes)...");
// Initiate multipart upload
CreateMultipartUploadResponse createResp = s3Client.createMultipartUpload(
CreateMultipartUploadRequest.builder()
.bucket(testBucketName)
.key(objectKey)
.build());
String uploadId = createResp.uploadId();
List<CompletedPart> completedParts = new ArrayList<>();
int partNumber = 1;
// Upload parts
for (int offset = 0; offset < totalSize; offset += MULTIPART_PART_SIZE) {
int end = Math.min(offset + MULTIPART_PART_SIZE, totalSize);
byte[] partData = new byte[end - offset];
System.arraycopy(testData, offset, partData, 0, partData.length);
UploadPartResponse uploadResp = s3Client.uploadPart(
UploadPartRequest.builder()
.bucket(testBucketName)
.key(objectKey)
.uploadId(uploadId)
.partNumber(partNumber)
.build(),
RequestBody.fromBytes(partData));
completedParts.add(CompletedPart.builder()
.partNumber(partNumber)
.eTag(uploadResp.eTag())
.build());
partNumber++;
}
// Complete multipart upload
CompleteMultipartUploadResponse completeResp = s3Client.completeMultipartUpload(
CompleteMultipartUploadRequest.builder()
.bucket(testBucketName)
.key(objectKey)
.uploadId(uploadId)
.multipartUpload(CompletedMultipartUpload.builder()
.parts(completedParts)
.build())
.build());
String etag = completeResp.eTag();
System.out.println("Multipart upload ETag: " + etag + " (" + completedParts.size() + " parts)");
// Multipart uploads SHOULD have composite ETag
assertThat(etag)
.describedAs("Multipart upload ETag SHOULD be composite format")
.matches(COMPOSITE_PATTERN);
assertThat(etag)
.describedAs("Multipart ETag should contain hyphen")
.contains("-");
// Verify part count in ETag
String[] parts = cleanETag(etag).split("-");
assertThat(parts).hasSize(2);
assertThat(parts[1])
.describedAs("Part count in ETag should match uploaded parts")
.isEqualTo(String.valueOf(completedParts.size()));
}
@Test
@DisplayName("ETag should be consistent across PUT, HEAD, and GET")
void testETagConsistency() throws Exception {
byte[] testData = generateRandomData(LARGE_FILE_SIZE);
String objectKey = "consistency-test-" + UUID.randomUUID() + ".bin";
// PUT
PutObjectResponse putResp = s3Client.putObject(
PutObjectRequest.builder()
.bucket(testBucketName)
.key(objectKey)
.build(),
RequestBody.fromBytes(testData));
String putETag = putResp.eTag();
// HEAD
HeadObjectResponse headResp = s3Client.headObject(
HeadObjectRequest.builder()
.bucket(testBucketName)
.key(objectKey)
.build());
String headETag = headResp.eTag();
// GET
GetObjectResponse getResp = s3Client.getObject(
GetObjectRequest.builder()
.bucket(testBucketName)
.key(objectKey)
.build())
.response();
String getETag = getResp.eTag();
System.out.println("PUT ETag: " + putETag + ", HEAD ETag: " + headETag + ", GET ETag: " + getETag);
assertThat(putETag).isEqualTo(headETag);
assertThat(putETag).isEqualTo(getETag);
}
@Test
@DisplayName("Multiple large file uploads should all return pure MD5 ETags")
void testMultipleLargeFileUploads() throws Exception {
int numFiles = 3;
for (int i = 0; i < numFiles; i++) {
byte[] testData = generateRandomData(LARGE_FILE_SIZE);
String expectedMD5 = calculateMD5Hex(testData);
String objectKey = "multi-large-" + i + "-" + UUID.randomUUID() + ".bin";
PutObjectResponse response = s3Client.putObject(
PutObjectRequest.builder()
.bucket(testBucketName)
.key(objectKey)
.build(),
RequestBody.fromBytes(testData));
String etag = response.eTag();
System.out.println("File " + i + " ETag: " + etag);
assertThat(etag)
.describedAs("File " + i + " ETag should be pure MD5")
.matches(PURE_MD5_PATTERN);
assertThat(cleanETag(etag))
.describedAs("File " + i + " ETag should match MD5")
.isEqualTo(expectedMD5);
// Validate hex decoding
assertThatCode(() -> hexToBytes(cleanETag(etag)))
.doesNotThrowAnyException();
}
}
// Helper methods
private String getConfig(String key, String defaultValue) {
String value = System.getProperty(key);
if (value == null) {
value = System.getenv(key);
}
return value != null ? value : defaultValue;
}
private byte[] generateRandomData(int size) {
byte[] data = new byte[size];
random.nextBytes(data);
return data;
}
private String calculateMD5Hex(byte[] data) throws Exception {
MessageDigest md = MessageDigest.getInstance("MD5");
byte[] digest = md.digest(data);
StringBuilder sb = new StringBuilder();
for (byte b : digest) {
sb.append(String.format("%02x", b));
}
return sb.toString();
}
private String cleanETag(String etag) {
if (etag == null) return null;
return etag.replace("\"", "");
}
private byte[] hexToBytes(String hex) {
int len = hex.length();
byte[] data = new byte[len / 2];
for (int i = 0; i < len; i += 2) {
data[i / 2] = (byte) ((Character.digit(hex.charAt(i), 16) << 4)
+ Character.digit(hex.charAt(i + 1), 16));
}
return data;
}
}

View File

@@ -0,0 +1,38 @@
package com.seaweedfs.s3;
import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
/**
* Unit test for simple App.
*/
public class PutObjectTest
extends TestCase
{
/**
* Create the test case
*
* @param testName name of the test case
*/
public PutObjectTest(String testName )
{
super( testName );
}
/**
* @return the suite of tests being tested
*/
public static Test suite()
{
return new TestSuite( PutObjectTest.class );
}
/**
* Rigorous Test :-)
*/
public void testApp()
{
assertTrue( true );
}
}