snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
110
other/java/s3copier/README.md
Normal file
110
other/java/s3copier/README.md
Normal file
@@ -0,0 +1,110 @@
|
||||
# SeaweedFS S3 Java SDK Compatibility Tests
|
||||
|
||||
This project contains Java-based integration tests for SeaweedFS S3 API compatibility.
|
||||
|
||||
## Overview
|
||||
|
||||
Tests are provided for both AWS SDK v1 and v2 to ensure compatibility with the various SDK versions commonly used in production.
|
||||
|
||||
## SDK Versions
|
||||
|
||||
| SDK | Version | Notes |
|
||||
|-----|---------|-------|
|
||||
| AWS SDK v1 for Java | 1.12.600 | Legacy SDK, less strict ETag validation |
|
||||
| AWS SDK v2 for Java | 2.20.127 | Modern SDK with strict checksum validation |
|
||||
|
||||
## Running Tests
|
||||
|
||||
### Prerequisites
|
||||
|
||||
1. SeaweedFS running with S3 API enabled:
|
||||
```bash
|
||||
weed server -s3
|
||||
```
|
||||
|
||||
2. Java 18+ and Maven
|
||||
|
||||
### Run All Tests
|
||||
|
||||
```bash
|
||||
mvn test
|
||||
```
|
||||
|
||||
### Run Specific Tests
|
||||
|
||||
```bash
|
||||
# Run only ETag validation tests (AWS SDK v2)
|
||||
mvn test -Dtest=ETagValidationTest
|
||||
|
||||
# Run with custom endpoint
|
||||
mvn test -Dtest=ETagValidationTest -DS3_ENDPOINT=http://localhost:8333
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `S3_ENDPOINT` | `http://127.0.0.1:8333` | S3 API endpoint URL |
|
||||
| `S3_ACCESS_KEY` | `some_access_key1` | Access key ID |
|
||||
| `S3_SECRET_KEY` | `some_secret_key1` | Secret access key |
|
||||
| `S3_REGION` | `us-east-1` | AWS region |
|
||||
|
||||
## Test Coverage
|
||||
|
||||
### ETagValidationTest (AWS SDK v2)
|
||||
|
||||
Tests for [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768) - ETag format validation.
|
||||
|
||||
| Test | Description |
|
||||
|------|-------------|
|
||||
| `testSmallFilePutObject` | Verify small files return pure MD5 ETag |
|
||||
| `testLargeFilePutObject_Issue7768` | **Critical**: Verify large files (>8MB) return pure MD5 ETag |
|
||||
| `testExtraLargeFilePutObject` | Verify very large files (>24MB) return pure MD5 ETag |
|
||||
| `testMultipartUploadETag` | Verify multipart uploads return composite ETag |
|
||||
| `testETagConsistency` | Verify ETag consistency across PUT/HEAD/GET |
|
||||
| `testMultipleLargeFileUploads` | Stress test multiple large uploads |
|
||||
|
||||
### Background: Issue #7768
|
||||
|
||||
AWS SDK v2 for Java includes checksum validation that decodes the ETag as hexadecimal. When SeaweedFS returned composite ETags (`<md5>-<count>`) for regular `PutObject` with internally auto-chunked files, the SDK failed with:
|
||||
|
||||
```
|
||||
java.lang.IllegalArgumentException: Invalid base 16 character: '-'
|
||||
```
|
||||
|
||||
**Per AWS S3 specification:**
|
||||
- `PutObject`: ETag is always a pure MD5 hex string (32 chars)
|
||||
- `CompleteMultipartUpload`: ETag is composite format (`<md5>-<partcount>`)
|
||||
|
||||
The fix ensures SeaweedFS follows this specification.
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
src/
|
||||
├── main/java/com/seaweedfs/s3/
|
||||
│ ├── PutObject.java # Example PutObject with SDK v1
|
||||
│ └── HighLevelMultipartUpload.java
|
||||
└── test/java/com/seaweedfs/s3/
|
||||
├── PutObjectTest.java # Basic SDK v1 test
|
||||
└── ETagValidationTest.java # Comprehensive SDK v2 ETag tests
|
||||
```
|
||||
|
||||
## Validated SDK Versions
|
||||
|
||||
This Java test project validates:
|
||||
|
||||
- ✅ AWS SDK v2 for Java 2.20.127+
|
||||
- ✅ AWS SDK v1 for Java 1.12.600+
|
||||
|
||||
Go SDK validation is performed by separate test suites:
|
||||
- See [Go ETag Tests](/test/s3/etag/) for AWS SDK v2 for Go tests
|
||||
- See [test/s3/SDK_COMPATIBILITY.md](/test/s3/SDK_COMPATIBILITY.md) for full SDK compatibility matrix
|
||||
|
||||
## Related
|
||||
|
||||
- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
|
||||
- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
|
||||
- [Go ETag Tests](/test/s3/etag/)
|
||||
- [SDK Compatibility Matrix](/test/s3/SDK_COMPATIBILITY.md)
|
||||
|
||||
85
other/java/s3copier/pom.xml
Normal file
85
other/java/s3copier/pom.xml
Normal file
@@ -0,0 +1,85 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>com.seaweedfs.s3</groupId>
|
||||
<artifactId>copier</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<properties>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<aws.sdk.v1.version>1.12.600</aws.sdk.v1.version>
|
||||
<aws.sdk.v2.version>2.20.127</aws.sdk.v2.version>
|
||||
</properties>
|
||||
<name>copier</name>
|
||||
<url>http://maven.apache.org</url>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<!-- AWS SDK v1 BOM -->
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-bom</artifactId>
|
||||
<version>${aws.sdk.v1.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
<!-- AWS SDK v2 BOM -->
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>bom</artifactId>
|
||||
<version>${aws.sdk.v2.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<dependencies>
|
||||
<!-- AWS SDK v1 (for backward compatibility with existing code) -->
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-s3</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- AWS SDK v2 (for modern tests with checksum validation) -->
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>s3</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>s3-transfer-manager</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- Test dependencies -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.13.2</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<version>5.10.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.assertj</groupId>
|
||||
<artifactId>assertj-core</artifactId>
|
||||
<version>3.27.7</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>3.2.2</version>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
@@ -0,0 +1,78 @@
|
||||
package com.seaweedfs.s3;
|
||||
|
||||
import com.amazonaws.AmazonServiceException;
|
||||
import com.amazonaws.ClientConfiguration;
|
||||
import com.amazonaws.SdkClientException;
|
||||
import com.amazonaws.auth.AWSCredentials;
|
||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.client.builder.AwsClientBuilder;
|
||||
import com.amazonaws.regions.Regions;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||
import com.amazonaws.services.s3.model.CreateBucketRequest;
|
||||
import com.amazonaws.services.s3.model.GetBucketLocationRequest;
|
||||
import com.amazonaws.services.s3.transfer.TransferManager;
|
||||
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
|
||||
import com.amazonaws.services.s3.transfer.Upload;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
public class HighLevelMultipartUpload {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
String bucketName = "javabucket";
|
||||
String filePath = args[0];
|
||||
File file = new File(filePath);
|
||||
String keyName = "path/to/" + file.getName();
|
||||
|
||||
try {
|
||||
AWSCredentials credentials = new BasicAWSCredentials("ANY-ACCESSKEYID", "ANY-SECRETACCESSKEY");
|
||||
ClientConfiguration clientConfiguration = new ClientConfiguration();
|
||||
clientConfiguration.setSignerOverride("AWSS3V4SignerType");
|
||||
|
||||
AmazonS3 s3Client = AmazonS3ClientBuilder
|
||||
.standard()
|
||||
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
|
||||
"http://localhost:8333", Regions.US_WEST_1.name()))
|
||||
.withPathStyleAccessEnabled(true)
|
||||
.withClientConfiguration(clientConfiguration)
|
||||
.withCredentials(new AWSStaticCredentialsProvider(credentials))
|
||||
.build();
|
||||
|
||||
TransferManager tm = TransferManagerBuilder.standard()
|
||||
.withS3Client(s3Client)
|
||||
.build();
|
||||
|
||||
|
||||
if (!s3Client.doesBucketExistV2(bucketName)) {
|
||||
// Because the CreateBucketRequest object doesn't specify a region, the
|
||||
// bucket is created in the region specified in the client.
|
||||
s3Client.createBucket(new CreateBucketRequest(bucketName));
|
||||
|
||||
// Verify that the bucket was created by retrieving it and checking its location.
|
||||
String bucketLocation = s3Client.getBucketLocation(new GetBucketLocationRequest(bucketName));
|
||||
System.out.println("Bucket location: " + bucketLocation);
|
||||
} else {
|
||||
System.out.println("Bucket already exists");
|
||||
}
|
||||
|
||||
// TransferManager processes all transfers asynchronously,
|
||||
// so this call returns immediately.
|
||||
Upload upload = tm.upload(bucketName, keyName, file);
|
||||
System.out.println("Object upload started");
|
||||
|
||||
// Optionally, wait for the upload to finish before continuing.
|
||||
upload.waitForCompletion();
|
||||
System.out.println("Object upload complete");
|
||||
} catch (AmazonServiceException e) {
|
||||
// The call was transmitted successfully, but Amazon S3 couldn't process
|
||||
// it, so it returned an error response.
|
||||
e.printStackTrace();
|
||||
} catch (SdkClientException e) {
|
||||
// Amazon S3 couldn't be contacted for a response, or the client
|
||||
// couldn't parse the response from Amazon S3.
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package com.seaweedfs.s3;
|
||||
|
||||
import com.amazonaws.AmazonServiceException;
|
||||
import com.amazonaws.ClientConfiguration;
|
||||
import com.amazonaws.SdkClientException;
|
||||
import com.amazonaws.auth.AWSCredentials;
|
||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.client.builder.AwsClientBuilder;
|
||||
import com.amazonaws.regions.Regions;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||
import com.amazonaws.services.s3.model.ObjectMetadata;
|
||||
import com.amazonaws.services.s3.model.PutObjectRequest;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import com.amazonaws.util.IOUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Hello world!
|
||||
*/
|
||||
public class PutObject {
|
||||
|
||||
private static Log log = LogFactory.getLog(PutObject.class);
|
||||
|
||||
public static void main(String[] args) {
|
||||
|
||||
AWSCredentials credentials = new BasicAWSCredentials("ANY-ACCESSKEYID", "ANY-SECRETACCESSKEY");
|
||||
ClientConfiguration clientConfiguration = new ClientConfiguration();
|
||||
clientConfiguration.setSignerOverride("AWSS3V4SignerType");
|
||||
|
||||
AmazonS3 s3Client = AmazonS3ClientBuilder
|
||||
.standard()
|
||||
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
|
||||
"http://localhost:8333", Regions.US_WEST_1.name()))
|
||||
.withPathStyleAccessEnabled(true)
|
||||
.withClientConfiguration(clientConfiguration)
|
||||
.withCredentials(new AWSStaticCredentialsProvider(credentials))
|
||||
.build();
|
||||
|
||||
String bucketName = "javabucket";
|
||||
String stringObjKeyName = "strObject2";
|
||||
String fileObjKeyName = "fileObject2";
|
||||
String fileName = args[0];
|
||||
|
||||
String stringContent = "Uploaded String Object v3";
|
||||
try {
|
||||
|
||||
// Upload a text string as a new object.
|
||||
s3Client.putObject(bucketName, stringObjKeyName, stringContent);
|
||||
|
||||
// Upload a file as a new object with ContentType and title specified.
|
||||
PutObjectRequest request = new PutObjectRequest(bucketName, fileObjKeyName, new File(fileName));
|
||||
ObjectMetadata metadata = new ObjectMetadata();
|
||||
metadata.setContentType("plain/text");
|
||||
metadata.addUserMetadata("x-amz-meta-title", "someTitle");
|
||||
request.setMetadata(metadata);
|
||||
s3Client.putObject(request);
|
||||
|
||||
// test reads
|
||||
S3Object written = s3Client.getObject(bucketName, stringObjKeyName);
|
||||
try {
|
||||
String expected = IOUtils.toString(written.getObjectContent());
|
||||
|
||||
if (!stringContent.equals(expected)){
|
||||
System.out.println("Failed to put and get back the content!");
|
||||
}
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new SdkClientException("Error streaming content from S3 during download");
|
||||
} finally {
|
||||
IOUtils.closeQuietly(written, log);
|
||||
}
|
||||
|
||||
// test deletes
|
||||
s3Client.deleteObject(bucketName, stringObjKeyName);
|
||||
|
||||
|
||||
// delete bucket
|
||||
String tmpBucket = "tmpbucket";
|
||||
s3Client.createBucket(tmpBucket);
|
||||
s3Client.putObject(tmpBucket, stringObjKeyName, stringContent);
|
||||
s3Client.deleteBucket(tmpBucket);
|
||||
|
||||
} catch (AmazonServiceException e) {
|
||||
// The call was transmitted successfully, but Amazon S3 couldn't process
|
||||
// it, so it returned an error response.
|
||||
e.printStackTrace();
|
||||
} catch (SdkClientException e) {
|
||||
// Amazon S3 couldn't be contacted for a response, or the client
|
||||
// couldn't parse the response from Amazon S3.
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
System.out.println("Hello World!");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,439 @@
|
||||
package com.seaweedfs.s3;
|
||||
|
||||
import org.junit.jupiter.api.*;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.*;
|
||||
|
||||
import java.net.URI;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.SecureRandom;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static org.assertj.core.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* AWS SDK v2 Integration Tests for S3 ETag Format Validation.
|
||||
*
|
||||
* These tests verify that SeaweedFS returns correct ETag formats that are
|
||||
* compatible with AWS SDK v2's checksum validation.
|
||||
*
|
||||
* Background (GitHub Issue #7768):
|
||||
* AWS SDK v2 for Java validates ETags as hexadecimal MD5 hashes for PutObject
|
||||
* responses. If the ETag contains non-hex characters (like '-' in composite
|
||||
* format), the SDK fails with "Invalid base 16 character: '-'".
|
||||
*
|
||||
* Per AWS S3 specification:
|
||||
* - Regular PutObject: ETag is always a pure MD5 hex string (32 chars)
|
||||
* - CompleteMultipartUpload: ETag is composite format "<md5>-<partcount>"
|
||||
*
|
||||
* To run these tests:
|
||||
* mvn test -Dtest=ETagValidationTest -DS3_ENDPOINT=http://localhost:8333
|
||||
*
|
||||
* Or set environment variable:
|
||||
* export S3_ENDPOINT=http://localhost:8333
|
||||
* mvn test -Dtest=ETagValidationTest
|
||||
*/
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
@DisplayName("S3 ETag Format Validation Tests (AWS SDK v2)")
|
||||
class ETagValidationTest {
|
||||
|
||||
// Configuration - can be overridden via system properties or environment variables
|
||||
private static final String DEFAULT_ENDPOINT = "http://127.0.0.1:8333";
|
||||
private static final String DEFAULT_ACCESS_KEY = "some_access_key1";
|
||||
private static final String DEFAULT_SECRET_KEY = "some_secret_key1";
|
||||
private static final String DEFAULT_REGION = "us-east-1";
|
||||
|
||||
// Auto-chunking threshold in SeaweedFS (must match s3api_object_handlers_put.go)
|
||||
private static final int AUTO_CHUNK_SIZE = 8 * 1024 * 1024; // 8MB
|
||||
|
||||
// Test sizes
|
||||
private static final int SMALL_FILE_SIZE = 1024; // 1KB
|
||||
private static final int LARGE_FILE_SIZE = 10 * 1024 * 1024; // 10MB (triggers auto-chunking)
|
||||
private static final int XL_FILE_SIZE = 25 * 1024 * 1024; // 25MB (multiple chunks)
|
||||
private static final int MULTIPART_PART_SIZE = 5 * 1024 * 1024; // 5MB per part
|
||||
|
||||
// ETag format patterns
|
||||
private static final Pattern PURE_MD5_PATTERN = Pattern.compile("^\"?[a-f0-9]{32}\"?$");
|
||||
private static final Pattern COMPOSITE_PATTERN = Pattern.compile("^\"?[a-f0-9]{32}-\\d+\"?$");
|
||||
|
||||
private S3Client s3Client;
|
||||
private String testBucketName;
|
||||
private final SecureRandom random = new SecureRandom();
|
||||
|
||||
@BeforeAll
|
||||
void setUp() {
|
||||
String endpoint = getConfig("S3_ENDPOINT", DEFAULT_ENDPOINT);
|
||||
String accessKey = getConfig("S3_ACCESS_KEY", DEFAULT_ACCESS_KEY);
|
||||
String secretKey = getConfig("S3_SECRET_KEY", DEFAULT_SECRET_KEY);
|
||||
String region = getConfig("S3_REGION", DEFAULT_REGION);
|
||||
|
||||
System.out.println("Connecting to S3 endpoint: " + endpoint);
|
||||
|
||||
s3Client = S3Client.builder()
|
||||
.endpointOverride(URI.create(endpoint))
|
||||
.credentialsProvider(StaticCredentialsProvider.create(
|
||||
AwsBasicCredentials.create(accessKey, secretKey)))
|
||||
.region(Region.of(region))
|
||||
.forcePathStyle(true) // Required for SeaweedFS
|
||||
.build();
|
||||
|
||||
// Create test bucket
|
||||
testBucketName = "test-etag-" + UUID.randomUUID().toString().substring(0, 8);
|
||||
s3Client.createBucket(CreateBucketRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.build());
|
||||
|
||||
System.out.println("Created test bucket: " + testBucketName);
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
void tearDown() {
|
||||
if (s3Client != null && testBucketName != null) {
|
||||
try {
|
||||
// Delete all objects with pagination
|
||||
String continuationToken = null;
|
||||
do {
|
||||
ListObjectsV2Response listResp = s3Client.listObjectsV2(
|
||||
ListObjectsV2Request.builder()
|
||||
.bucket(testBucketName)
|
||||
.continuationToken(continuationToken)
|
||||
.build());
|
||||
for (S3Object obj : listResp.contents()) {
|
||||
s3Client.deleteObject(DeleteObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(obj.key())
|
||||
.build());
|
||||
}
|
||||
continuationToken = listResp.nextContinuationToken();
|
||||
} while (continuationToken != null);
|
||||
|
||||
// Abort any multipart uploads
|
||||
ListMultipartUploadsResponse mpResp = s3Client.listMultipartUploads(
|
||||
ListMultipartUploadsRequest.builder().bucket(testBucketName).build());
|
||||
for (MultipartUpload upload : mpResp.uploads()) {
|
||||
s3Client.abortMultipartUpload(AbortMultipartUploadRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(upload.key())
|
||||
.uploadId(upload.uploadId())
|
||||
.build());
|
||||
}
|
||||
|
||||
// Delete bucket
|
||||
s3Client.deleteBucket(DeleteBucketRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.build());
|
||||
|
||||
System.out.println("Cleaned up test bucket: " + testBucketName);
|
||||
} catch (Exception e) {
|
||||
System.err.println("Warning: Failed to cleanup test bucket: " + e.getMessage());
|
||||
}
|
||||
s3Client.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Small file PutObject should return pure MD5 ETag")
|
||||
void testSmallFilePutObject() throws Exception {
|
||||
byte[] testData = generateRandomData(SMALL_FILE_SIZE);
|
||||
String expectedMD5 = calculateMD5Hex(testData);
|
||||
String objectKey = "small-file-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
PutObjectResponse response = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
|
||||
String etag = response.eTag();
|
||||
System.out.println("Small file ETag: " + etag + " (expected MD5: " + expectedMD5 + ")");
|
||||
|
||||
assertThat(etag)
|
||||
.describedAs("Small file ETag should be pure MD5")
|
||||
.matches(PURE_MD5_PATTERN);
|
||||
assertThat(cleanETag(etag))
|
||||
.describedAs("ETag should match calculated MD5")
|
||||
.isEqualTo(expectedMD5);
|
||||
assertThat(etag)
|
||||
.describedAs("ETag should not contain hyphen")
|
||||
.doesNotContain("-");
|
||||
}
|
||||
|
||||
/**
|
||||
* Critical test for GitHub Issue #7768.
|
||||
*
|
||||
* This test uploads a file larger than the auto-chunking threshold (8MB),
|
||||
* which triggers SeaweedFS to split the file into multiple internal chunks.
|
||||
*
|
||||
* Previously, this caused SeaweedFS to return a composite ETag like
|
||||
* "d41d8cd98f00b204e9800998ecf8427e-2", which AWS SDK v2 rejected because
|
||||
* it validates the ETag as hexadecimal and '-' is not a valid hex character.
|
||||
*
|
||||
* The fix ensures that regular PutObject always returns a pure MD5 ETag,
|
||||
* regardless of internal chunking.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Large file PutObject (>8MB) should return pure MD5 ETag - Issue #7768")
|
||||
void testLargeFilePutObject_Issue7768() throws Exception {
|
||||
byte[] testData = generateRandomData(LARGE_FILE_SIZE);
|
||||
String expectedMD5 = calculateMD5Hex(testData);
|
||||
String objectKey = "large-file-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
System.out.println("Uploading large file (" + LARGE_FILE_SIZE + " bytes, " +
|
||||
"> " + AUTO_CHUNK_SIZE + " byte auto-chunk threshold)...");
|
||||
|
||||
// This is where Issue #7768 would manifest - SDK v2 validates ETag
|
||||
PutObjectResponse response = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
|
||||
String etag = response.eTag();
|
||||
int expectedChunks = (LARGE_FILE_SIZE / AUTO_CHUNK_SIZE) + 1;
|
||||
System.out.println("Large file ETag: " + etag +
|
||||
" (expected MD5: " + expectedMD5 + ", internal chunks: ~" + expectedChunks + ")");
|
||||
|
||||
// These assertions would fail before the fix
|
||||
assertThat(etag)
|
||||
.describedAs("Large file PutObject ETag MUST be pure MD5 (not composite)")
|
||||
.matches(PURE_MD5_PATTERN);
|
||||
assertThat(etag)
|
||||
.describedAs("Large file ETag should NOT be composite format")
|
||||
.doesNotMatch(COMPOSITE_PATTERN);
|
||||
assertThat(etag)
|
||||
.describedAs("ETag should not contain hyphen for regular PutObject")
|
||||
.doesNotContain("-");
|
||||
assertThat(cleanETag(etag))
|
||||
.describedAs("ETag should match calculated MD5")
|
||||
.isEqualTo(expectedMD5);
|
||||
|
||||
// Verify hex decoding works (this is what fails in Issue #7768)
|
||||
assertThatCode(() -> hexToBytes(cleanETag(etag)))
|
||||
.describedAs("ETag should be valid hexadecimal (AWS SDK v2 validation)")
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Extra large file PutObject (>24MB) should return pure MD5 ETag")
|
||||
void testExtraLargeFilePutObject() throws Exception {
|
||||
byte[] testData = generateRandomData(XL_FILE_SIZE);
|
||||
String expectedMD5 = calculateMD5Hex(testData);
|
||||
String objectKey = "xl-file-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
int expectedChunks = (XL_FILE_SIZE / AUTO_CHUNK_SIZE) + 1;
|
||||
System.out.println("Uploading XL file (" + XL_FILE_SIZE + " bytes, ~" +
|
||||
expectedChunks + " internal chunks)...");
|
||||
|
||||
PutObjectResponse response = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
|
||||
String etag = response.eTag();
|
||||
System.out.println("XL file ETag: " + etag);
|
||||
|
||||
assertThat(etag)
|
||||
.describedAs("XL file PutObject ETag MUST be pure MD5")
|
||||
.matches(PURE_MD5_PATTERN);
|
||||
assertThat(cleanETag(etag))
|
||||
.describedAs("ETag should match calculated MD5")
|
||||
.isEqualTo(expectedMD5);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Multipart upload should return composite ETag")
|
||||
void testMultipartUploadETag() throws Exception {
|
||||
int totalSize = 15 * 1024 * 1024; // 15MB = 3 parts
|
||||
byte[] testData = generateRandomData(totalSize);
|
||||
String objectKey = "multipart-file-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
System.out.println("Performing multipart upload (" + totalSize + " bytes)...");
|
||||
|
||||
// Initiate multipart upload
|
||||
CreateMultipartUploadResponse createResp = s3Client.createMultipartUpload(
|
||||
CreateMultipartUploadRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build());
|
||||
String uploadId = createResp.uploadId();
|
||||
|
||||
List<CompletedPart> completedParts = new ArrayList<>();
|
||||
int partNumber = 1;
|
||||
|
||||
// Upload parts
|
||||
for (int offset = 0; offset < totalSize; offset += MULTIPART_PART_SIZE) {
|
||||
int end = Math.min(offset + MULTIPART_PART_SIZE, totalSize);
|
||||
byte[] partData = new byte[end - offset];
|
||||
System.arraycopy(testData, offset, partData, 0, partData.length);
|
||||
|
||||
UploadPartResponse uploadResp = s3Client.uploadPart(
|
||||
UploadPartRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.uploadId(uploadId)
|
||||
.partNumber(partNumber)
|
||||
.build(),
|
||||
RequestBody.fromBytes(partData));
|
||||
|
||||
completedParts.add(CompletedPart.builder()
|
||||
.partNumber(partNumber)
|
||||
.eTag(uploadResp.eTag())
|
||||
.build());
|
||||
partNumber++;
|
||||
}
|
||||
|
||||
// Complete multipart upload
|
||||
CompleteMultipartUploadResponse completeResp = s3Client.completeMultipartUpload(
|
||||
CompleteMultipartUploadRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.uploadId(uploadId)
|
||||
.multipartUpload(CompletedMultipartUpload.builder()
|
||||
.parts(completedParts)
|
||||
.build())
|
||||
.build());
|
||||
|
||||
String etag = completeResp.eTag();
|
||||
System.out.println("Multipart upload ETag: " + etag + " (" + completedParts.size() + " parts)");
|
||||
|
||||
// Multipart uploads SHOULD have composite ETag
|
||||
assertThat(etag)
|
||||
.describedAs("Multipart upload ETag SHOULD be composite format")
|
||||
.matches(COMPOSITE_PATTERN);
|
||||
assertThat(etag)
|
||||
.describedAs("Multipart ETag should contain hyphen")
|
||||
.contains("-");
|
||||
|
||||
// Verify part count in ETag
|
||||
String[] parts = cleanETag(etag).split("-");
|
||||
assertThat(parts).hasSize(2);
|
||||
assertThat(parts[1])
|
||||
.describedAs("Part count in ETag should match uploaded parts")
|
||||
.isEqualTo(String.valueOf(completedParts.size()));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("ETag should be consistent across PUT, HEAD, and GET")
|
||||
void testETagConsistency() throws Exception {
|
||||
byte[] testData = generateRandomData(LARGE_FILE_SIZE);
|
||||
String objectKey = "consistency-test-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
// PUT
|
||||
PutObjectResponse putResp = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
String putETag = putResp.eTag();
|
||||
|
||||
// HEAD
|
||||
HeadObjectResponse headResp = s3Client.headObject(
|
||||
HeadObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build());
|
||||
String headETag = headResp.eTag();
|
||||
|
||||
// GET
|
||||
GetObjectResponse getResp = s3Client.getObject(
|
||||
GetObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build())
|
||||
.response();
|
||||
String getETag = getResp.eTag();
|
||||
|
||||
System.out.println("PUT ETag: " + putETag + ", HEAD ETag: " + headETag + ", GET ETag: " + getETag);
|
||||
|
||||
assertThat(putETag).isEqualTo(headETag);
|
||||
assertThat(putETag).isEqualTo(getETag);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Multiple large file uploads should all return pure MD5 ETags")
|
||||
void testMultipleLargeFileUploads() throws Exception {
|
||||
int numFiles = 3;
|
||||
|
||||
for (int i = 0; i < numFiles; i++) {
|
||||
byte[] testData = generateRandomData(LARGE_FILE_SIZE);
|
||||
String expectedMD5 = calculateMD5Hex(testData);
|
||||
String objectKey = "multi-large-" + i + "-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
PutObjectResponse response = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
|
||||
String etag = response.eTag();
|
||||
System.out.println("File " + i + " ETag: " + etag);
|
||||
|
||||
assertThat(etag)
|
||||
.describedAs("File " + i + " ETag should be pure MD5")
|
||||
.matches(PURE_MD5_PATTERN);
|
||||
assertThat(cleanETag(etag))
|
||||
.describedAs("File " + i + " ETag should match MD5")
|
||||
.isEqualTo(expectedMD5);
|
||||
|
||||
// Validate hex decoding
|
||||
assertThatCode(() -> hexToBytes(cleanETag(etag)))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
}
|
||||
|
||||
// Helper methods
|
||||
|
||||
private String getConfig(String key, String defaultValue) {
|
||||
String value = System.getProperty(key);
|
||||
if (value == null) {
|
||||
value = System.getenv(key);
|
||||
}
|
||||
return value != null ? value : defaultValue;
|
||||
}
|
||||
|
||||
private byte[] generateRandomData(int size) {
|
||||
byte[] data = new byte[size];
|
||||
random.nextBytes(data);
|
||||
return data;
|
||||
}
|
||||
|
||||
private String calculateMD5Hex(byte[] data) throws Exception {
|
||||
MessageDigest md = MessageDigest.getInstance("MD5");
|
||||
byte[] digest = md.digest(data);
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (byte b : digest) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private String cleanETag(String etag) {
|
||||
if (etag == null) return null;
|
||||
return etag.replace("\"", "");
|
||||
}
|
||||
|
||||
private byte[] hexToBytes(String hex) {
|
||||
int len = hex.length();
|
||||
byte[] data = new byte[len / 2];
|
||||
for (int i = 0; i < len; i += 2) {
|
||||
data[i / 2] = (byte) ((Character.digit(hex.charAt(i), 16) << 4)
|
||||
+ Character.digit(hex.charAt(i + 1), 16));
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package com.seaweedfs.s3;
|
||||
|
||||
import junit.framework.Test;
|
||||
import junit.framework.TestCase;
|
||||
import junit.framework.TestSuite;
|
||||
|
||||
/**
|
||||
* Unit test for simple App.
|
||||
*/
|
||||
public class PutObjectTest
|
||||
extends TestCase
|
||||
{
|
||||
/**
|
||||
* Create the test case
|
||||
*
|
||||
* @param testName name of the test case
|
||||
*/
|
||||
public PutObjectTest(String testName )
|
||||
{
|
||||
super( testName );
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the suite of tests being tested
|
||||
*/
|
||||
public static Test suite()
|
||||
{
|
||||
return new TestSuite( PutObjectTest.class );
|
||||
}
|
||||
|
||||
/**
|
||||
* Rigorous Test :-)
|
||||
*/
|
||||
public void testApp()
|
||||
{
|
||||
assertTrue( true );
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user