snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
352
test/s3/retention/Makefile
Normal file
352
test/s3/retention/Makefile
Normal file
@@ -0,0 +1,352 @@
|
||||
# S3 API Retention Test Makefile
|
||||
# This Makefile provides comprehensive targets for running S3 retention tests
|
||||
|
||||
.PHONY: help build-weed setup-server start-server stop-server test-retention test-retention-quick test-retention-comprehensive test-retention-worm test-all clean logs check-deps
|
||||
|
||||
# Configuration
|
||||
WEED_BINARY := ../../../weed/weed_binary
|
||||
S3_PORT := 8333
|
||||
ACCESS_KEY ?= some_access_key1
|
||||
SECRET_KEY ?= some_secret_key1
|
||||
MASTER_PORT := 9333
|
||||
VOLUME_PORT := 8080
|
||||
FILER_PORT := 8888
|
||||
TEST_TIMEOUT := 15m
|
||||
TEST_PATTERN := TestRetention
|
||||
SERVER_DIR := ./test-volume-data/server-data
|
||||
|
||||
# Default target
|
||||
help:
|
||||
@echo "S3 API Retention Test Makefile"
|
||||
@echo ""
|
||||
@echo "Available targets:"
|
||||
@echo " help - Show this help message"
|
||||
@echo " build-weed - Build the SeaweedFS binary"
|
||||
@echo " check-deps - Check dependencies and build binary if needed"
|
||||
@echo " start-server - Start SeaweedFS server for testing"
|
||||
@echo " start-server-simple - Start server without process cleanup (for CI)"
|
||||
@echo " stop-server - Stop SeaweedFS server"
|
||||
@echo " test-retention - Run all retention tests"
|
||||
@echo " test-retention-quick - Run core retention tests only"
|
||||
@echo " test-retention-simple - Run tests without server management"
|
||||
@echo " test-retention-comprehensive - Run comprehensive retention tests"
|
||||
@echo " test-retention-worm - Run WORM integration tests"
|
||||
@echo " test-all - Run all S3 API retention tests"
|
||||
@echo " test-with-server - Start server, run tests, stop server"
|
||||
@echo " logs - Show server logs"
|
||||
@echo " clean - Clean up test artifacts and stop server"
|
||||
@echo " health-check - Check if server is accessible"
|
||||
@echo ""
|
||||
@echo "Configuration:"
|
||||
@echo " S3_PORT=${S3_PORT}"
|
||||
@echo " TEST_TIMEOUT=${TEST_TIMEOUT}"
|
||||
|
||||
# Build the SeaweedFS binary
|
||||
build-weed:
|
||||
@echo "Building SeaweedFS binary..."
|
||||
@cd ../../../weed && go build -o weed_binary .
|
||||
@chmod +x $(WEED_BINARY)
|
||||
@echo "✅ SeaweedFS binary built at $(WEED_BINARY)"
|
||||
|
||||
check-deps: build-weed
|
||||
@echo "Checking dependencies..."
|
||||
@echo "🔍 DEBUG: Checking Go installation..."
|
||||
@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1)
|
||||
@echo "🔍 DEBUG: Go version: $$(go version)"
|
||||
@echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)..."
|
||||
@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1)
|
||||
@echo "🔍 DEBUG: Binary size: $$(ls -lh $(WEED_BINARY) | awk '{print $$5}')"
|
||||
@echo "🔍 DEBUG: Binary permissions: $$(ls -la $(WEED_BINARY) | awk '{print $$1}')"
|
||||
@echo "🔍 DEBUG: Checking Go module dependencies..."
|
||||
@go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1)
|
||||
@go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1)
|
||||
@echo "✅ All dependencies are available"
|
||||
|
||||
# Start SeaweedFS server for testing
|
||||
start-server: check-deps
|
||||
@echo "Starting SeaweedFS server..."
|
||||
@echo "🔍 DEBUG: Current working directory: $$(pwd)"
|
||||
@echo "🔍 DEBUG: Checking for existing weed processes..."
|
||||
@ps aux | grep weed | grep -v grep || echo "No existing weed processes found"
|
||||
@echo "🔍 DEBUG: Cleaning up any existing PID file..."
|
||||
@rm -f weed-server.pid
|
||||
@echo "🔍 DEBUG: Checking for port conflicts..."
|
||||
@if netstat -tlnp 2>/dev/null | grep $(S3_PORT) >/dev/null; then \
|
||||
echo "⚠️ Port $(S3_PORT) is already in use, trying to find the process..."; \
|
||||
netstat -tlnp 2>/dev/null | grep $(S3_PORT) || true; \
|
||||
else \
|
||||
echo "✅ Port $(S3_PORT) is available"; \
|
||||
fi
|
||||
@echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)"
|
||||
@ls -la $(WEED_BINARY) || (echo "❌ Binary not found!" && exit 1)
|
||||
@echo "🔍 DEBUG: Checking config file at ../../../docker/compose/s3.json"
|
||||
@ls -la ../../../docker/compose/s3.json || echo "⚠️ Config file not found, continuing without it"
|
||||
@echo "🔍 DEBUG: Creating volume directory..."
|
||||
@mkdir -p ./test-volume-data
|
||||
@echo "🔍 DEBUG: Creating server data directory..."
|
||||
@mkdir -p $(SERVER_DIR)
|
||||
@echo "🔍 DEBUG: Launching SeaweedFS S3 server in background..."
|
||||
@echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT)"
|
||||
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \
|
||||
-dir=$(SERVER_DIR) \
|
||||
-s3.port=$(S3_PORT) \
|
||||
> weed-test.log 2>&1 & \
|
||||
echo $$! > weed-server.pid
|
||||
@echo "🔍 DEBUG: Server PID: $$(cat weed-test.pid 2>/dev/null || echo 'PID file not found')"
|
||||
@echo "🔍 DEBUG: Checking if PID is still running..."
|
||||
@sleep 2
|
||||
@if [ -f weed-server.pid ]; then \
|
||||
SERVER_PID=$$(cat weed-server.pid); \
|
||||
ps -p $$SERVER_PID || echo "⚠️ Server PID $$SERVER_PID not found after 2 seconds"; \
|
||||
else \
|
||||
echo "⚠️ PID file not found"; \
|
||||
fi
|
||||
@for i in $$(seq 1 90); do \
|
||||
echo "🔍 DEBUG: Attempt $$i/90 - checking port $(S3_PORT)"; \
|
||||
if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
|
||||
echo "✅ SeaweedFS server started successfully on port $(S3_PORT) after $$i seconds"; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
if [ $$i -eq 5 ]; then \
|
||||
echo "🔍 DEBUG: After 5 seconds, checking process and logs..."; \
|
||||
ps aux | grep weed | grep -v grep || echo "No weed processes found"; \
|
||||
if [ -f weed-test.log ]; then \
|
||||
echo "=== First server logs ==="; \
|
||||
head -20 weed-test.log; \
|
||||
fi; \
|
||||
fi; \
|
||||
if [ $$i -eq 15 ]; then \
|
||||
echo "🔍 DEBUG: After 15 seconds, checking port bindings..."; \
|
||||
netstat -tlnp 2>/dev/null | grep $(S3_PORT) || echo "Port $(S3_PORT) not bound"; \
|
||||
fi; \
|
||||
if [ $$i -eq 30 ]; then \
|
||||
echo "⚠️ Server taking longer than expected (30s), checking logs..."; \
|
||||
if [ -f weed-test.log ]; then \
|
||||
echo "=== Recent server logs ==="; \
|
||||
tail -20 weed-test.log; \
|
||||
fi; \
|
||||
fi; \
|
||||
sleep 1; \
|
||||
done; \
|
||||
echo "❌ Server failed to start within 90 seconds"; \
|
||||
echo "🔍 DEBUG: Final process check:"; \
|
||||
ps aux | grep weed | grep -v grep || echo "No weed processes found"; \
|
||||
echo "🔍 DEBUG: Final port check:"; \
|
||||
netstat -tlnp 2>/dev/null | grep -E "(8333|9333|8080)" || echo "No ports bound"; \
|
||||
echo "=== Full server logs ==="; \
|
||||
if [ -f weed-test.log ]; then \
|
||||
cat weed-test.log; \
|
||||
else \
|
||||
echo "No log file found"; \
|
||||
fi; \
|
||||
exit 1
|
||||
|
||||
# Stop SeaweedFS server
|
||||
stop-server:
|
||||
@echo "Stopping SeaweedFS server..."
|
||||
@if [ -f weed-server.pid ]; then \
|
||||
SERVER_PID=$$(cat weed-server.pid); \
|
||||
echo "Killing server PID $$SERVER_PID"; \
|
||||
if ps -p $$SERVER_PID >/dev/null 2>&1; then \
|
||||
kill -TERM $$SERVER_PID 2>/dev/null || true; \
|
||||
sleep 2; \
|
||||
if ps -p $$SERVER_PID >/dev/null 2>&1; then \
|
||||
echo "Process still running, sending KILL signal..."; \
|
||||
kill -KILL $$SERVER_PID 2>/dev/null || true; \
|
||||
sleep 1; \
|
||||
fi; \
|
||||
else \
|
||||
echo "Process $$SERVER_PID not found (already stopped)"; \
|
||||
fi; \
|
||||
rm -f weed-server.pid; \
|
||||
else \
|
||||
echo "No PID file found, checking for running processes..."; \
|
||||
echo "⚠️ Skipping automatic process cleanup to avoid CI issues"; \
|
||||
echo "Note: Any remaining weed processes should be cleaned up by the CI environment"; \
|
||||
fi
|
||||
@echo "✅ SeaweedFS server stopped"
|
||||
|
||||
# Show server logs
|
||||
logs:
|
||||
@if test -f weed-test.log; then \
|
||||
echo "=== SeaweedFS Server Logs ==="; \
|
||||
tail -f weed-test.log; \
|
||||
else \
|
||||
echo "No log file found. Server may not be running."; \
|
||||
fi
|
||||
|
||||
# Core retention tests (basic functionality)
|
||||
test-retention-quick: check-deps
|
||||
@echo "Running core S3 retention tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestBasicRetentionWorkflow|TestRetentionModeCompliance|TestLegalHoldWorkflow" .
|
||||
@echo "✅ Core retention tests completed"
|
||||
|
||||
# All retention tests (comprehensive)
|
||||
test-retention: check-deps
|
||||
@echo "Running all S3 retention tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" .
|
||||
@echo "✅ All retention tests completed"
|
||||
|
||||
# WORM integration tests
|
||||
test-retention-worm: check-deps
|
||||
@echo "Running WORM integration tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestWORM|TestRetentionExtendedAttributes|TestRetentionConcurrentOperations" .
|
||||
@echo "✅ WORM integration tests completed"
|
||||
|
||||
# Comprehensive retention tests (all features)
|
||||
test-retention-comprehensive: check-deps
|
||||
@echo "Running comprehensive S3 retention tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetention|TestObjectLock|TestLegalHold|TestWORM" .
|
||||
@echo "✅ Comprehensive retention tests completed"
|
||||
|
||||
# All tests without server management
|
||||
test-retention-simple: check-deps
|
||||
@echo "Running retention tests (assuming server is already running)..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) .
|
||||
@echo "✅ All retention tests completed"
|
||||
|
||||
# Start server, run tests, stop server
|
||||
test-with-server: start-server
|
||||
@echo "Running retention tests with managed server..."
|
||||
@sleep 5 # Give server time to fully start
|
||||
@make test-retention-comprehensive || (echo "Tests failed, stopping server..." && make stop-server && exit 1)
|
||||
@make stop-server
|
||||
@echo "✅ All tests completed with managed server"
|
||||
|
||||
# Health check
|
||||
health-check:
|
||||
@echo "Checking server health..."
|
||||
@if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
|
||||
echo "✅ Server is accessible on port $(S3_PORT)"; \
|
||||
else \
|
||||
echo "❌ Server is not accessible on port $(S3_PORT)"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
# Clean up
|
||||
clean:
|
||||
@echo "Cleaning up test artifacts..."
|
||||
@make stop-server
|
||||
@rm -f weed-test.log
|
||||
@rm -f weed-server.pid
|
||||
@rm -rf ./test-volume-data
|
||||
@echo "✅ Cleanup completed"
|
||||
|
||||
# Individual test targets for specific functionality
|
||||
test-basic-retention:
|
||||
@echo "Running basic retention tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestBasicRetentionWorkflow" .
|
||||
|
||||
test-compliance-retention:
|
||||
@echo "Running compliance retention tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionModeCompliance" .
|
||||
|
||||
test-legal-hold:
|
||||
@echo "Running legal hold tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestLegalHoldWorkflow" .
|
||||
|
||||
test-object-lock-config:
|
||||
@echo "Running object lock configuration tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestObjectLockConfiguration" .
|
||||
|
||||
test-retention-versions:
|
||||
@echo "Running retention with versions tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionWithVersions" .
|
||||
|
||||
test-retention-combination:
|
||||
@echo "Running retention and legal hold combination tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionAndLegalHoldCombination" .
|
||||
|
||||
test-expired-retention:
|
||||
@echo "Running expired retention tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestExpiredRetention" .
|
||||
|
||||
test-retention-errors:
|
||||
@echo "Running retention error case tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionErrorCases" .
|
||||
|
||||
# WORM-specific test targets
|
||||
test-worm-integration:
|
||||
@echo "Running WORM integration tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestWORMRetentionIntegration" .
|
||||
|
||||
test-worm-legacy:
|
||||
@echo "Running WORM legacy compatibility tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestWORMLegacyCompatibility" .
|
||||
|
||||
test-retention-overwrite:
|
||||
@echo "Running retention overwrite protection tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionOverwriteProtection" .
|
||||
|
||||
test-retention-bulk:
|
||||
@echo "Running retention bulk operations tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionBulkOperations" .
|
||||
|
||||
test-retention-multipart:
|
||||
@echo "Running retention multipart upload tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionWithMultipartUpload" .
|
||||
|
||||
test-retention-extended-attrs:
|
||||
@echo "Running retention extended attributes tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionExtendedAttributes" .
|
||||
|
||||
test-retention-defaults:
|
||||
@echo "Running retention bucket defaults tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionBucketDefaults" .
|
||||
|
||||
test-retention-concurrent:
|
||||
@echo "Running retention concurrent operations tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestRetentionConcurrentOperations" .
|
||||
|
||||
# Development targets
|
||||
dev-start: start-server
|
||||
@echo "Development server started. Access S3 API at http://localhost:$(S3_PORT)"
|
||||
@echo "To stop: make stop-server"
|
||||
|
||||
dev-test: check-deps
|
||||
@echo "Running tests in development mode..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "TestBasicRetentionWorkflow" .
|
||||
|
||||
# CI targets
|
||||
ci-test: check-deps
|
||||
@echo "Running tests in CI mode..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -race .
|
||||
|
||||
# All targets
|
||||
test-all: test-retention test-retention-worm
|
||||
@echo "✅ All S3 retention tests completed"
|
||||
|
||||
# Benchmark targets
|
||||
benchmark-retention:
|
||||
@echo "Running retention performance benchmarks..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -bench=. -benchmem .
|
||||
|
||||
# Coverage targets
|
||||
coverage:
|
||||
@echo "Running tests with coverage..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -coverprofile=coverage.out .
|
||||
@go tool cover -html=coverage.out -o coverage.html
|
||||
@echo "Coverage report generated: coverage.html"
|
||||
|
||||
# Format and lint
|
||||
fmt:
|
||||
@echo "Formatting Go code..."
|
||||
@go fmt .
|
||||
|
||||
lint:
|
||||
@echo "Running linter..."
|
||||
@golint . || echo "golint not available, skipping..."
|
||||
|
||||
# Install dependencies for development
|
||||
install-deps:
|
||||
@echo "Installing Go dependencies..."
|
||||
@go mod tidy
|
||||
@go mod download
|
||||
|
||||
# Show current configuration
|
||||
show-config:
|
||||
@echo "Current configuration:"
|
||||
@echo " WEED_BINARY: $(WEED_BINARY)"
|
||||
@echo " S3_PORT: $(S3_PORT)"
|
||||
@echo " TEST_TIMEOUT: $(TEST_TIMEOUT)"
|
||||
@echo " TEST_PATTERN: $(TEST_PATTERN)"
|
||||
264
test/s3/retention/README.md
Normal file
264
test/s3/retention/README.md
Normal file
@@ -0,0 +1,264 @@
|
||||
# SeaweedFS S3 Object Retention Tests
|
||||
|
||||
This directory contains comprehensive tests for SeaweedFS S3 Object Retention functionality, including Object Lock, Legal Hold, and WORM (Write Once Read Many) capabilities.
|
||||
|
||||
## Overview
|
||||
|
||||
The test suite validates AWS S3-compatible object retention features including:
|
||||
|
||||
- **Object Retention**: GOVERNANCE and COMPLIANCE modes with retain-until-date
|
||||
- **Legal Hold**: Independent protection that can be applied/removed
|
||||
- **Object Lock Configuration**: Bucket-level default retention policies
|
||||
- **WORM Integration**: Compatibility with legacy WORM functionality
|
||||
- **Version-specific Retention**: Different retention policies per object version
|
||||
- **Enforcement**: Protection against deletion and overwriting
|
||||
|
||||
## Test Files
|
||||
|
||||
- `s3_retention_test.go` - Core retention functionality tests
|
||||
- `s3_worm_integration_test.go` - WORM integration and advanced scenarios
|
||||
- `test_config.json` - Test configuration (endpoints, credentials)
|
||||
- `Makefile` - Comprehensive test automation
|
||||
- `go.mod` - Go module dependencies
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Go 1.21 or later
|
||||
- SeaweedFS binary built (`make build-weed`)
|
||||
- AWS SDK Go v2
|
||||
- Testify testing framework
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Build and Start Server
|
||||
```bash
|
||||
# Build SeaweedFS and start test server
|
||||
make start-server
|
||||
```
|
||||
|
||||
### 2. Run Tests
|
||||
```bash
|
||||
# Run core retention tests
|
||||
make test-retention-quick
|
||||
|
||||
# Run all retention tests
|
||||
make test-retention
|
||||
|
||||
# Run WORM integration tests
|
||||
make test-retention-worm
|
||||
|
||||
# Run all tests with managed server
|
||||
make test-with-server
|
||||
```
|
||||
|
||||
### 3. Cleanup
|
||||
```bash
|
||||
make clean
|
||||
```
|
||||
|
||||
## Test Categories
|
||||
|
||||
### Core Retention Tests
|
||||
- `TestBasicRetentionWorkflow` - Basic GOVERNANCE mode retention
|
||||
- `TestRetentionModeCompliance` - COMPLIANCE mode (immutable)
|
||||
- `TestLegalHoldWorkflow` - Legal hold on/off functionality
|
||||
- `TestObjectLockConfiguration` - Bucket object lock settings
|
||||
|
||||
### Advanced Tests
|
||||
- `TestRetentionWithVersions` - Version-specific retention policies
|
||||
- `TestRetentionAndLegalHoldCombination` - Multiple protection types
|
||||
- `TestExpiredRetention` - Post-expiration behavior
|
||||
- `TestRetentionErrorCases` - Error handling and edge cases
|
||||
|
||||
### WORM Integration Tests
|
||||
- `TestWORMRetentionIntegration` - New retention + legacy WORM
|
||||
- `TestWORMLegacyCompatibility` - Backward compatibility
|
||||
- `TestRetentionOverwriteProtection` - Prevent overwrites
|
||||
- `TestRetentionBulkOperations` - Bulk delete with retention
|
||||
- `TestRetentionWithMultipartUpload` - Multipart upload retention
|
||||
- `TestRetentionExtendedAttributes` - Extended attribute storage
|
||||
- `TestRetentionBucketDefaults` - Default retention application
|
||||
- `TestRetentionConcurrentOperations` - Concurrent operation safety
|
||||
|
||||
## Individual Test Targets
|
||||
|
||||
Run specific test categories:
|
||||
|
||||
```bash
|
||||
# Basic functionality
|
||||
make test-basic-retention
|
||||
make test-compliance-retention
|
||||
make test-legal-hold
|
||||
|
||||
# Advanced features
|
||||
make test-retention-versions
|
||||
make test-retention-combination
|
||||
make test-expired-retention
|
||||
|
||||
# WORM integration
|
||||
make test-worm-integration
|
||||
make test-worm-legacy
|
||||
make test-retention-bulk
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Server Configuration
|
||||
The tests use these default settings:
|
||||
- S3 Port: 8333
|
||||
- Test timeout: 15 minutes
|
||||
- Volume directory: `./test-volume-data`
|
||||
|
||||
### Test Configuration (`test_config.json`)
|
||||
```json
|
||||
{
|
||||
"endpoint": "http://localhost:8333",
|
||||
"access_key": "some_access_key1",
|
||||
"secret_key": "some_secret_key1",
|
||||
"region": "us-east-1",
|
||||
"bucket_prefix": "test-retention-",
|
||||
"use_ssl": false,
|
||||
"skip_verify_ssl": true
|
||||
}
|
||||
```
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
### GOVERNANCE Mode
|
||||
- Objects protected until retain-until-date
|
||||
- Can be bypassed with `x-amz-bypass-governance-retention` header
|
||||
- Supports time extension (not reduction)
|
||||
|
||||
### COMPLIANCE Mode
|
||||
- Objects immutably protected until retain-until-date
|
||||
- Cannot be bypassed or shortened
|
||||
- Strictest protection level
|
||||
|
||||
### Legal Hold
|
||||
- Independent ON/OFF protection
|
||||
- Can coexist with retention policies
|
||||
- Must be explicitly removed to allow deletion
|
||||
|
||||
### Version Support
|
||||
- Each object version can have individual retention
|
||||
- Applies to both versioned and non-versioned buckets
|
||||
- Version-specific retention retrieval
|
||||
|
||||
## Development
|
||||
|
||||
### Running in Development Mode
|
||||
```bash
|
||||
# Start server for development
|
||||
make dev-start
|
||||
|
||||
# Run quick test
|
||||
make dev-test
|
||||
```
|
||||
|
||||
### Code Quality
|
||||
```bash
|
||||
# Format code
|
||||
make fmt
|
||||
|
||||
# Run linter
|
||||
make lint
|
||||
|
||||
# Generate coverage report
|
||||
make coverage
|
||||
```
|
||||
|
||||
### Performance Testing
|
||||
```bash
|
||||
# Run benchmarks
|
||||
make benchmark-retention
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Server Won't Start
|
||||
```bash
|
||||
# Check if port is in use
|
||||
netstat -tlnp | grep 8333
|
||||
|
||||
# View server logs
|
||||
make logs
|
||||
|
||||
# Force cleanup
|
||||
make clean
|
||||
```
|
||||
|
||||
### Test Failures
|
||||
```bash
|
||||
# Run with verbose output
|
||||
go test -v -timeout=15m .
|
||||
|
||||
# Run specific test
|
||||
go test -v -run TestBasicRetentionWorkflow .
|
||||
|
||||
# Check server health
|
||||
make health-check
|
||||
```
|
||||
|
||||
### Dependencies
|
||||
```bash
|
||||
# Install/update dependencies
|
||||
make install-deps
|
||||
|
||||
# Check dependency status
|
||||
make check-deps
|
||||
```
|
||||
|
||||
## Integration with SeaweedFS
|
||||
|
||||
These tests validate the retention implementation in:
|
||||
- `weed/s3api/s3api_object_retention.go` - Core retention logic
|
||||
- `weed/s3api/s3api_object_handlers_retention.go` - HTTP handlers
|
||||
- `weed/s3api/s3_constants/extend_key.go` - Extended attribute keys
|
||||
- `weed/s3api/s3err/s3api_errors.go` - Error definitions
|
||||
- `weed/s3api/s3api_object_handlers_delete.go` - Deletion enforcement
|
||||
- `weed/s3api/s3api_object_handlers_put.go` - Upload enforcement
|
||||
|
||||
## AWS CLI Compatibility
|
||||
|
||||
The retention implementation supports standard AWS CLI commands:
|
||||
|
||||
```bash
|
||||
# Set object retention
|
||||
aws s3api put-object-retention \
|
||||
--bucket mybucket \
|
||||
--key myobject \
|
||||
--retention Mode=GOVERNANCE,RetainUntilDate=2024-12-31T23:59:59Z
|
||||
|
||||
# Get object retention
|
||||
aws s3api get-object-retention \
|
||||
--bucket mybucket \
|
||||
--key myobject
|
||||
|
||||
# Set legal hold
|
||||
aws s3api put-object-legal-hold \
|
||||
--bucket mybucket \
|
||||
--key myobject \
|
||||
--legal-hold Status=ON
|
||||
|
||||
# Configure bucket object lock
|
||||
aws s3api put-object-lock-configuration \
|
||||
--bucket mybucket \
|
||||
--object-lock-configuration ObjectLockEnabled=Enabled,Rule='{DefaultRetention={Mode=GOVERNANCE,Days=30}}'
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
When adding new retention tests:
|
||||
|
||||
1. Follow existing test patterns
|
||||
2. Use descriptive test names
|
||||
3. Include both positive and negative test cases
|
||||
4. Test error conditions
|
||||
5. Update this README with new test descriptions
|
||||
6. Add appropriate Makefile targets for new test categories
|
||||
|
||||
## References
|
||||
|
||||
- [AWS S3 Object Lock Documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock.html)
|
||||
- [AWS S3 API Reference - Object Retention](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectRetention.html)
|
||||
- [SeaweedFS S3 API Documentation](https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API)
|
||||
118
test/s3/retention/object_lock_reproduce_test.go
Normal file
118
test/s3/retention/object_lock_reproduce_test.go
Normal file
@@ -0,0 +1,118 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestReproduceObjectLockIssue reproduces the Object Lock header processing issue step by step
|
||||
func TestReproduceObjectLockIssue(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := fmt.Sprintf("object-lock-test-%d", time.Now().UnixNano())
|
||||
|
||||
t.Logf("=== Reproducing Object Lock Header Processing Issue ===")
|
||||
t.Logf("Bucket name: %s", bucketName)
|
||||
|
||||
cleanupLeftoverTestBuckets(t, client)
|
||||
|
||||
// Step 1: Create bucket with Object Lock enabled header
|
||||
t.Logf("\n1. Creating bucket with ObjectLockEnabledForBucket=true")
|
||||
t.Logf(" This should send x-amz-bucket-object-lock-enabled: true header")
|
||||
|
||||
createResp, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
ObjectLockEnabledForBucket: aws.Bool(true), // This sets the x-amz-bucket-object-lock-enabled header
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Bucket creation failed: %v", err)
|
||||
}
|
||||
t.Logf("Bucket created successfully")
|
||||
t.Logf(" Response: %+v", createResp)
|
||||
|
||||
// Step 2: Check if Object Lock is actually enabled
|
||||
t.Logf("\n2. Checking Object Lock configuration to verify it was enabled")
|
||||
|
||||
objectLockResp, err := client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
t.Logf("GetObjectLockConfiguration FAILED: %v", err)
|
||||
t.Logf(" This demonstrates the issue with header processing!")
|
||||
t.Logf(" S3 clients expect this call to succeed if Object Lock is supported")
|
||||
t.Logf(" When this fails, clients conclude that Object Lock is not supported")
|
||||
|
||||
// This failure demonstrates the bug - the bucket was created but Object Lock wasn't enabled
|
||||
t.Logf("\nBUG CONFIRMED:")
|
||||
t.Logf(" - Bucket creation with ObjectLockEnabledForBucket=true succeeded")
|
||||
t.Logf(" - But GetObjectLockConfiguration fails")
|
||||
t.Logf(" - This means the x-amz-bucket-object-lock-enabled header was ignored")
|
||||
|
||||
} else {
|
||||
t.Logf("GetObjectLockConfiguration succeeded!")
|
||||
t.Logf(" Response: %+v", objectLockResp)
|
||||
t.Logf(" Object Lock is properly enabled - this is the expected behavior")
|
||||
}
|
||||
|
||||
// Step 3: Check versioning status (required for Object Lock)
|
||||
t.Logf("\n3. Checking bucket versioning status (required for Object Lock)")
|
||||
|
||||
versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Logf(" Versioning status: %v", versioningResp.Status)
|
||||
if versioningResp.Status != "Enabled" {
|
||||
t.Logf(" Versioning should be automatically enabled when Object Lock is enabled")
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
t.Logf("\n4. Cleaning up test bucket")
|
||||
_, err = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf(" Warning: Failed to delete bucket: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("\n=== Issue Reproduction Complete ===")
|
||||
t.Logf("Expected behavior after fix:")
|
||||
t.Logf(" - CreateBucket with ObjectLockEnabledForBucket=true should enable Object Lock")
|
||||
t.Logf(" - GetObjectLockConfiguration should return enabled configuration")
|
||||
t.Logf(" - Versioning should be automatically enabled")
|
||||
}
|
||||
|
||||
// TestNormalBucketCreationStillWorks tests that normal bucket creation still works
|
||||
func TestNormalBucketCreationStillWorks(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := fmt.Sprintf("normal-test-%d", time.Now().UnixNano())
|
||||
|
||||
t.Logf("=== Testing Normal Bucket Creation ===")
|
||||
|
||||
cleanupLeftoverTestBuckets(t, client)
|
||||
|
||||
// Create bucket without Object Lock
|
||||
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Logf("Normal bucket creation works")
|
||||
|
||||
// Object Lock should NOT be enabled
|
||||
_, err = client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.Error(t, err, "GetObjectLockConfiguration should fail for bucket without Object Lock")
|
||||
t.Logf("GetObjectLockConfiguration correctly fails for normal bucket")
|
||||
|
||||
// Cleanup
|
||||
client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
|
||||
}
|
||||
119
test/s3/retention/object_lock_validation_test.go
Normal file
119
test/s3/retention/object_lock_validation_test.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestObjectLockValidation tests that S3 Object Lock functionality works end-to-end
|
||||
// This test focuses on the complete Object Lock workflow that S3 clients expect
|
||||
func TestObjectLockValidation(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := fmt.Sprintf("object-lock-test-%d", time.Now().UnixNano())
|
||||
|
||||
t.Logf("=== Validating S3 Object Lock Functionality ===")
|
||||
t.Logf("Bucket: %s", bucketName)
|
||||
|
||||
cleanupLeftoverTestBuckets(t, client)
|
||||
|
||||
// Step 1: Create bucket with Object Lock header
|
||||
t.Log("\n1. Creating bucket with x-amz-bucket-object-lock-enabled: true")
|
||||
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
ObjectLockEnabledForBucket: aws.Bool(true), // This sends x-amz-bucket-object-lock-enabled: true
|
||||
})
|
||||
require.NoError(t, err, "Bucket creation should succeed")
|
||||
defer client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
|
||||
t.Log(" Bucket created successfully")
|
||||
|
||||
// Step 2: Check if Object Lock is supported (standard S3 client behavior)
|
||||
t.Log("\n2. Testing Object Lock support detection")
|
||||
_, err = client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err, "GetObjectLockConfiguration should succeed for Object Lock enabled bucket")
|
||||
t.Log(" GetObjectLockConfiguration succeeded - Object Lock is properly enabled")
|
||||
|
||||
// Step 3: Verify versioning is enabled (required for Object Lock)
|
||||
t.Log("\n3. Verifying versioning is automatically enabled")
|
||||
versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, types.BucketVersioningStatusEnabled, versioningResp.Status, "Versioning should be automatically enabled")
|
||||
t.Log(" Versioning automatically enabled")
|
||||
|
||||
// Step 4: Test actual Object Lock functionality
|
||||
t.Log("\n4. Testing Object Lock retention functionality")
|
||||
|
||||
// Create an object
|
||||
key := "protected-object.dat"
|
||||
content := "Important data that needs immutable protection"
|
||||
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(content),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, putResp.VersionId, "Object should have a version ID")
|
||||
t.Log(" Object created with versioning")
|
||||
|
||||
// Apply Object Lock retention
|
||||
retentionUntil := time.Now().Add(24 * time.Hour)
|
||||
_, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeCompliance,
|
||||
RetainUntilDate: aws.Time(retentionUntil),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Setting Object Lock retention should succeed")
|
||||
t.Log(" Object Lock retention applied successfully")
|
||||
|
||||
// Verify retention allows simple DELETE (creates delete marker) but blocks version deletion
|
||||
// AWS S3 behavior: Simple DELETE (without version ID) is ALWAYS allowed and creates delete marker
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Simple DELETE should succeed and create delete marker (AWS S3 behavior)")
|
||||
t.Log(" Simple DELETE succeeded (creates delete marker - correct AWS behavior)")
|
||||
|
||||
// Now verify that DELETE with version ID is properly blocked by retention
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
})
|
||||
require.Error(t, err, "DELETE with version ID should be blocked by COMPLIANCE retention")
|
||||
t.Log(" Object version is properly protected by retention policy")
|
||||
|
||||
// Verify we can read the object version (should still work)
|
||||
// Note: Need to specify version ID since latest version is now a delete marker
|
||||
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
})
|
||||
require.NoError(t, err, "Reading protected object version should still work")
|
||||
defer getResp.Body.Close()
|
||||
t.Log(" Protected object can still be read")
|
||||
|
||||
t.Log("\nS3 OBJECT LOCK VALIDATION SUCCESSFUL!")
|
||||
t.Log(" - Bucket creation with Object Lock header works")
|
||||
t.Log(" - Object Lock support detection works (GetObjectLockConfiguration succeeds)")
|
||||
t.Log(" - Versioning is automatically enabled")
|
||||
t.Log(" - Object Lock retention functionality works")
|
||||
t.Log(" - Objects are properly protected from deletion")
|
||||
t.Log("")
|
||||
t.Log("S3 clients will now recognize SeaweedFS as supporting Object Lock!")
|
||||
}
|
||||
249
test/s3/retention/s3_bucket_delete_with_lock_test.go
Normal file
249
test/s3/retention/s3_bucket_delete_with_lock_test.go
Normal file
@@ -0,0 +1,249 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestBucketDeletionWithObjectLock tests that buckets with object lock enabled
|
||||
// cannot be deleted if they contain objects with active retention or legal hold
|
||||
func TestBucketDeletionWithObjectLock(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket with object lock enabled
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
|
||||
// Table-driven test for retention modes
|
||||
retentionTestCases := []struct {
|
||||
name string
|
||||
lockMode types.ObjectLockMode
|
||||
key string
|
||||
content string
|
||||
}{
|
||||
{
|
||||
name: "ComplianceRetention",
|
||||
lockMode: types.ObjectLockModeCompliance,
|
||||
key: "test-compliance-retention",
|
||||
content: "test content for compliance retention",
|
||||
},
|
||||
{
|
||||
name: "GovernanceRetention",
|
||||
lockMode: types.ObjectLockModeGovernance,
|
||||
key: "test-governance-retention",
|
||||
content: "test content for governance retention",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range retentionTestCases {
|
||||
t.Run(fmt.Sprintf("CannotDeleteBucketWith%s", tc.name), func(t *testing.T) {
|
||||
retainUntilDate := time.Now().Add(10 * time.Second) // 10 seconds in future
|
||||
|
||||
// Upload object with retention
|
||||
_, err := client.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(tc.key),
|
||||
Body: strings.NewReader(tc.content),
|
||||
ObjectLockMode: tc.lockMode,
|
||||
ObjectLockRetainUntilDate: aws.Time(retainUntilDate),
|
||||
})
|
||||
require.NoError(t, err, "PutObject with %s should succeed", tc.name)
|
||||
|
||||
// Try to delete bucket - should fail because object has active retention
|
||||
_, err = client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.Error(t, err, "DeleteBucket should fail when objects have active retention")
|
||||
assert.Contains(t, err.Error(), "BucketNotEmpty", "Error should be BucketNotEmpty")
|
||||
t.Logf("Expected error: %v", err)
|
||||
|
||||
// Wait for retention to expire with dynamic sleep based on actual retention time
|
||||
t.Logf("Waiting for %s to expire...", tc.name)
|
||||
time.Sleep(time.Until(retainUntilDate) + time.Second)
|
||||
|
||||
// Delete the object
|
||||
_, err = client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(tc.key),
|
||||
})
|
||||
require.NoError(t, err, "DeleteObject should succeed after retention expires")
|
||||
|
||||
// Clean up versions
|
||||
deleteAllObjectVersions(t, client, bucketName)
|
||||
})
|
||||
}
|
||||
|
||||
// Test 3: Bucket deletion with legal hold should fail
|
||||
t.Run("CannotDeleteBucketWithLegalHold", func(t *testing.T) {
|
||||
key := "test-legal-hold"
|
||||
content := "test content for legal hold"
|
||||
|
||||
// Upload object first
|
||||
_, err := client.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(content),
|
||||
})
|
||||
require.NoError(t, err, "PutObject should succeed")
|
||||
|
||||
// Set legal hold on the object
|
||||
_, err = client.PutObjectLegalHold(context.Background(), &s3.PutObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
LegalHold: &types.ObjectLockLegalHold{Status: types.ObjectLockLegalHoldStatusOn},
|
||||
})
|
||||
require.NoError(t, err, "PutObjectLegalHold should succeed")
|
||||
|
||||
// Try to delete bucket - should fail because object has active legal hold
|
||||
_, err = client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.Error(t, err, "DeleteBucket should fail when objects have active legal hold")
|
||||
assert.Contains(t, err.Error(), "BucketNotEmpty", "Error should be BucketNotEmpty")
|
||||
t.Logf("Expected error: %v", err)
|
||||
|
||||
// Remove legal hold
|
||||
_, err = client.PutObjectLegalHold(context.Background(), &s3.PutObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
LegalHold: &types.ObjectLockLegalHold{Status: types.ObjectLockLegalHoldStatusOff},
|
||||
})
|
||||
require.NoError(t, err, "Removing legal hold should succeed")
|
||||
|
||||
// Delete the object
|
||||
_, err = client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "DeleteObject should succeed after legal hold is removed")
|
||||
|
||||
// Clean up versions
|
||||
deleteAllObjectVersions(t, client, bucketName)
|
||||
})
|
||||
|
||||
// Test 4: Bucket deletion should succeed when no objects have active locks
|
||||
t.Run("CanDeleteBucketWithoutActiveLocks", func(t *testing.T) {
|
||||
// Make sure all objects are deleted
|
||||
deleteAllObjectVersions(t, client, bucketName)
|
||||
|
||||
// Use retry mechanism for eventual consistency instead of fixed sleep
|
||||
require.Eventually(t, func() bool {
|
||||
_, err := client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf("Retrying DeleteBucket due to: %v", err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}, 5*time.Second, 500*time.Millisecond, "DeleteBucket should succeed when no objects have active locks")
|
||||
|
||||
t.Logf("Successfully deleted bucket without active locks")
|
||||
})
|
||||
}
|
||||
|
||||
// TestBucketDeletionWithVersionedLocks tests deletion with versioned objects under lock
|
||||
func TestBucketDeletionWithVersionedLocks(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket with object lock enabled
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName) // Best effort cleanup
|
||||
|
||||
key := "test-versioned-locks"
|
||||
content1 := "version 1 content"
|
||||
content2 := "version 2 content"
|
||||
retainUntilDate := time.Now().Add(10 * time.Second)
|
||||
|
||||
// Upload first version with retention
|
||||
putResp1, err := client.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(content1),
|
||||
ObjectLockMode: types.ObjectLockModeGovernance,
|
||||
ObjectLockRetainUntilDate: aws.Time(retainUntilDate),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
version1 := *putResp1.VersionId
|
||||
|
||||
// Upload second version with retention
|
||||
putResp2, err := client.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(content2),
|
||||
ObjectLockMode: types.ObjectLockModeGovernance,
|
||||
ObjectLockRetainUntilDate: aws.Time(retainUntilDate),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
version2 := *putResp2.VersionId
|
||||
|
||||
t.Logf("Created two versions: %s, %s", version1, version2)
|
||||
|
||||
// Try to delete bucket - should fail because versions have active retention
|
||||
_, err = client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.Error(t, err, "DeleteBucket should fail when object versions have active retention")
|
||||
assert.Contains(t, err.Error(), "BucketNotEmpty", "Error should be BucketNotEmpty")
|
||||
t.Logf("Expected error: %v", err)
|
||||
|
||||
// Wait for retention to expire with dynamic sleep based on actual retention time
|
||||
t.Logf("Waiting for retention to expire on all versions...")
|
||||
time.Sleep(time.Until(retainUntilDate) + time.Second)
|
||||
|
||||
// Clean up all versions
|
||||
deleteAllObjectVersions(t, client, bucketName)
|
||||
|
||||
// Wait for eventual consistency and attempt to delete the bucket with retry
|
||||
require.Eventually(t, func() bool {
|
||||
_, err := client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf("Retrying DeleteBucket due to: %v", err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}, 5*time.Second, 500*time.Millisecond, "DeleteBucket should succeed after all locks expire")
|
||||
|
||||
t.Logf("Successfully deleted bucket after locks expired")
|
||||
}
|
||||
|
||||
// TestBucketDeletionWithoutObjectLock tests that buckets without object lock can be deleted normally
|
||||
func TestBucketDeletionWithoutObjectLock(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create regular bucket without object lock
|
||||
createBucket(t, client, bucketName)
|
||||
|
||||
// Upload some objects
|
||||
for i := 0; i < 3; i++ {
|
||||
_, err := client.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(fmt.Sprintf("test-object-%d", i)),
|
||||
Body: strings.NewReader("test content"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Delete all objects
|
||||
deleteAllObjectVersions(t, client, bucketName)
|
||||
|
||||
// Delete bucket should succeed
|
||||
_, err := client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err, "DeleteBucket should succeed for regular bucket")
|
||||
t.Logf("Successfully deleted regular bucket without object lock")
|
||||
}
|
||||
185
test/s3/retention/s3_bucket_object_lock_test.go
Normal file
185
test/s3/retention/s3_bucket_object_lock_test.go
Normal file
@@ -0,0 +1,185 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestBucketCreationWithObjectLockEnabled tests creating a bucket with the
|
||||
// x-amz-bucket-object-lock-enabled header, which is required for S3 Object Lock compatibility
|
||||
func TestBucketCreationWithObjectLockEnabled(t *testing.T) {
|
||||
// This test verifies that bucket creation with
|
||||
// x-amz-bucket-object-lock-enabled header should automatically enable Object Lock
|
||||
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
defer func() {
|
||||
// Best effort cleanup
|
||||
deleteBucket(t, client, bucketName)
|
||||
}()
|
||||
|
||||
// Test 1: Create bucket with Object Lock enabled header using custom HTTP client
|
||||
t.Run("CreateBucketWithObjectLockHeader", func(t *testing.T) {
|
||||
// Create bucket with x-amz-bucket-object-lock-enabled header
|
||||
// This simulates what S3 clients do when testing Object Lock support
|
||||
createResp, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
ObjectLockEnabledForBucket: aws.Bool(true), // This should set x-amz-bucket-object-lock-enabled header
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, createResp)
|
||||
|
||||
// Verify bucket was created
|
||||
_, err = client.HeadBucket(context.TODO(), &s3.HeadBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
// Test 2: Verify that Object Lock is automatically enabled for the bucket
|
||||
t.Run("VerifyObjectLockAutoEnabled", func(t *testing.T) {
|
||||
// Try to get the Object Lock configuration
|
||||
// If the header was processed correctly, this should return an enabled configuration
|
||||
configResp, err := client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
|
||||
require.NoError(t, err, "GetObjectLockConfiguration should not fail if Object Lock is enabled")
|
||||
require.NotNil(t, configResp.ObjectLockConfiguration, "ObjectLockConfiguration should not be nil")
|
||||
assert.Equal(t, types.ObjectLockEnabledEnabled, configResp.ObjectLockConfiguration.ObjectLockEnabled, "Object Lock should be enabled")
|
||||
})
|
||||
|
||||
// Test 3: Verify versioning is automatically enabled (required for Object Lock)
|
||||
t.Run("VerifyVersioningAutoEnabled", func(t *testing.T) {
|
||||
// Object Lock requires versioning to be enabled
|
||||
// When Object Lock is enabled via header, versioning should also be enabled automatically
|
||||
versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Versioning should be automatically enabled for Object Lock
|
||||
assert.Equal(t, types.BucketVersioningStatusEnabled, versioningResp.Status, "Versioning should be automatically enabled for Object Lock")
|
||||
})
|
||||
}
|
||||
|
||||
// TestBucketCreationWithoutObjectLockHeader tests normal bucket creation
|
||||
// to ensure we don't break existing functionality
|
||||
func TestBucketCreationWithoutObjectLockHeader(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
// Create bucket without Object Lock header
|
||||
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify bucket was created
|
||||
_, err = client.HeadBucket(context.TODO(), &s3.HeadBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Object Lock should NOT be enabled
|
||||
_, err = client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
// This should fail since Object Lock is not enabled
|
||||
require.Error(t, err)
|
||||
t.Logf("GetObjectLockConfiguration correctly failed for bucket without Object Lock: %v", err)
|
||||
|
||||
// Versioning should not be enabled by default
|
||||
versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should be either empty/unset or Suspended, but not Enabled
|
||||
if versioningResp.Status != types.BucketVersioningStatusEnabled {
|
||||
t.Logf("Versioning correctly not enabled: %v", versioningResp.Status)
|
||||
} else {
|
||||
t.Errorf("Versioning should not be enabled for bucket without Object Lock header")
|
||||
}
|
||||
}
|
||||
|
||||
// TestS3ObjectLockWorkflow tests the complete Object Lock workflow that S3 clients would use
|
||||
func TestS3ObjectLockWorkflow(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
// Step 1: Client creates bucket with Object Lock enabled
|
||||
t.Run("ClientCreatesBucket", func(t *testing.T) {
|
||||
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
ObjectLockEnabledForBucket: aws.Bool(true),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
// Step 2: Client checks if Object Lock is supported by getting the configuration
|
||||
t.Run("ClientChecksObjectLockSupport", func(t *testing.T) {
|
||||
configResp, err := client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
|
||||
require.NoError(t, err, "Object Lock configuration check should succeed")
|
||||
|
||||
// S3 clients should see Object Lock is enabled
|
||||
require.NotNil(t, configResp.ObjectLockConfiguration)
|
||||
assert.Equal(t, types.ObjectLockEnabledEnabled, configResp.ObjectLockConfiguration.ObjectLockEnabled)
|
||||
t.Log("Object Lock configuration retrieved successfully - S3 clients would see this as supported")
|
||||
})
|
||||
|
||||
// Step 3: Client would then configure retention policies and use Object Lock
|
||||
t.Run("ClientConfiguresRetention", func(t *testing.T) {
|
||||
// Verify versioning is automatically enabled (required for Object Lock)
|
||||
versioningResp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, types.BucketVersioningStatusEnabled, versioningResp.Status, "Versioning should be automatically enabled")
|
||||
|
||||
// Create an object
|
||||
key := "protected-backup-object"
|
||||
content := "Backup data with Object Lock protection"
|
||||
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(content),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// Set Object Lock retention (what backup clients do to protect data)
|
||||
retentionUntil := time.Now().Add(24 * time.Hour)
|
||||
_, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeCompliance,
|
||||
RetainUntilDate: aws.Time(retentionUntil),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify object is protected
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.Error(t, err, "Object should be protected by retention policy")
|
||||
|
||||
t.Log("Object Lock retention successfully applied - data is immutable")
|
||||
})
|
||||
}
|
||||
231
test/s3/retention/s3_object_lock_directory_marker_test.go
Normal file
231
test/s3/retention/s3_object_lock_directory_marker_test.go
Normal file
@@ -0,0 +1,231 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func requireAccessDenied(t *testing.T, err error, msg string) {
|
||||
t.Helper()
|
||||
require.Error(t, err, msg)
|
||||
var apiErr smithy.APIError
|
||||
require.True(t, errors.As(err, &apiErr), "expected an API error, got %T", err)
|
||||
assert.Equal(t, "AccessDenied", apiErr.ErrorCode(), msg)
|
||||
}
|
||||
|
||||
// A key ending in "/" is stored as the filer directory rather than as an object
|
||||
// beside it, and is deleted the unversioned way. Object Lock still covers it: the
|
||||
// gateway lists it as an object and serves retention set on it.
|
||||
func TestObjectLockDirectoryMarker(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
retainUntil := time.Now().Add(24 * time.Hour)
|
||||
|
||||
t.Run("retention headers are honored, not dropped", func(t *testing.T) {
|
||||
key := "records/evidence/"
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader("marker"),
|
||||
ObjectLockMode: types.ObjectLockModeCompliance,
|
||||
ObjectLockRetainUntilDate: aws.Time(retainUntil),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
requireAccessDenied(t, err, "a retained marker must not be deletable")
|
||||
|
||||
_, err = client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
assert.NoError(t, err, "the marker must still be there after the refused delete")
|
||||
})
|
||||
|
||||
t.Run("retention set through PutObjectRetention is honored", func(t *testing.T) {
|
||||
key := "records/ledger/"
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader("marker"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeCompliance,
|
||||
RetainUntilDate: aws.Time(retainUntil),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
requireAccessDenied(t, err, "retention the gateway serves back must also block the delete")
|
||||
|
||||
_, err = client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
assert.NoError(t, err, "the marker must still be there after the refused delete")
|
||||
})
|
||||
|
||||
t.Run("multi-object delete is refused too", func(t *testing.T) {
|
||||
key := "records/batch/"
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader("marker"),
|
||||
ObjectLockMode: types.ObjectLockModeCompliance,
|
||||
ObjectLockRetainUntilDate: aws.Time(retainUntil),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
resp, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Delete: &types.Delete{Objects: []types.ObjectIdentifier{{Key: aws.String(key)}}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, resp.Deleted, "a retained marker must not be reported deleted")
|
||||
require.Len(t, resp.Errors, 1)
|
||||
assert.Equal(t, key, aws.ToString(resp.Errors[0].Key))
|
||||
assert.Equal(t, "AccessDenied", aws.ToString(resp.Errors[0].Code))
|
||||
|
||||
_, err = client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
assert.NoError(t, err, "the marker must survive the batch delete")
|
||||
})
|
||||
|
||||
t.Run("invalid lock headers are rejected, as on a regular key", func(t *testing.T) {
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("records/bad-mode/"),
|
||||
Body: strings.NewReader("marker"),
|
||||
ObjectLockMode: "INVALID_MODE",
|
||||
ObjectLockRetainUntilDate: aws.Time(retainUntil),
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("records/no-date/"),
|
||||
Body: strings.NewReader("marker"),
|
||||
ObjectLockMode: types.ObjectLockModeGovernance,
|
||||
})
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
// Past 1KiB a trailing-slash key is a real versioned object, and the delete
|
||||
// takes the whole history at once, so an older retained version has to block
|
||||
// it even when the version on top carries no retention of its own.
|
||||
t.Run("a retained version under an unretained one still blocks", func(t *testing.T) {
|
||||
key := "records/history/"
|
||||
body := strings.Repeat("x", 2048)
|
||||
|
||||
first, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(body),
|
||||
ObjectLockMode: types.ObjectLockModeCompliance,
|
||||
ObjectLockRetainUntilDate: aws.Time(retainUntil),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, first.VersionId)
|
||||
|
||||
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(body),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
requireAccessDenied(t, err, "the retained version underneath must block the delete")
|
||||
|
||||
_, err = client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: first.VersionId,
|
||||
})
|
||||
assert.NoError(t, err, "the retained version must survive")
|
||||
})
|
||||
|
||||
// mkdir replaces the marker entry outright, taking its lock metadata with it,
|
||||
// so a plain PUT over a retained marker has to be refused - including once the
|
||||
// key has grown a version history that the latest-version lookup would find
|
||||
// instead of the marker.
|
||||
t.Run("a plain PUT cannot replace a retained marker", func(t *testing.T) {
|
||||
key := "records/overwrite/"
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader("marker"),
|
||||
ObjectLockMode: types.ObjectLockModeCompliance,
|
||||
ObjectLockRetainUntilDate: aws.Time(retainUntil),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader("replacement"),
|
||||
})
|
||||
requireAccessDenied(t, err, "a retained marker must not be replaceable")
|
||||
|
||||
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(strings.Repeat("x", 2048)),
|
||||
})
|
||||
require.NoError(t, err, "a versioned write of the same key adds a version")
|
||||
|
||||
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader("replacement"),
|
||||
})
|
||||
requireAccessDenied(t, err, "the history must not hide the marker's own lock")
|
||||
})
|
||||
|
||||
t.Run("an unretained marker still deletes", func(t *testing.T) {
|
||||
key := "records/plain/"
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader("marker"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
}
|
||||
308
test/s3/retention/s3_object_lock_headers_test.go
Normal file
308
test/s3/retention/s3_object_lock_headers_test.go
Normal file
@@ -0,0 +1,308 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestPutObjectWithLockHeaders tests that object lock headers in PUT requests
|
||||
// are properly stored and returned in HEAD responses
|
||||
func TestPutObjectWithLockHeaders(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket with object lock enabled and versioning
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
key := "test-object-lock-headers"
|
||||
content := "test content with object lock headers"
|
||||
retainUntilDate := time.Now().Add(24 * time.Hour)
|
||||
|
||||
// Test 1: PUT with COMPLIANCE mode and retention date
|
||||
t.Run("PUT with COMPLIANCE mode", func(t *testing.T) {
|
||||
testKey := key + "-compliance"
|
||||
|
||||
// PUT object with lock headers
|
||||
putResp := putObjectWithLockHeaders(t, client, bucketName, testKey, content,
|
||||
"COMPLIANCE", retainUntilDate, "")
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// HEAD object and verify lock headers are returned
|
||||
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(testKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify object lock metadata is present in response
|
||||
assert.Equal(t, types.ObjectLockModeCompliance, headResp.ObjectLockMode)
|
||||
assert.NotNil(t, headResp.ObjectLockRetainUntilDate)
|
||||
assert.WithinDuration(t, retainUntilDate, *headResp.ObjectLockRetainUntilDate, 5*time.Second)
|
||||
})
|
||||
|
||||
// Test 2: PUT with GOVERNANCE mode and retention date
|
||||
t.Run("PUT with GOVERNANCE mode", func(t *testing.T) {
|
||||
testKey := key + "-governance"
|
||||
|
||||
putResp := putObjectWithLockHeaders(t, client, bucketName, testKey, content,
|
||||
"GOVERNANCE", retainUntilDate, "")
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(testKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, types.ObjectLockModeGovernance, headResp.ObjectLockMode)
|
||||
assert.NotNil(t, headResp.ObjectLockRetainUntilDate)
|
||||
assert.WithinDuration(t, retainUntilDate, *headResp.ObjectLockRetainUntilDate, 5*time.Second)
|
||||
})
|
||||
|
||||
// Test 3: PUT with legal hold
|
||||
t.Run("PUT with legal hold", func(t *testing.T) {
|
||||
testKey := key + "-legal-hold"
|
||||
|
||||
putResp := putObjectWithLockHeaders(t, client, bucketName, testKey, content,
|
||||
"", time.Time{}, "ON")
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(testKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, headResp.ObjectLockLegalHoldStatus)
|
||||
})
|
||||
|
||||
// Test 4: PUT with both retention and legal hold
|
||||
t.Run("PUT with both retention and legal hold", func(t *testing.T) {
|
||||
testKey := key + "-both"
|
||||
|
||||
putResp := putObjectWithLockHeaders(t, client, bucketName, testKey, content,
|
||||
"GOVERNANCE", retainUntilDate, "ON")
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(testKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, types.ObjectLockModeGovernance, headResp.ObjectLockMode)
|
||||
assert.NotNil(t, headResp.ObjectLockRetainUntilDate)
|
||||
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, headResp.ObjectLockLegalHoldStatus)
|
||||
})
|
||||
}
|
||||
|
||||
// TestGetObjectWithLockHeaders verifies that GET requests also return object lock metadata
|
||||
func TestGetObjectWithLockHeaders(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
key := "test-get-object-lock"
|
||||
content := "test content for GET with lock headers"
|
||||
retainUntilDate := time.Now().Add(24 * time.Hour)
|
||||
|
||||
// PUT object with lock headers
|
||||
putResp := putObjectWithLockHeaders(t, client, bucketName, key, content,
|
||||
"COMPLIANCE", retainUntilDate, "ON")
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// GET object and verify lock headers are returned
|
||||
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer getResp.Body.Close()
|
||||
|
||||
// Verify object lock metadata is present in GET response
|
||||
assert.Equal(t, types.ObjectLockModeCompliance, getResp.ObjectLockMode)
|
||||
assert.NotNil(t, getResp.ObjectLockRetainUntilDate)
|
||||
assert.WithinDuration(t, retainUntilDate, *getResp.ObjectLockRetainUntilDate, 5*time.Second)
|
||||
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, getResp.ObjectLockLegalHoldStatus)
|
||||
}
|
||||
|
||||
// TestVersionedObjectLockHeaders tests object lock headers work with versioned objects
|
||||
func TestVersionedObjectLockHeaders(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
key := "test-versioned-lock"
|
||||
content1 := "version 1 content"
|
||||
content2 := "version 2 content"
|
||||
retainUntilDate1 := time.Now().Add(12 * time.Hour)
|
||||
retainUntilDate2 := time.Now().Add(24 * time.Hour)
|
||||
|
||||
// PUT first version with GOVERNANCE mode
|
||||
putResp1 := putObjectWithLockHeaders(t, client, bucketName, key, content1,
|
||||
"GOVERNANCE", retainUntilDate1, "")
|
||||
require.NotNil(t, putResp1.VersionId)
|
||||
|
||||
// PUT second version with COMPLIANCE mode
|
||||
putResp2 := putObjectWithLockHeaders(t, client, bucketName, key, content2,
|
||||
"COMPLIANCE", retainUntilDate2, "ON")
|
||||
require.NotNil(t, putResp2.VersionId)
|
||||
require.NotEqual(t, *putResp1.VersionId, *putResp2.VersionId)
|
||||
|
||||
// HEAD latest version (version 2)
|
||||
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, types.ObjectLockModeCompliance, headResp.ObjectLockMode)
|
||||
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, headResp.ObjectLockLegalHoldStatus)
|
||||
|
||||
// HEAD specific version 1
|
||||
headResp1, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp1.VersionId,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, types.ObjectLockModeGovernance, headResp1.ObjectLockMode)
|
||||
assert.NotEqual(t, types.ObjectLockLegalHoldStatusOn, headResp1.ObjectLockLegalHoldStatus)
|
||||
}
|
||||
|
||||
// TestObjectLockHeadersErrorCases tests various error scenarios
|
||||
func TestObjectLockHeadersErrorCases(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucketWithObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
key := "test-error-cases"
|
||||
content := "test content for error cases"
|
||||
|
||||
// Test 1: Invalid retention mode should be rejected
|
||||
t.Run("Invalid retention mode", func(t *testing.T) {
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key + "-invalid-mode"),
|
||||
Body: strings.NewReader(content),
|
||||
ObjectLockMode: "INVALID_MODE", // Invalid mode
|
||||
ObjectLockRetainUntilDate: aws.Time(time.Now().Add(24 * time.Hour)),
|
||||
})
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
// Test 2: Retention date in the past should be rejected
|
||||
t.Run("Past retention date", func(t *testing.T) {
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key + "-past-date"),
|
||||
Body: strings.NewReader(content),
|
||||
ObjectLockMode: "GOVERNANCE",
|
||||
ObjectLockRetainUntilDate: aws.Time(time.Now().Add(-24 * time.Hour)), // Past date
|
||||
})
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
// Test 3: Mode without date should be rejected
|
||||
t.Run("Mode without retention date", func(t *testing.T) {
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key + "-no-date"),
|
||||
Body: strings.NewReader(content),
|
||||
ObjectLockMode: "GOVERNANCE",
|
||||
// Missing ObjectLockRetainUntilDate
|
||||
})
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
// TestObjectLockHeadersNonVersionedBucket tests that object lock fails on non-versioned buckets
|
||||
func TestObjectLockHeadersNonVersionedBucket(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create regular bucket without object lock/versioning
|
||||
createBucketWithoutObjectLock(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
key := "test-non-versioned"
|
||||
content := "test content"
|
||||
retainUntilDate := time.Now().Add(24 * time.Hour)
|
||||
|
||||
// Attempting to PUT with object lock headers should fail
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(content),
|
||||
ObjectLockMode: "GOVERNANCE",
|
||||
ObjectLockRetainUntilDate: aws.Time(retainUntilDate),
|
||||
})
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
// Helper Functions
|
||||
|
||||
// putObjectWithLockHeaders puts an object with object lock headers
|
||||
func putObjectWithLockHeaders(t *testing.T, client *s3.Client, bucketName, key, content string,
|
||||
mode string, retainUntilDate time.Time, legalHold string) *s3.PutObjectOutput {
|
||||
|
||||
input := &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(content),
|
||||
}
|
||||
|
||||
// Add retention mode and date if specified
|
||||
if mode != "" {
|
||||
switch mode {
|
||||
case "COMPLIANCE":
|
||||
input.ObjectLockMode = types.ObjectLockModeCompliance
|
||||
case "GOVERNANCE":
|
||||
input.ObjectLockMode = types.ObjectLockModeGovernance
|
||||
}
|
||||
if !retainUntilDate.IsZero() {
|
||||
input.ObjectLockRetainUntilDate = aws.Time(retainUntilDate)
|
||||
}
|
||||
}
|
||||
|
||||
// Add legal hold if specified
|
||||
if legalHold != "" {
|
||||
switch legalHold {
|
||||
case "ON":
|
||||
input.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
|
||||
case "OFF":
|
||||
input.ObjectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOff
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := client.PutObject(context.TODO(), input)
|
||||
require.NoError(t, err)
|
||||
return resp
|
||||
}
|
||||
|
||||
// createBucketWithObjectLock creates a bucket with object lock enabled
|
||||
func createBucketWithObjectLock(t *testing.T, client *s3.Client, bucketName string) {
|
||||
cleanupLeftoverTestBuckets(t, client)
|
||||
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
ObjectLockEnabledForBucket: aws.Bool(true),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Enable versioning (required for object lock)
|
||||
enableVersioning(t, client, bucketName)
|
||||
}
|
||||
808
test/s3/retention/s3_retention_test.go
Normal file
808
test/s3/retention/s3_retention_test.go
Normal file
@@ -0,0 +1,808 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// S3TestConfig holds configuration for S3 tests
|
||||
type S3TestConfig struct {
|
||||
Endpoint string
|
||||
MasterEndpoint string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
Region string
|
||||
BucketPrefix string
|
||||
UseSSL bool
|
||||
SkipVerifySSL bool
|
||||
}
|
||||
|
||||
// Default test configuration - should match test_config.json
|
||||
var defaultConfig = &S3TestConfig{
|
||||
Endpoint: "http://localhost:8333", // Default SeaweedFS S3 port
|
||||
MasterEndpoint: "http://localhost:9333", // Default SeaweedFS master HTTP port
|
||||
AccessKey: "some_access_key1",
|
||||
SecretKey: "some_secret_key1",
|
||||
Region: "us-east-1",
|
||||
BucketPrefix: "test-retention-",
|
||||
UseSSL: false,
|
||||
SkipVerifySSL: true,
|
||||
}
|
||||
|
||||
// getS3Client creates an AWS S3 client for testing
|
||||
func getS3Client(t *testing.T) *s3.Client {
|
||||
cfg, err := config.LoadDefaultConfig(context.TODO(),
|
||||
config.WithRegion(defaultConfig.Region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
||||
defaultConfig.AccessKey,
|
||||
defaultConfig.SecretKey,
|
||||
"",
|
||||
)),
|
||||
config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
|
||||
func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
URL: defaultConfig.Endpoint,
|
||||
SigningRegion: defaultConfig.Region,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
})),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
return s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.UsePathStyle = true // Important for SeaweedFS
|
||||
})
|
||||
}
|
||||
|
||||
// allTestBucketPrefixes lists every prefix used to name buckets in this test suite.
|
||||
// cleanupLeftoverTestBuckets uses it to find stale buckets from prior tests/runs.
|
||||
// Add the new prefix here whenever a test introduces one.
|
||||
var allTestBucketPrefixes = []string{
|
||||
"test-retention-",
|
||||
"object-lock-test-",
|
||||
"object-lock-config-",
|
||||
"bucket-defaults-",
|
||||
"normal-test-",
|
||||
}
|
||||
|
||||
// getNewBucketName generates a unique bucket name
|
||||
func getNewBucketName() string {
|
||||
timestamp := time.Now().UnixNano()
|
||||
return fmt.Sprintf("%s%d", defaultConfig.BucketPrefix, timestamp)
|
||||
}
|
||||
|
||||
// createBucket creates a new bucket for testing with Object Lock enabled
|
||||
// Object Lock is required for retention and legal hold functionality per AWS S3 specification
|
||||
func createBucket(t *testing.T, client *s3.Client, bucketName string) {
|
||||
cleanupLeftoverTestBuckets(t, client)
|
||||
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
ObjectLockEnabledForBucket: aws.Bool(true),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// createBucketWithoutObjectLock creates a new bucket without Object Lock enabled
|
||||
// Use this only for tests that specifically need to verify non-Object-Lock bucket behavior
|
||||
func createBucketWithoutObjectLock(t *testing.T, client *s3.Client, bucketName string) {
|
||||
cleanupLeftoverTestBuckets(t, client)
|
||||
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// deleteBucket deletes a bucket and all its contents
|
||||
func deleteBucket(t *testing.T, client *s3.Client, bucketName string) {
|
||||
// First, try to delete all objects and versions
|
||||
err := deleteAllObjectVersions(t, client, bucketName)
|
||||
if err != nil {
|
||||
t.Logf("Warning: failed to delete all object versions in first attempt: %v", err)
|
||||
// Try once more in case of transient errors
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
err = deleteAllObjectVersions(t, client, bucketName)
|
||||
if err != nil {
|
||||
t.Logf("Warning: failed to delete all object versions in second attempt: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait a bit for eventual consistency
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Try to delete the bucket multiple times in case of eventual consistency issues.
|
||||
// Always force-drop the underlying collection at the master afterwards: COMPLIANCE-mode
|
||||
// retention can leave undeletable objects, so the S3 DeleteBucket may keep failing with
|
||||
// BucketNotEmpty and leak the collection's volumes. Without this, running enough tests
|
||||
// on a single `weed mini` server exhausts the data node's volume slots and every
|
||||
// subsequent PutObject 500s with "Not enough data nodes found".
|
||||
defer forceDeleteCollection(t, bucketName)
|
||||
for retries := 0; retries < 3; retries++ {
|
||||
_, err = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
if err == nil {
|
||||
t.Logf("Successfully deleted bucket %s", bucketName)
|
||||
return
|
||||
}
|
||||
|
||||
t.Logf("Warning: failed to delete bucket %s (attempt %d): %v", bucketName, retries+1, err)
|
||||
if retries < 2 {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// forceDeleteCollection drops the SeaweedFS collection backing a test bucket via the master's
|
||||
// /col/delete admin endpoint. The S3 layer normally drops the collection on DeleteBucket, but
|
||||
// when retention/legal-hold blocks the bucket cleanup, the collection (and its reserved
|
||||
// volumes) leaks. Best-effort: a 400 from the master means the collection was already gone,
|
||||
// which is the success path and not an error.
|
||||
func forceDeleteCollection(t *testing.T, bucketName string) {
|
||||
if defaultConfig.MasterEndpoint == "" {
|
||||
return
|
||||
}
|
||||
endpoint := strings.TrimRight(defaultConfig.MasterEndpoint, "/") + "/col/delete?collection=" + url.QueryEscape(bucketName)
|
||||
req, err := http.NewRequest(http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
t.Logf("Note: building collection delete request for %s failed: %v", bucketName, err)
|
||||
return
|
||||
}
|
||||
httpClient := &http.Client{Timeout: 5 * time.Second}
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
t.Logf("Note: force-delete collection %s failed: %v", bucketName, err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
switch resp.StatusCode {
|
||||
case http.StatusNoContent:
|
||||
t.Logf("Force-deleted collection %s", bucketName)
|
||||
case http.StatusBadRequest:
|
||||
// Collection already gone - normal path when DeleteBucket succeeded.
|
||||
default:
|
||||
t.Logf("Note: force-delete collection %s returned HTTP %d", bucketName, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
// deleteAllObjectVersions deletes all object versions in a bucket
|
||||
func deleteAllObjectVersions(t *testing.T, client *s3.Client, bucketName string) error {
|
||||
// List all object versions
|
||||
paginator := s3.NewListObjectVersionsPaginator(client, &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
|
||||
for paginator.HasMorePages() {
|
||||
page, err := paginator.NextPage(context.TODO())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var objectsToDelete []types.ObjectIdentifier
|
||||
|
||||
// Add versions - first try to remove retention/legal hold
|
||||
for _, version := range page.Versions {
|
||||
// Try to remove legal hold if present
|
||||
_, err := client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: version.Key,
|
||||
VersionId: version.VersionId,
|
||||
LegalHold: &types.ObjectLockLegalHold{
|
||||
Status: types.ObjectLockLegalHoldStatusOff,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
// Legal hold might not be set, ignore error
|
||||
t.Logf("Note: could not remove legal hold for %s@%s: %v", *version.Key, *version.VersionId, err)
|
||||
}
|
||||
|
||||
objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{
|
||||
Key: version.Key,
|
||||
VersionId: version.VersionId,
|
||||
})
|
||||
}
|
||||
|
||||
// Add delete markers
|
||||
for _, deleteMarker := range page.DeleteMarkers {
|
||||
objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{
|
||||
Key: deleteMarker.Key,
|
||||
VersionId: deleteMarker.VersionId,
|
||||
})
|
||||
}
|
||||
|
||||
// Delete objects in batches with bypass governance retention
|
||||
if len(objectsToDelete) > 0 {
|
||||
_, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
BypassGovernanceRetention: aws.Bool(true),
|
||||
Delete: &types.Delete{
|
||||
Objects: objectsToDelete,
|
||||
Quiet: aws.Bool(true),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf("Warning: batch delete failed, trying individual deletion: %v", err)
|
||||
// Try individual deletion for each object
|
||||
for _, obj := range objectsToDelete {
|
||||
_, delErr := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: obj.Key,
|
||||
VersionId: obj.VersionId,
|
||||
BypassGovernanceRetention: aws.Bool(true),
|
||||
})
|
||||
if delErr != nil {
|
||||
t.Logf("Warning: failed to delete object %s@%s: %v", *obj.Key, *obj.VersionId, delErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// enableVersioning enables versioning on a bucket
|
||||
func enableVersioning(t *testing.T, client *s3.Client, bucketName string) {
|
||||
_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
VersioningConfiguration: &types.VersioningConfiguration{
|
||||
Status: types.BucketVersioningStatusEnabled,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// putObject puts an object into a bucket
|
||||
func putObject(t *testing.T, client *s3.Client, bucketName, key, content string) *s3.PutObjectOutput {
|
||||
resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(content),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return resp
|
||||
}
|
||||
|
||||
// cleanupAllTestBuckets cleans up any leftover test buckets matching any prefix this
|
||||
// suite uses. Called both at test-suite teardown and before each new bucket creation
|
||||
// so a single `weed mini` data node does not exhaust its volume slots after many tests.
|
||||
func cleanupAllTestBuckets(t *testing.T, client *s3.Client) {
|
||||
listResp, err := client.ListBuckets(context.TODO(), &s3.ListBucketsInput{})
|
||||
if err != nil {
|
||||
t.Logf("Warning: failed to list buckets for cleanup: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, bucket := range listResp.Buckets {
|
||||
if bucket.Name == nil {
|
||||
continue
|
||||
}
|
||||
for _, prefix := range allTestBucketPrefixes {
|
||||
if strings.HasPrefix(*bucket.Name, prefix) {
|
||||
t.Logf("Cleaning up leftover test bucket: %s", *bucket.Name)
|
||||
deleteBucket(t, client, *bucket.Name)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupLeftoverTestBuckets is invoked from the createBucket* helpers so each new bucket
|
||||
// starts with a clean slate even when a prior test panicked, was interrupted, or its
|
||||
// volumes have not yet been reclaimed by the master. Without this the WORM suite
|
||||
// accumulates ~3 reserved volumes per bucket and the data node hits its volume cap.
|
||||
func cleanupLeftoverTestBuckets(t *testing.T, client *s3.Client) {
|
||||
cleanupAllTestBuckets(t, client)
|
||||
}
|
||||
|
||||
// TestBasicRetentionWorkflow tests the basic retention functionality
|
||||
func TestBasicRetentionWorkflow(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
// Enable versioning (required for retention)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Create object
|
||||
key := "test-object"
|
||||
content := "test content for retention"
|
||||
putResp := putObject(t, client, bucketName, key, content)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// Set retention with GOVERNANCE mode
|
||||
retentionUntil := time.Now().Add(24 * time.Hour)
|
||||
_, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeGovernance,
|
||||
RetainUntilDate: aws.Time(retentionUntil),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get retention and verify it was set correctly
|
||||
retentionResp, err := client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, types.ObjectLockRetentionModeGovernance, retentionResp.Retention.Mode)
|
||||
assert.WithinDuration(t, retentionUntil, *retentionResp.Retention.RetainUntilDate, time.Second)
|
||||
|
||||
// Try to delete object without bypass - should fail
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
// Delete object with bypass governance - should succeed
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
BypassGovernanceRetention: aws.Bool(true),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestRetentionModeCompliance tests COMPLIANCE mode retention
|
||||
func TestRetentionModeCompliance(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Create object
|
||||
key := "compliance-test-object"
|
||||
content := "compliance test content"
|
||||
putResp := putObject(t, client, bucketName, key, content)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// Set retention with COMPLIANCE mode
|
||||
retentionUntil := time.Now().Add(1 * time.Hour)
|
||||
_, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeCompliance,
|
||||
RetainUntilDate: aws.Time(retentionUntil),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get retention and verify
|
||||
retentionResp, err := client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, types.ObjectLockRetentionModeCompliance, retentionResp.Retention.Mode)
|
||||
|
||||
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
|
||||
|
||||
// Try DELETE with version ID - should fail for COMPLIANCE mode
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
})
|
||||
require.Error(t, err, "DELETE with version ID should be blocked by COMPLIANCE retention")
|
||||
|
||||
// Try DELETE with version ID and bypass - should still fail (COMPLIANCE mode ignores bypass)
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
BypassGovernanceRetention: aws.Bool(true),
|
||||
})
|
||||
require.Error(t, err, "COMPLIANCE mode should ignore governance bypass")
|
||||
}
|
||||
|
||||
// TestLegalHoldWorkflow tests legal hold functionality
|
||||
func TestLegalHoldWorkflow(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Create object
|
||||
key := "legal-hold-test-object"
|
||||
content := "legal hold test content"
|
||||
putResp := putObject(t, client, bucketName, key, content)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// Set legal hold ON
|
||||
_, err := client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
LegalHold: &types.ObjectLockLegalHold{
|
||||
Status: types.ObjectLockLegalHoldStatusOn,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get legal hold and verify
|
||||
legalHoldResp, err := client.GetObjectLegalHold(context.TODO(), &s3.GetObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, legalHoldResp.LegalHold.Status)
|
||||
|
||||
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
|
||||
|
||||
// Try DELETE with version ID - should fail due to legal hold
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
})
|
||||
require.Error(t, err, "DELETE with version ID should be blocked by legal hold")
|
||||
|
||||
// Remove legal hold (must specify version ID since latest version is now delete marker)
|
||||
_, err = client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
LegalHold: &types.ObjectLockLegalHold{
|
||||
Status: types.ObjectLockLegalHoldStatusOff,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify legal hold is off (must specify version ID)
|
||||
legalHoldResp, err = client.GetObjectLegalHold(context.TODO(), &s3.GetObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, types.ObjectLockLegalHoldStatusOff, legalHoldResp.LegalHold.Status)
|
||||
|
||||
// Now DELETE with version ID should succeed after legal hold removed
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
})
|
||||
require.NoError(t, err, "DELETE with version ID should succeed after legal hold removed")
|
||||
}
|
||||
|
||||
// TestObjectLockConfiguration tests bucket object lock configuration
|
||||
func TestObjectLockConfiguration(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
// Use a more unique bucket name to avoid conflicts
|
||||
bucketName := fmt.Sprintf("object-lock-config-%d-%d", time.Now().UnixNano(), time.Now().UnixMilli()%10000)
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Set object lock configuration
|
||||
_, err := client.PutObjectLockConfiguration(context.TODO(), &s3.PutObjectLockConfigurationInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
ObjectLockConfiguration: &types.ObjectLockConfiguration{
|
||||
ObjectLockEnabled: types.ObjectLockEnabledEnabled,
|
||||
Rule: &types.ObjectLockRule{
|
||||
DefaultRetention: &types.DefaultRetention{
|
||||
Mode: types.ObjectLockRetentionModeGovernance,
|
||||
Days: aws.Int32(30),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf("PutObjectLockConfiguration failed (may not be supported): %v", err)
|
||||
t.Skip("Object lock configuration not supported, skipping test")
|
||||
return
|
||||
}
|
||||
|
||||
// Get object lock configuration and verify
|
||||
configResp, err := client.GetObjectLockConfiguration(context.TODO(), &s3.GetObjectLockConfigurationInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, types.ObjectLockEnabledEnabled, configResp.ObjectLockConfiguration.ObjectLockEnabled)
|
||||
require.NotNil(t, configResp.ObjectLockConfiguration.Rule.DefaultRetention, "DefaultRetention should not be nil")
|
||||
require.NotNil(t, configResp.ObjectLockConfiguration.Rule.DefaultRetention.Days, "Days should not be nil")
|
||||
assert.Equal(t, types.ObjectLockRetentionModeGovernance, configResp.ObjectLockConfiguration.Rule.DefaultRetention.Mode)
|
||||
assert.Equal(t, int32(30), *configResp.ObjectLockConfiguration.Rule.DefaultRetention.Days)
|
||||
}
|
||||
|
||||
// TestRetentionWithVersions tests retention with specific object versions
|
||||
func TestRetentionWithVersions(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Create multiple versions of the same object
|
||||
key := "versioned-retention-test"
|
||||
content1 := "version 1 content"
|
||||
content2 := "version 2 content"
|
||||
|
||||
putResp1 := putObject(t, client, bucketName, key, content1)
|
||||
require.NotNil(t, putResp1.VersionId)
|
||||
|
||||
putResp2 := putObject(t, client, bucketName, key, content2)
|
||||
require.NotNil(t, putResp2.VersionId)
|
||||
|
||||
// Set retention on first version only
|
||||
retentionUntil := time.Now().Add(1 * time.Hour)
|
||||
_, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp1.VersionId,
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeGovernance,
|
||||
RetainUntilDate: aws.Time(retentionUntil),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get retention for first version
|
||||
retentionResp, err := client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp1.VersionId,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, types.ObjectLockRetentionModeGovernance, retentionResp.Retention.Mode)
|
||||
|
||||
// Try to get retention for second version - should fail (no retention set)
|
||||
_, err = client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp2.VersionId,
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
// Delete second version should succeed (no retention)
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp2.VersionId,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Delete first version should fail (has retention)
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp1.VersionId,
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
// Delete first version with bypass should succeed
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp1.VersionId,
|
||||
BypassGovernanceRetention: aws.Bool(true),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestRetentionAndLegalHoldCombination tests retention and legal hold together
|
||||
func TestRetentionAndLegalHoldCombination(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Create object
|
||||
key := "combined-protection-test"
|
||||
content := "combined protection test content"
|
||||
putResp := putObject(t, client, bucketName, key, content)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// Set both retention and legal hold
|
||||
retentionUntil := time.Now().Add(1 * time.Hour)
|
||||
|
||||
// Set retention
|
||||
_, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeGovernance,
|
||||
RetainUntilDate: aws.Time(retentionUntil),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Set legal hold
|
||||
_, err = client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
LegalHold: &types.ObjectLockLegalHold{
|
||||
Status: types.ObjectLockLegalHoldStatusOn,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
|
||||
|
||||
// Try DELETE with version ID and bypass - should still fail due to legal hold
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
BypassGovernanceRetention: aws.Bool(true),
|
||||
})
|
||||
require.Error(t, err, "Legal hold should prevent deletion even with governance bypass")
|
||||
|
||||
// Remove legal hold (must specify version ID since latest version is now delete marker)
|
||||
_, err = client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
LegalHold: &types.ObjectLockLegalHold{
|
||||
Status: types.ObjectLockLegalHoldStatusOff,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Now DELETE with version ID and bypass governance should succeed
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
BypassGovernanceRetention: aws.Bool(true),
|
||||
})
|
||||
require.NoError(t, err, "DELETE with version ID should succeed after legal hold removed and with governance bypass")
|
||||
}
|
||||
|
||||
// TestExpiredRetention tests that objects can be deleted after retention expires
|
||||
func TestExpiredRetention(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Create object
|
||||
key := "expired-retention-test"
|
||||
content := "expired retention test content"
|
||||
putResp := putObject(t, client, bucketName, key, content)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// Set retention for a very short time (2 seconds)
|
||||
retentionUntil := time.Now().Add(2 * time.Second)
|
||||
_, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeGovernance,
|
||||
RetainUntilDate: aws.Time(retentionUntil),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Try to delete immediately - should fail
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
// Wait for retention to expire
|
||||
time.Sleep(3 * time.Second)
|
||||
|
||||
// Now delete should succeed
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestRetentionErrorCases tests various error conditions
|
||||
func TestRetentionErrorCases(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Test setting retention on non-existent object
|
||||
_, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("non-existent-key"),
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeGovernance,
|
||||
RetainUntilDate: aws.Time(time.Now().Add(1 * time.Hour)),
|
||||
},
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
// Test getting retention on non-existent object
|
||||
_, err = client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("non-existent-key"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
// Test setting legal hold on non-existent object
|
||||
_, err = client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("non-existent-key"),
|
||||
LegalHold: &types.ObjectLockLegalHold{
|
||||
Status: types.ObjectLockLegalHoldStatusOn,
|
||||
},
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
// Test getting legal hold on non-existent object
|
||||
_, err = client.GetObjectLegalHold(context.TODO(), &s3.GetObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("non-existent-key"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
|
||||
// Test setting retention with past date
|
||||
key := "retention-past-date-test"
|
||||
content := "test content"
|
||||
putObject(t, client, bucketName, key, content)
|
||||
|
||||
pastDate := time.Now().Add(-1 * time.Hour)
|
||||
_, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeGovernance,
|
||||
RetainUntilDate: aws.Time(pastDate),
|
||||
},
|
||||
})
|
||||
require.Error(t, err)
|
||||
}
|
||||
536
test/s3/retention/s3_worm_integration_test.go
Normal file
536
test/s3/retention/s3_worm_integration_test.go
Normal file
@@ -0,0 +1,536 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestWORMRetentionIntegration tests that both retention and legacy WORM work together
|
||||
func TestWORMRetentionIntegration(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Create object
|
||||
key := "worm-retention-integration-test"
|
||||
content := "worm retention integration test content"
|
||||
putResp := putObject(t, client, bucketName, key, content)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// Set retention (new system)
|
||||
retentionUntil := time.Now().Add(1 * time.Hour)
|
||||
_, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeGovernance,
|
||||
RetainUntilDate: aws.Time(retentionUntil),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
|
||||
|
||||
// Try DELETE with version ID - should fail due to GOVERNANCE retention
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
})
|
||||
require.Error(t, err, "DELETE with version ID should be blocked by GOVERNANCE retention")
|
||||
|
||||
// Delete with version ID and bypass should succeed
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
BypassGovernanceRetention: aws.Bool(true),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestWORMLegacyCompatibility tests that legacy WORM functionality still works
|
||||
func TestWORMLegacyCompatibility(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Create object with legacy WORM headers (if supported)
|
||||
key := "legacy-worm-test"
|
||||
content := "legacy worm test content"
|
||||
|
||||
// Try to create object with legacy WORM TTL header
|
||||
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(content),
|
||||
// Add legacy WORM headers if supported
|
||||
Metadata: map[string]string{
|
||||
"x-amz-meta-worm-ttl": fmt.Sprintf("%d", time.Now().Add(1*time.Hour).Unix()),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// Object should be created successfully
|
||||
resp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, resp.Metadata)
|
||||
}
|
||||
|
||||
// TestRetentionOverwriteProtection tests that retention prevents overwrites
|
||||
func TestRetentionOverwriteProtection(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Create object
|
||||
key := "overwrite-protection-test"
|
||||
content := "original content"
|
||||
putResp := putObject(t, client, bucketName, key, content)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// Verify object exists before setting retention
|
||||
_, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Object should exist before setting retention")
|
||||
|
||||
// Set retention with specific version ID
|
||||
retentionUntil := time.Now().Add(1 * time.Hour)
|
||||
_, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeGovernance,
|
||||
RetainUntilDate: aws.Time(retentionUntil),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Try to overwrite object - should fail in non-versioned bucket context
|
||||
content2 := "new content"
|
||||
_, err = client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(content2),
|
||||
})
|
||||
// Note: In a real scenario, this might fail or create a new version
|
||||
// The actual behavior depends on the implementation
|
||||
if err != nil {
|
||||
t.Logf("Expected behavior: overwrite blocked due to retention: %v", err)
|
||||
} else {
|
||||
t.Logf("Overwrite allowed, likely created new version")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetentionBulkOperations tests retention with bulk operations
|
||||
func TestRetentionBulkOperations(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Create multiple objects with retention
|
||||
var objectsToDelete []types.ObjectIdentifier
|
||||
retentionUntil := time.Now().Add(1 * time.Hour)
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
key := fmt.Sprintf("bulk-test-object-%d", i)
|
||||
content := fmt.Sprintf("bulk test content %d", i)
|
||||
|
||||
putResp := putObject(t, client, bucketName, key, content)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// Set retention on each object with version ID
|
||||
_, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeGovernance,
|
||||
RetainUntilDate: aws.Time(retentionUntil),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
})
|
||||
}
|
||||
|
||||
// Try bulk delete without bypass - should fail or have errors
|
||||
deleteResp, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Delete: &types.Delete{
|
||||
Objects: objectsToDelete,
|
||||
Quiet: aws.Bool(false),
|
||||
},
|
||||
})
|
||||
|
||||
// Check if operation failed or returned errors for protected objects
|
||||
if err != nil {
|
||||
t.Logf("Expected: bulk delete failed due to retention: %v", err)
|
||||
} else if deleteResp != nil && len(deleteResp.Errors) > 0 {
|
||||
t.Logf("Expected: bulk delete returned %d errors due to retention", len(deleteResp.Errors))
|
||||
for _, delErr := range deleteResp.Errors {
|
||||
t.Logf("Delete error: %s - %s", *delErr.Code, *delErr.Message)
|
||||
}
|
||||
} else {
|
||||
t.Logf("Warning: bulk delete succeeded - retention may not be enforced for bulk operations")
|
||||
}
|
||||
|
||||
// Try bulk delete with bypass - should succeed
|
||||
_, err = client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
BypassGovernanceRetention: aws.Bool(true),
|
||||
Delete: &types.Delete{
|
||||
Objects: objectsToDelete,
|
||||
Quiet: aws.Bool(false),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf("Bulk delete with bypass failed (may not be supported): %v", err)
|
||||
} else {
|
||||
t.Logf("Bulk delete with bypass succeeded")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetentionWithMultipartUpload tests retention with multipart uploads
|
||||
func TestRetentionWithMultipartUpload(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Start multipart upload
|
||||
key := "multipart-retention-test"
|
||||
createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
uploadId := createResp.UploadId
|
||||
|
||||
// Upload a part
|
||||
partContent := "This is a test part for multipart upload"
|
||||
uploadResp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
PartNumber: aws.Int32(1),
|
||||
UploadId: uploadId,
|
||||
Body: strings.NewReader(partContent),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Complete multipart upload
|
||||
completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
UploadId: uploadId,
|
||||
MultipartUpload: &types.CompletedMultipartUpload{
|
||||
Parts: []types.CompletedPart{
|
||||
{
|
||||
ETag: uploadResp.ETag,
|
||||
PartNumber: aws.Int32(1),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add a small delay to ensure the object is fully created
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
// Verify object exists after multipart upload - retry if needed
|
||||
var headErr error
|
||||
for retries := 0; retries < 10; retries++ {
|
||||
_, headErr = client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if headErr == nil {
|
||||
break
|
||||
}
|
||||
t.Logf("HeadObject attempt %d failed: %v", retries+1, headErr)
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
|
||||
if headErr != nil {
|
||||
t.Logf("Object not found after multipart upload completion, checking if multipart upload is fully supported")
|
||||
// Check if the object exists by trying to list it
|
||||
listResp, listErr := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(key),
|
||||
})
|
||||
if listErr != nil || len(listResp.Contents) == 0 {
|
||||
t.Skip("Multipart upload may not be fully supported, skipping test")
|
||||
return
|
||||
}
|
||||
// If object exists in listing but not accessible via HeadObject, skip test
|
||||
t.Skip("Object exists in listing but not accessible via HeadObject, multipart upload may not be fully supported")
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(t, headErr, "Object should exist after multipart upload")
|
||||
|
||||
// Set retention on the completed multipart object with version ID
|
||||
retentionUntil := time.Now().Add(1 * time.Hour)
|
||||
_, err = client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: completeResp.VersionId,
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeGovernance,
|
||||
RetainUntilDate: aws.Time(retentionUntil),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
|
||||
|
||||
// Try DELETE with version ID - should fail due to GOVERNANCE retention
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: completeResp.VersionId,
|
||||
})
|
||||
require.Error(t, err, "DELETE with version ID should be blocked by GOVERNANCE retention")
|
||||
}
|
||||
|
||||
// TestRetentionExtendedAttributes tests that retention uses extended attributes correctly
|
||||
func TestRetentionExtendedAttributes(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Create object
|
||||
key := "extended-attrs-test"
|
||||
content := "extended attributes test content"
|
||||
putResp := putObject(t, client, bucketName, key, content)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// Set retention
|
||||
retentionUntil := time.Now().Add(1 * time.Hour)
|
||||
_, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeGovernance,
|
||||
RetainUntilDate: aws.Time(retentionUntil),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Set legal hold
|
||||
_, err = client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: putResp.VersionId,
|
||||
LegalHold: &types.ObjectLockLegalHold{
|
||||
Status: types.ObjectLockLegalHoldStatusOn,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get object metadata to verify extended attributes are set
|
||||
resp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Check that the object has metadata (may be empty in some implementations)
|
||||
// Note: The actual metadata keys depend on the implementation
|
||||
if resp.Metadata != nil && len(resp.Metadata) > 0 {
|
||||
t.Logf("Object metadata: %+v", resp.Metadata)
|
||||
} else {
|
||||
t.Logf("Object metadata: empty (extended attributes may be stored internally)")
|
||||
}
|
||||
|
||||
// Verify retention can be retrieved
|
||||
retentionResp, err := client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, types.ObjectLockRetentionModeGovernance, retentionResp.Retention.Mode)
|
||||
|
||||
// Verify legal hold can be retrieved
|
||||
legalHoldResp, err := client.GetObjectLegalHold(context.TODO(), &s3.GetObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, legalHoldResp.LegalHold.Status)
|
||||
}
|
||||
|
||||
// TestRetentionBucketDefaults tests object lock configuration defaults
|
||||
func TestRetentionBucketDefaults(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
// Use a very unique bucket name to avoid conflicts
|
||||
bucketName := fmt.Sprintf("bucket-defaults-%d-%d", time.Now().UnixNano(), time.Now().UnixMilli()%10000)
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Set bucket object lock configuration with default retention
|
||||
_, err := client.PutObjectLockConfiguration(context.TODO(), &s3.PutObjectLockConfigurationInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
ObjectLockConfiguration: &types.ObjectLockConfiguration{
|
||||
ObjectLockEnabled: types.ObjectLockEnabledEnabled,
|
||||
Rule: &types.ObjectLockRule{
|
||||
DefaultRetention: &types.DefaultRetention{
|
||||
Mode: types.ObjectLockRetentionModeGovernance,
|
||||
Days: aws.Int32(1), // 1 day default
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf("PutObjectLockConfiguration failed (may not be supported): %v", err)
|
||||
t.Skip("Object lock configuration not supported, skipping test")
|
||||
return
|
||||
}
|
||||
|
||||
// Create object (should inherit default retention)
|
||||
key := "bucket-defaults-test"
|
||||
content := "bucket defaults test content"
|
||||
putResp := putObject(t, client, bucketName, key, content)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// Check if object has default retention applied
|
||||
// Note: This depends on the implementation - some S3 services apply
|
||||
// default retention automatically, others require explicit setting
|
||||
retentionResp, err := client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf("No automatic default retention applied: %v", err)
|
||||
} else {
|
||||
t.Logf("Default retention applied: %+v", retentionResp.Retention)
|
||||
assert.Equal(t, types.ObjectLockRetentionModeGovernance, retentionResp.Retention.Mode)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetentionConcurrentOperations tests concurrent retention operations
|
||||
func TestRetentionConcurrentOperations(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket and enable versioning
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Create object
|
||||
key := "concurrent-ops-test"
|
||||
content := "concurrent operations test content"
|
||||
putResp := putObject(t, client, bucketName, key, content)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
|
||||
// Test concurrent retention and legal hold operations
|
||||
retentionUntil := time.Now().Add(1 * time.Hour)
|
||||
|
||||
// Set retention and legal hold concurrently
|
||||
errChan := make(chan error, 2)
|
||||
|
||||
go func() {
|
||||
_, err := client.PutObjectRetention(context.TODO(), &s3.PutObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Retention: &types.ObjectLockRetention{
|
||||
Mode: types.ObjectLockRetentionModeGovernance,
|
||||
RetainUntilDate: aws.Time(retentionUntil),
|
||||
},
|
||||
})
|
||||
errChan <- err
|
||||
}()
|
||||
|
||||
go func() {
|
||||
_, err := client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
LegalHold: &types.ObjectLockLegalHold{
|
||||
Status: types.ObjectLockLegalHoldStatusOn,
|
||||
},
|
||||
})
|
||||
errChan <- err
|
||||
}()
|
||||
|
||||
// Wait for both operations to complete
|
||||
for i := 0; i < 2; i++ {
|
||||
err := <-errChan
|
||||
if err != nil {
|
||||
t.Logf("Concurrent operation failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify both settings are applied
|
||||
retentionResp, err := client.GetObjectRetention(context.TODO(), &s3.GetObjectRetentionInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err == nil {
|
||||
assert.Equal(t, types.ObjectLockRetentionModeGovernance, retentionResp.Retention.Mode)
|
||||
}
|
||||
|
||||
legalHoldResp, err := client.GetObjectLegalHold(context.TODO(), &s3.GetObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if err == nil {
|
||||
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, legalHoldResp.LegalHold.Status)
|
||||
}
|
||||
}
|
||||
9
test/s3/retention/test_config.json
Normal file
9
test/s3/retention/test_config.json
Normal file
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"endpoint": "http://localhost:8333",
|
||||
"access_key": "some_access_key1",
|
||||
"secret_key": "some_secret_key1",
|
||||
"region": "us-east-1",
|
||||
"bucket_prefix": "test-retention-",
|
||||
"use_ssl": false,
|
||||
"skip_verify_ssl": true
|
||||
}
|
||||
Reference in New Issue
Block a user