snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/kms/Makefile

@@ -0,0 +1,280 @@
+# SeaweedFS KMS Integration Testing Makefile
+
+# Configuration
+OPENBAO_ADDR ?= http://127.0.0.1:8200
+OPENBAO_TOKEN ?= root-token-for-testing
+SEAWEEDFS_S3_ENDPOINT ?= http://127.0.0.1:8333
+TEST_TIMEOUT ?= 5m
+DOCKER_COMPOSE ?= docker-compose
+
+# CI configuration (build weed from source, run natively)
+SEAWEEDFS_BINARY ?= weed
+S3_PORT ?= 8333
+ACCESS_KEY ?= some_access_key1
+SECRET_KEY ?= some_secret_key1
+TEST_DIR := $(shell pwd)
+SEAWEEDFS_ROOT := $(shell cd ../.. && pwd)
+
+# Colors for output
+BLUE := \033[36m
+GREEN := \033[32m
+YELLOW := \033[33m
+RED := \033[31m
+NC := \033[0m # No Color
+
+.PHONY: help setup test test-unit test-integration test-e2e clean logs status \
+       build-weed check-binary start-openbao-ci stop-openbao-ci \
+       start-seaweedfs-ci stop-seaweedfs-ci clean-ci \
+       test-provider-ci test-s3-kms-ci
+
+help: ## Show this help message
+	@echo "$(BLUE)SeaweedFS KMS Integration Testing$(NC)"
+	@echo ""
+	@echo "Available targets:"
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  $(GREEN)%-15s$(NC) %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+setup: ## Set up test environment (OpenBao + SeaweedFS)
+	@echo "$(YELLOW)Setting up test environment...$(NC)"
+	@chmod +x setup_openbao.sh
+	@$(DOCKER_COMPOSE) up -d openbao
+	@sleep 5
+	@echo "$(BLUE)Configuring OpenBao...$(NC)"
+	@OPENBAO_ADDR=$(OPENBAO_ADDR) OPENBAO_TOKEN=$(OPENBAO_TOKEN) ./setup_openbao.sh
+	@echo "$(GREEN)✅ Test environment ready!$(NC)"
+
+test: setup test-unit test-integration ## Run all tests
+
+test-unit: ## Run unit tests for KMS providers
+	@echo "$(YELLOW)Running KMS provider unit tests...$(NC)"
+	@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) ./weed/kms/...
+
+test-integration: ## Run integration tests with OpenBao
+	@echo "$(YELLOW)Running KMS integration tests...$(NC)"
+	@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) ./test/kms/...
+
+test-benchmark: ## Run performance benchmarks
+	@echo "$(YELLOW)Running KMS performance benchmarks...$(NC)"
+	@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -bench=. ./test/kms/...
+
+test-e2e: setup-seaweedfs ## Run end-to-end tests with SeaweedFS + KMS
+	@echo "$(YELLOW)Running end-to-end KMS tests...$(NC)"
+	@sleep 10  # Wait for SeaweedFS to be ready
+	@./test_s3_kms.sh
+
+setup-seaweedfs: ## Start complete SeaweedFS cluster with KMS
+	@echo "$(YELLOW)Starting SeaweedFS cluster...$(NC)"
+	@$(DOCKER_COMPOSE) up -d
+	@echo "$(BLUE)Waiting for services to be ready...$(NC)"
+	@./wait_for_services.sh
+
+test-aws-compat: ## Test AWS KMS API compatibility
+	@echo "$(YELLOW)Testing AWS KMS compatibility...$(NC)"
+	@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -run TestAWSKMSCompat ./test/kms/...
+
+clean: ## Clean up test environment
+	@echo "$(YELLOW)Cleaning up test environment...$(NC)"
+	@$(DOCKER_COMPOSE) down -v --remove-orphans
+	@docker system prune -f
+	@echo "$(GREEN)✅ Environment cleaned up!$(NC)"
+
+logs: ## Show logs from all services
+	@$(DOCKER_COMPOSE) logs --tail=50 -f
+
+logs-openbao: ## Show OpenBao logs
+	@$(DOCKER_COMPOSE) logs --tail=100 -f openbao
+
+logs-seaweedfs: ## Show SeaweedFS logs
+	@$(DOCKER_COMPOSE) logs --tail=100 -f seaweedfs-filer seaweedfs-master seaweedfs-volume
+
+status: ## Show status of all services
+	@echo "$(BLUE)Service Status:$(NC)"
+	@$(DOCKER_COMPOSE) ps
+	@echo ""
+	@echo "$(BLUE)OpenBao Status:$(NC)"
+	@curl -s $(OPENBAO_ADDR)/v1/sys/health | jq '.' || echo "OpenBao not accessible"
+	@echo ""
+	@echo "$(BLUE)SeaweedFS S3 Status:$(NC)"
+	@curl -s $(SEAWEEDFS_S3_ENDPOINT) || echo "SeaweedFS S3 not accessible"
+
+debug: ## Debug test environment
+	@echo "$(BLUE)Debug Information:$(NC)"
+	@echo "OpenBao Address: $(OPENBAO_ADDR)"
+	@echo "SeaweedFS S3 Endpoint: $(SEAWEEDFS_S3_ENDPOINT)"
+	@echo "Docker Compose Status:"
+	@$(DOCKER_COMPOSE) ps
+	@echo ""
+	@echo "Network connectivity:"
+	@docker network ls | grep seaweedfs || echo "No SeaweedFS network found"
+	@echo ""
+	@echo "OpenBao health:"
+	@curl -v $(OPENBAO_ADDR)/v1/sys/health 2>&1 || true
+
+# Development targets
+dev-openbao: ## Start only OpenBao for development
+	@$(DOCKER_COMPOSE) up -d openbao
+	@sleep 5
+	@OPENBAO_ADDR=$(OPENBAO_ADDR) OPENBAO_TOKEN=$(OPENBAO_TOKEN) ./setup_openbao.sh
+
+dev-test: dev-openbao ## Quick test with just OpenBao
+	@cd ../../ && go test -v -timeout=30s -run TestOpenBaoKMSProvider_Integration ./test/kms/
+
+# Utility targets
+install-deps: ## Install required dependencies
+	@echo "$(YELLOW)Installing test dependencies...$(NC)"
+	@which docker > /dev/null || (echo "$(RED)Docker not found$(NC)" && exit 1)
+	@which docker-compose > /dev/null || (echo "$(RED)Docker Compose not found$(NC)" && exit 1)
+	@which jq > /dev/null || (echo "$(RED)jq not found - please install jq$(NC)" && exit 1)
+	@which curl > /dev/null || (echo "$(RED)curl not found$(NC)" && exit 1)
+	@echo "$(GREEN)✅ All dependencies available$(NC)"
+
+check-env: ## Check test environment setup
+	@echo "$(BLUE)Environment Check:$(NC)"
+	@echo "OPENBAO_ADDR: $(OPENBAO_ADDR)"
+	@echo "OPENBAO_TOKEN: $(OPENBAO_TOKEN)" 
+	@echo "SEAWEEDFS_S3_ENDPOINT: $(SEAWEEDFS_S3_ENDPOINT)"
+	@echo "TEST_TIMEOUT: $(TEST_TIMEOUT)"
+	@make install-deps
+
+# CI targets (docker-compose based)
+ci-test: ## Run tests in CI environment (docker-compose)
+	@echo "$(YELLOW)Running CI tests...$(NC)"
+	@make setup
+	@make test-unit
+	@make test-integration
+	@make clean
+
+ci-e2e: ## Run end-to-end tests in CI (docker-compose)
+	@echo "$(YELLOW)Running CI end-to-end tests...$(NC)"
+	@make setup-seaweedfs
+	@make test-e2e
+	@make clean
+
+####################################################
+# GitHub Actions CI targets (build weed from source)
+####################################################
+
+build-weed: ## Build SeaweedFS binary from source
+	@echo "Building SeaweedFS binary..."
+	@cd $(SEAWEEDFS_ROOT)/weed && go install -buildvcs=false
+	@echo "$(GREEN)SeaweedFS binary built successfully$(NC)"
+
+check-binary:
+	@if ! command -v $(SEAWEEDFS_BINARY) > /dev/null 2>&1; then \
+		echo "$(RED)Error: SeaweedFS binary '$(SEAWEEDFS_BINARY)' not found in PATH$(NC)"; \
+		exit 1; \
+	fi
+
+start-openbao-ci: ## Start OpenBao via Docker for CI
+	@echo "Starting OpenBao for CI..."
+	@docker rm -f openbao-ci 2>/dev/null || true
+	@docker run -d --name openbao-ci \
+		-p 8200:8200 \
+		-e BAO_DEV_ROOT_TOKEN_ID=$(OPENBAO_TOKEN) \
+		-e BAO_DEV_LISTEN_ADDRESS=0.0.0.0:8200 \
+		ghcr.io/openbao/openbao:latest \
+		bao server -dev -dev-root-token-id=$(OPENBAO_TOKEN) -dev-listen-address=0.0.0.0:8200
+	@echo "Waiting for OpenBao to be ready..."
+	@for i in $$(seq 1 30); do \
+		if curl -s $(OPENBAO_ADDR)/v1/sys/health > /dev/null 2>&1; then \
+			echo "$(GREEN)OpenBao is ready$(NC)"; \
+			break; \
+		fi; \
+		if [ $$i -eq 30 ]; then \
+			echo "$(RED)Timeout waiting for OpenBao$(NC)"; \
+			docker logs openbao-ci 2>&1 | tail -20; \
+			exit 1; \
+		fi; \
+		sleep 2; \
+	done
+	@chmod +x setup_openbao.sh
+	@OPENBAO_ADDR=$(OPENBAO_ADDR) OPENBAO_TOKEN=$(OPENBAO_TOKEN) ./setup_openbao.sh
+
+stop-openbao-ci: ## Stop OpenBao CI container
+	@echo "Stopping OpenBao..."
+	@docker stop openbao-ci 2>/dev/null || true
+	@docker rm openbao-ci 2>/dev/null || true
+
+start-seaweedfs-ci: check-binary ## Start weed mini with OpenBao KMS config
+	@echo "Starting SeaweedFS with OpenBao KMS..."
+	@mkdir -p /tmp/seaweedfs-test-kms
+	@rm -f /tmp/seaweedfs-kms-*.log || true
+	@sed -e 's/ACCESS_KEY_PLACEHOLDER/$(ACCESS_KEY)/g' \
+	     -e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \
+	     -e 's|OPENBAO_ADDR_PLACEHOLDER|$(OPENBAO_ADDR)|g' \
+	     -e 's/OPENBAO_TOKEN_PLACEHOLDER/$(OPENBAO_TOKEN)/g' \
+	     s3-config-openbao-template.json > /tmp/seaweedfs-kms-s3.json
+	@# Start weed mini from the data dir to avoid picking up test/kms/filer.toml
+	@# (filer.toml is read from "." first, and the one here has Docker-only paths)
+	@cd /tmp/seaweedfs-test-kms && \
+		AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
+		-dir=/tmp/seaweedfs-test-kms \
+		-s3.port=$(S3_PORT) \
+		-s3.config=/tmp/seaweedfs-kms-s3.json \
+		-ip=127.0.0.1 \
+		> /tmp/seaweedfs-kms-mini.log 2>&1 & echo $$! > /tmp/weed-kms-mini.pid
+	@echo "Checking S3 service is ready..."
+	@for i in $$(seq 1 30); do \
+		if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \
+			echo "$(GREEN)S3 service is ready$(NC)"; \
+			break; \
+		fi; \
+		if [ $$i -eq 30 ]; then \
+			echo "$(RED)Timeout waiting for S3 service$(NC)"; \
+			echo "=== Last 50 lines of server log ==="; \
+			tail -50 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"; \
+			exit 1; \
+		fi; \
+		sleep 1; \
+	done
+
+stop-seaweedfs-ci: ## Stop weed mini (CI-safe)
+	@echo "Stopping SeaweedFS..."
+	@if [ -f /tmp/weed-kms-mini.pid ]; then \
+		kill $$(cat /tmp/weed-kms-mini.pid) 2>/dev/null || true; \
+		rm -f /tmp/weed-kms-mini.pid; \
+	fi
+	@if command -v lsof >/dev/null 2>&1; then \
+		lsof -ti :$(S3_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
+	fi
+	@sleep 2
+
+clean-ci: ## Clean up CI test artifacts
+	@rm -rf /tmp/seaweedfs-test-kms
+	@rm -f /tmp/seaweedfs-kms-*.log /tmp/seaweedfs-kms-s3.json /tmp/weed-kms-mini.pid
+
+# Run KMS provider Go integration tests (GitHub Actions compatible)
+test-provider-ci: build-weed ## Run KMS provider tests with OpenBao in CI
+	@echo "$(YELLOW)Starting KMS provider integration tests...$(NC)"
+	@if $(MAKE) start-openbao-ci > /tmp/openbao-ci-setup.log 2>&1; then \
+		echo "$(GREEN)OpenBao started and configured$(NC)"; \
+		trap '$(MAKE) -C $(TEST_DIR) stop-openbao-ci || true' EXIT; \
+		cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/kms/... || exit 1; \
+		echo "$(GREEN)KMS provider tests completed successfully$(NC)"; \
+	else \
+		echo "$(RED)Failed to start OpenBao$(NC)"; \
+		cat /tmp/openbao-ci-setup.log 2>/dev/null || true; \
+		exit 1; \
+	fi
+
+# Run S3 KMS end-to-end tests (GitHub Actions compatible)
+test-s3-kms-ci: build-weed ## Run S3 KMS e2e tests with weed mini + OpenBao in CI
+	@echo "$(YELLOW)Starting S3 KMS end-to-end tests...$(NC)"
+	@if $(MAKE) start-openbao-ci > /tmp/openbao-ci-setup.log 2>&1; then \
+		echo "$(GREEN)OpenBao started and configured$(NC)"; \
+		trap 'docker logs openbao-ci > /tmp/openbao-ci-container.log 2>&1 || true; $(MAKE) -C $(TEST_DIR) stop-seaweedfs-ci || true; $(MAKE) -C $(TEST_DIR) stop-openbao-ci || true; $(MAKE) -C $(TEST_DIR) clean-ci || true' EXIT; \
+		if $(MAKE) start-seaweedfs-ci > /tmp/weed-kms-ci-setup.log 2>&1; then \
+			echo "$(GREEN)SeaweedFS started with OpenBao KMS$(NC)"; \
+			sleep 3; \
+			chmod +x test_s3_kms.sh; \
+			SEAWEEDFS_S3_ENDPOINT=http://127.0.0.1:$(S3_PORT) ACCESS_KEY=$(ACCESS_KEY) SECRET_KEY=$(SECRET_KEY) ./test_s3_kms.sh || exit 1; \
+			echo "$(GREEN)S3 KMS e2e tests completed successfully$(NC)"; \
+		else \
+			echo "$(RED)Failed to start SeaweedFS$(NC)"; \
+			cat /tmp/weed-kms-ci-setup.log 2>/dev/null || true; \
+			exit 1; \
+		fi; \
+	else \
+		echo "$(RED)Failed to start OpenBao$(NC)"; \
+		cat /tmp/openbao-ci-setup.log 2>/dev/null || true; \
+		exit 1; \
+	fi

test/kms/README.md

@@ -0,0 +1,394 @@
+# 🔐 SeaweedFS KMS Integration Tests
+
+This directory contains comprehensive integration tests for SeaweedFS Server-Side Encryption (SSE) with Key Management Service (KMS) providers. The tests validate the complete encryption/decryption workflow using **OpenBao** (open source fork of HashiCorp Vault) as the KMS provider.
+
+## 🎯 Overview
+
+The KMS integration tests simulate **AWS KMS** functionality using **OpenBao**, providing:
+
+- ✅ **Production-grade KMS testing** with real encryption/decryption operations
+- ✅ **S3 API compatibility testing** with SSE-KMS headers and bucket encryption
+- ✅ **Per-bucket KMS configuration** validation
+- ✅ **Performance benchmarks** for KMS operations
+- ✅ **Error handling and edge case** coverage
+- ✅ **End-to-end workflows** from S3 API to KMS provider
+
+## 🏗️ Architecture
+
+```
+┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
+│   S3 Client     │    │   SeaweedFS     │    │    OpenBao      │
+│   (aws s3)      │───▶│   S3 API        │───▶│   Transit       │
+└─────────────────┘    └─────────────────┘    └─────────────────┘
+        │                       │                       │
+        │              ┌─────────────────┐              │
+        │              │ KMS Manager     │              │
+        └──────────────▶│ - AWS Provider  │◀─────────────┘
+                       │ - Azure Provider│
+                       │ - GCP Provider  │
+                       │ - OpenBao       │
+                       └─────────────────┘
+```
+
+## 📋 Prerequisites
+
+### Required Tools
+
+- **Docker & Docker Compose** - For running OpenBao and SeaweedFS
+- **OpenBao CLI** (`bao`) - For direct OpenBao interaction *(optional)*
+- **AWS CLI** - For S3 API testing
+- **jq** - For JSON processing in scripts
+- **curl** - For HTTP API testing
+- **Go 1.19+** - For running Go tests
+
+### Installation
+
+```bash
+# Install Docker (macOS)
+brew install docker docker-compose
+
+# Install OpenBao (optional - used by some tests)
+brew install openbao
+
+# Install AWS CLI  
+brew install awscli
+
+# Install jq
+brew install jq
+```
+
+## 🚀 Quick Start
+
+### 1. Run All Tests
+
+```bash
+cd test/kms
+make test
+```
+
+### 2. Run Specific Test Types
+
+```bash
+# Unit tests only
+make test-unit
+
+# Integration tests with OpenBao
+make test-integration  
+
+# End-to-end S3 API tests
+make test-e2e
+
+# Performance benchmarks
+make test-benchmark
+```
+
+### 3. Manual Setup
+
+```bash
+# Start OpenBao only
+make dev-openbao
+
+# Start full environment (OpenBao + SeaweedFS)
+make setup-seaweedfs
+
+# Run manual tests
+make dev-test
+```
+
+## 🧪 Test Components
+
+### 1. **OpenBao KMS Provider** (`openbao_integration_test.go`)
+
+**What it tests:**
+- KMS provider registration and initialization  
+- Data key generation using Transit engine
+- Encryption/decryption of data keys
+- Key metadata and validation
+- Error handling (invalid tokens, missing keys, etc.)
+- Multiple key scenarios
+- Performance benchmarks
+
+**Key test cases:**
+```go
+TestOpenBaoKMSProvider_Integration
+TestOpenBaoKMSProvider_ErrorHandling  
+TestKMSManager_WithOpenBao
+BenchmarkOpenBaoKMS_GenerateDataKey
+BenchmarkOpenBaoKMS_Decrypt
+```
+
+### 2. **S3 API Integration** (`test_s3_kms.sh`)
+
+**What it tests:**
+- Bucket encryption configuration via S3 API
+- Default bucket encryption behavior
+- Explicit SSE-KMS headers in PUT operations
+- Object upload/download with encryption
+- Multipart uploads with KMS encryption
+- Encryption metadata in object headers
+- Cross-bucket KMS provider isolation
+
+**Key scenarios:**
+```bash
+# Bucket encryption setup
+aws s3api put-bucket-encryption --bucket test-openbao \
+  --server-side-encryption-configuration '{
+    "Rules": [{
+      "ApplyServerSideEncryptionByDefault": {
+        "SSEAlgorithm": "aws:kms", 
+        "KMSMasterKeyID": "test-key-1"
+      }
+    }]
+  }'
+
+# Object upload with encryption
+aws s3 cp file.txt s3://test-openbao/encrypted-file.txt \
+  --sse aws:kms --sse-kms-key-id "test-key-2"
+```
+
+### 3. **Docker Environment** (`docker-compose.yml`)
+
+**Services:**
+- **OpenBao** - KMS provider (port 8200)
+- **Vault** - Alternative KMS (port 8201)  
+- **SeaweedFS Master** - Cluster coordination (port 9333)
+- **SeaweedFS Volume** - Data storage (port 8080)
+- **SeaweedFS Filer** - S3 API endpoint (port 8333)
+
+### 4. **Configuration** (`filer.toml`)
+
+**KMS Configuration:**
+```toml
+[kms]
+default_provider = "openbao-test"
+
+[kms.providers.openbao-test]
+type = "openbao"
+address = "http://openbao:8200"
+token = "root-token-for-testing"
+transit_path = "transit"
+
+[kms.buckets.test-openbao]
+provider = "openbao-test"
+```
+
+## 📊 Test Data
+
+### Encryption Keys Created
+
+The setup script creates these test keys in OpenBao:
+
+| Key Name | Type | Purpose |
+|----------|------|---------|
+| `test-key-1` | AES256-GCM96 | Basic operations |  
+| `test-key-2` | AES256-GCM96 | Multi-key scenarios |
+| `seaweedfs-test-key` | AES256-GCM96 | Integration testing |
+| `bucket-default-key` | AES256-GCM96 | Default bucket encryption |
+| `high-security-key` | AES256-GCM96 | Security testing |
+| `performance-key` | AES256-GCM96 | Performance benchmarks |
+| `multipart-key` | AES256-GCM96 | Multipart upload testing |
+
+### Test Buckets
+
+| Bucket Name | KMS Provider | Purpose |
+|-------------|--------------|---------|
+| `test-openbao` | openbao-test | OpenBao integration |
+| `test-vault` | vault-test | Vault compatibility |
+| `test-local` | local-test | Local KMS testing |  
+| `secure-data` | openbao-test | High security scenarios |
+
+## 🔧 Configuration Options
+
+### Environment Variables
+
+```bash
+# OpenBao configuration
+export OPENBAO_ADDR="http://127.0.0.1:8200"
+export OPENBAO_TOKEN="root-token-for-testing"
+
+# SeaweedFS configuration  
+export SEAWEEDFS_S3_ENDPOINT="http://127.0.0.1:8333"
+export ACCESS_KEY="any"
+export SECRET_KEY="any"
+
+# Test configuration
+export TEST_TIMEOUT="5m"
+```
+
+### Makefile Targets
+
+| Target | Description |
+|--------|-------------|
+| `make help` | Show available commands |
+| `make setup` | Set up test environment |
+| `make test` | Run all tests |
+| `make test-unit` | Run unit tests only |
+| `make test-integration` | Run integration tests |
+| `make test-e2e` | Run end-to-end tests |
+| `make clean` | Clean up environment |
+| `make logs` | Show service logs |
+| `make status` | Check service status |
+
+## 🧩 How It Works
+
+### 1. **KMS Provider Registration**
+
+OpenBao provider is automatically registered via `init()`:
+
+```go
+func init() {
+    seaweedkms.RegisterProvider("openbao", NewOpenBaoKMSProvider)
+    seaweedkms.RegisterProvider("vault", NewOpenBaoKMSProvider) // Alias
+}
+```
+
+### 2. **Data Key Generation Flow**
+
+```
+1. S3 PUT with SSE-KMS headers
+2. SeaweedFS extracts KMS key ID  
+3. KMSManager routes to OpenBao provider
+4. OpenBao generates random data key
+5. OpenBao encrypts data key with master key
+6. SeaweedFS encrypts object with data key
+7. Encrypted data key stored in metadata
+```
+
+### 3. **Decryption Flow**
+
+```
+1. S3 GET request for encrypted object
+2. SeaweedFS extracts encrypted data key from metadata
+3. KMSManager routes to OpenBao provider
+4. OpenBao decrypts data key with master key
+5. SeaweedFS decrypts object with data key
+6. Plaintext object returned to client
+```
+
+## 🔍 Troubleshooting
+
+### Common Issues
+
+**OpenBao not starting:**
+```bash
+# Check if port 8200 is in use
+lsof -i :8200
+
+# Check Docker logs
+docker-compose logs openbao
+```
+
+**KMS provider not found:**
+```bash
+# Verify provider registration
+go test -v -run TestProviderRegistration ./test/kms/
+
+# Check imports in filer_kms.go
+grep -n "kms/" weed/command/filer_kms.go
+```
+
+**S3 API connection refused:**
+```bash  
+# Check SeaweedFS services
+make status
+
+# Wait for services to be ready
+./wait_for_services.sh
+```
+
+### Debug Commands
+
+```bash
+# Test OpenBao directly
+curl -H "X-Vault-Token: root-token-for-testing" \
+  http://127.0.0.1:8200/v1/sys/health
+
+# Test transit engine
+curl -X POST \
+  -H "X-Vault-Token: root-token-for-testing" \
+  -d '{"plaintext":"SGVsbG8gV29ybGQ="}' \
+  http://127.0.0.1:8200/v1/transit/encrypt/test-key-1
+
+# Test S3 API
+aws s3 ls --endpoint-url http://127.0.0.1:8333
+```
+
+## 🎯 AWS KMS Integration Testing
+
+This test suite **simulates AWS KMS behavior** using OpenBao, enabling:
+
+### ✅ **Compatibility Validation**
+
+- **S3 API compatibility** - Same headers, same behavior as AWS S3
+- **KMS API patterns** - GenerateDataKey, Decrypt, DescribeKey operations  
+- **Error codes** - AWS-compatible error responses
+- **Encryption context** - Proper context handling and validation
+
+### ✅ **Production Readiness Testing**
+
+- **Key rotation scenarios** - Multiple keys per bucket
+- **Performance characteristics** - Latency and throughput metrics
+- **Error recovery** - Network failures, invalid keys, timeout handling
+- **Security validation** - Encryption/decryption correctness
+
+### ✅ **Integration Patterns**
+
+- **Bucket-level configuration** - Different KMS keys per bucket
+- **Cross-region simulation** - Multiple KMS providers
+- **Caching behavior** - Data key caching validation  
+- **Metadata handling** - Encrypted metadata storage
+
+## 📈 Performance Expectations
+
+**Typical performance metrics** (local testing):
+
+- **Data key generation**: ~50-100ms (including network roundtrip)
+- **Data key decryption**: ~30-50ms (cached provider instance)
+- **Object encryption**: ~1-5ms per MB (AES-256-GCM)
+- **S3 PUT with SSE-KMS**: +100-200ms overhead vs. unencrypted
+
+## 🚀 Production Deployment
+
+After successful integration testing, deploy with real KMS providers:
+
+```toml
+[kms.providers.aws-prod]
+type = "aws"
+region = "us-east-1"
+# IAM roles preferred over access keys
+
+[kms.providers.azure-prod]  
+type = "azure"
+vault_url = "https://prod-vault.vault.azure.net/"
+use_default_creds = true # Managed identity
+
+[kms.providers.gcp-prod]
+type = "gcp" 
+project_id = "prod-project"
+use_default_credentials = true # Service account
+```
+
+## 🎉 Success Criteria
+
+Tests pass when:
+
+- ✅ All KMS providers register successfully
+- ✅ Data key generation/decryption works end-to-end  
+- ✅ S3 API encryption headers are handled correctly
+- ✅ Bucket-level KMS configuration is respected
+- ✅ Multipart uploads maintain encryption consistency
+- ✅ Performance meets acceptable thresholds
+- ✅ Error scenarios are handled gracefully
+
+---
+
+## 📞 Support
+
+For issues with KMS integration tests:
+
+1. **Check logs**: `make logs`
+2. **Verify environment**: `make status` 
+3. **Run debug**: `make debug`
+4. **Clean restart**: `make clean && make setup`
+
+**Happy testing!** 🔐✨

test/kms/docker-compose.yml

@@ -0,0 +1,101 @@
+services:
+  # OpenBao server for KMS integration testing
+  openbao:
+    image: ghcr.io/openbao/openbao:latest
+    ports:
+      - "8200:8200"
+    environment:
+      - BAO_DEV_ROOT_TOKEN_ID=root-token-for-testing
+      - BAO_DEV_LISTEN_ADDRESS=0.0.0.0:8200
+      - BAO_LOCAL_CONFIG={"backend":{"file":{"path":"/bao/data"}},"default_lease_ttl":"168h","max_lease_ttl":"720h","ui":true,"disable_mlock":true}
+    command:
+      - bao
+      - server
+      - -dev
+      - -dev-root-token-id=root-token-for-testing
+      - -dev-listen-address=0.0.0.0:8200
+    volumes:
+      - openbao-data:/bao/data
+    healthcheck:
+      test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8200/v1/sys/health"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+      start_period: 10s
+
+  # HashiCorp Vault for compatibility testing (alternative to OpenBao)
+  vault:
+    image: vault:latest
+    ports:
+      - "8201:8200"
+    environment:
+      - VAULT_DEV_ROOT_TOKEN_ID=root-token-for-testing
+      - VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200
+    command:
+      - vault
+      - server
+      - -dev
+      - -dev-root-token-id=root-token-for-testing
+      - -dev-listen-address=0.0.0.0:8200
+    cap_add:
+      - IPC_LOCK
+    healthcheck:
+      test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8200/v1/sys/health"]
+      interval: 5s
+      timeout: 3s
+      retries: 5
+      start_period: 10s
+
+  # SeaweedFS components for end-to-end testing
+  seaweedfs-master:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "9333:9333"
+    command: 
+      - master
+      - -ip=seaweedfs-master
+      - -volumeSizeLimitMB=1024
+    volumes:
+      - seaweedfs-master-data:/data
+
+  seaweedfs-volume:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "8080:8080"
+    command:
+      - volume
+      - -master=seaweedfs-master:9333
+      - -ip=seaweedfs-volume
+      - -publicUrl=seaweedfs-volume:8080
+    depends_on:
+      - seaweedfs-master
+    volumes:
+      - seaweedfs-volume-data:/data
+
+  seaweedfs-filer:
+    image: chrislusf/seaweedfs:latest
+    ports:
+      - "8888:8888"
+      - "8333:8333" # S3 API port
+    command:
+      - filer
+      - -master=seaweedfs-master:9333
+      - -ip=seaweedfs-filer
+      - -s3
+      - -s3.port=8333
+    depends_on:
+      - seaweedfs-master
+      - seaweedfs-volume
+    volumes:
+      - ./filer.toml:/etc/seaweedfs/filer.toml
+      - seaweedfs-filer-data:/data
+
+volumes:
+  openbao-data:
+  seaweedfs-master-data:
+  seaweedfs-volume-data:
+  seaweedfs-filer-data:
+
+networks:
+  default:
+    name: seaweedfs-kms-test

test/kms/filer.toml

@@ -0,0 +1,85 @@
+# SeaweedFS Filer Configuration for KMS Integration Testing
+
+[leveldb2]
+# Use LevelDB for simple testing
+enabled = true
+dir = "/data/filerdb"
+
+# KMS Configuration for Integration Testing
+[kms]
+# Default KMS provider
+default_provider = "openbao-test"
+
+# KMS provider configurations
+[kms.providers]
+
+# OpenBao provider for integration testing
+[kms.providers.openbao-test]
+type = "openbao"
+address = "http://openbao:8200"
+token = "root-token-for-testing"
+transit_path = "transit"
+tls_skip_verify = true
+request_timeout = 30
+cache_enabled = true
+cache_ttl = "5m"  # Shorter TTL for testing
+max_cache_size = 100
+
+# Alternative Vault provider (for compatibility testing)
+[kms.providers.vault-test]
+type = "vault"
+address = "http://vault:8200"
+token = "root-token-for-testing"
+transit_path = "transit"
+tls_skip_verify = true
+request_timeout = 30
+cache_enabled = true
+cache_ttl = "5m"
+max_cache_size = 100
+
+# Local KMS provider (for comparison/fallback)
+[kms.providers.local-test]
+type = "local"
+enableOnDemandCreate = true
+cache_enabled = false  # Local doesn't need caching
+
+# Simulated AWS KMS provider (for testing AWS integration patterns)
+[kms.providers.aws-localstack]
+type = "aws"
+region = "us-east-1"
+endpoint = "http://localstack:4566"  # LocalStack endpoint
+access_key = "test"
+secret_key = "test"
+tls_skip_verify = true
+connect_timeout = 10
+request_timeout = 30
+max_retries = 3
+cache_enabled = true
+cache_ttl = "10m"
+
+# Bucket-specific KMS provider assignments for testing
+[kms.buckets]
+
+# Test bucket using OpenBao
+[kms.buckets.test-openbao]
+provider = "openbao-test"
+
+# Test bucket using Vault (compatibility)
+[kms.buckets.test-vault]
+provider = "vault-test"
+
+# Test bucket using local KMS
+[kms.buckets.test-local]
+provider = "local-test"
+
+# Test bucket using simulated AWS KMS
+[kms.buckets.test-aws]
+provider = "aws-localstack"
+
+# High security test bucket
+[kms.buckets.secure-data]
+provider = "openbao-test"
+
+# Performance test bucket
+[kms.buckets.perf-test]
+provider = "openbao-test"

test/kms/openbao_integration_test.go

@@ -0,0 +1,598 @@
+package kms_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/vault/api"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/kms"
+	_ "github.com/seaweedfs/seaweedfs/weed/kms/openbao"
+)
+
+const (
+	OpenBaoAddress = "http://127.0.0.1:8200"
+	OpenBaoToken   = "root-token-for-testing"
+	TransitPath    = "transit"
+)
+
+// Test configuration for OpenBao KMS provider
+type testConfig struct {
+	config map[string]interface{}
+}
+
+func (c *testConfig) GetString(key string) string {
+	if val, ok := c.config[key]; ok {
+		if str, ok := val.(string); ok {
+			return str
+		}
+	}
+	return ""
+}
+
+func (c *testConfig) GetBool(key string) bool {
+	if val, ok := c.config[key]; ok {
+		if b, ok := val.(bool); ok {
+			return b
+		}
+	}
+	return false
+}
+
+func (c *testConfig) GetInt(key string) int {
+	if val, ok := c.config[key]; ok {
+		if i, ok := val.(int); ok {
+			return i
+		}
+		if f, ok := val.(float64); ok {
+			return int(f)
+		}
+	}
+	return 0
+}
+
+func (c *testConfig) GetStringSlice(key string) []string {
+	if val, ok := c.config[key]; ok {
+		if slice, ok := val.([]string); ok {
+			return slice
+		}
+	}
+	return nil
+}
+
+func (c *testConfig) SetDefault(key string, value interface{}) {
+	if c.config == nil {
+		c.config = make(map[string]interface{})
+	}
+	if _, exists := c.config[key]; !exists {
+		c.config[key] = value
+	}
+}
+
+// setupOpenBao starts OpenBao in development mode for testing
+func setupOpenBao(t *testing.T) (*exec.Cmd, func()) {
+	// Check if OpenBao is running in Docker (via make dev-openbao)
+	client, err := api.NewClient(&api.Config{Address: OpenBaoAddress})
+	if err == nil {
+		client.SetToken(OpenBaoToken)
+		_, err = client.Sys().Health()
+		if err == nil {
+			glog.V(1).Infof("Using existing OpenBao server at %s", OpenBaoAddress)
+			// Return dummy command and cleanup function for existing server
+			return nil, func() {}
+		}
+	}
+
+	// Check if OpenBao binary is available for starting locally
+	_, err = exec.LookPath("bao")
+	if err != nil {
+		t.Skip("OpenBao not running and bao binary not found. Run 'cd test/kms && make dev-openbao' first")
+	}
+
+	// Start OpenBao in dev mode
+	cmd := exec.Command("bao", "server", "-dev", "-dev-root-token-id="+OpenBaoToken, "-dev-listen-address=127.0.0.1:8200")
+	cmd.Env = append(os.Environ(), "BAO_DEV_ROOT_TOKEN_ID="+OpenBaoToken)
+
+	// Capture output for debugging
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	err = cmd.Start()
+	require.NoError(t, err, "Failed to start OpenBao server")
+
+	// Wait for OpenBao to be ready
+	client, err = api.NewClient(&api.Config{Address: OpenBaoAddress})
+	require.NoError(t, err)
+	client.SetToken(OpenBaoToken)
+
+	// Wait up to 30 seconds for OpenBao to be ready
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	for {
+		select {
+		case <-ctx.Done():
+			cmd.Process.Kill()
+			t.Fatal("Timeout waiting for OpenBao to start")
+		default:
+			// Try to check health
+			resp, err := client.Sys().Health()
+			if err == nil && resp.Initialized {
+				glog.V(1).Infof("OpenBao server ready")
+				goto ready
+			}
+			time.Sleep(500 * time.Millisecond)
+		}
+	}
+
+ready:
+	// Setup cleanup function
+	cleanup := func() {
+		if cmd != nil && cmd.Process != nil {
+			glog.V(1).Infof("Stopping OpenBao server")
+			cmd.Process.Kill()
+			cmd.Wait()
+		}
+	}
+
+	return cmd, cleanup
+}
+
+// setupTransitEngine enables and configures the transit secrets engine
+func setupTransitEngine(t *testing.T) {
+	client, err := api.NewClient(&api.Config{Address: OpenBaoAddress})
+	require.NoError(t, err)
+	client.SetToken(OpenBaoToken)
+
+	// Enable transit secrets engine
+	err = client.Sys().Mount(TransitPath, &api.MountInput{
+		Type:        "transit",
+		Description: "Transit engine for KMS testing",
+	})
+	if err != nil && !strings.Contains(err.Error(), "path is already in use") {
+		require.NoError(t, err, "Failed to enable transit engine")
+	}
+
+	// Create test encryption keys
+	testKeys := []string{"test-key-1", "test-key-2", "seaweedfs-test-key"}
+
+	for _, keyName := range testKeys {
+		keyData := map[string]interface{}{
+			"type": "aes256-gcm96",
+		}
+
+		path := fmt.Sprintf("%s/keys/%s", TransitPath, keyName)
+		_, err = client.Logical().Write(path, keyData)
+		if err != nil && !strings.Contains(err.Error(), "key already exists") {
+			require.NoError(t, err, "Failed to create test key %s", keyName)
+		}
+
+		glog.V(2).Infof("Created/verified test key: %s", keyName)
+	}
+}
+
+func TestOpenBaoKMSProvider_Integration(t *testing.T) {
+	// Start OpenBao server
+	_, cleanup := setupOpenBao(t)
+	defer cleanup()
+
+	// Setup transit engine and keys
+	setupTransitEngine(t)
+
+	t.Run("CreateProvider", func(t *testing.T) {
+		config := &testConfig{
+			config: map[string]interface{}{
+				"address":      OpenBaoAddress,
+				"token":        OpenBaoToken,
+				"transit_path": TransitPath,
+			},
+		}
+
+		provider, err := kms.GetProvider("openbao", config)
+		require.NoError(t, err)
+		require.NotNil(t, provider)
+
+		defer provider.Close()
+	})
+
+	t.Run("ProviderRegistration", func(t *testing.T) {
+		// Test that the provider is registered
+		providers := kms.ListProviders()
+		assert.Contains(t, providers, "openbao")
+		assert.Contains(t, providers, "vault") // Compatibility alias
+	})
+
+	t.Run("GenerateDataKey", func(t *testing.T) {
+		config := &testConfig{
+			config: map[string]interface{}{
+				"address":      OpenBaoAddress,
+				"token":        OpenBaoToken,
+				"transit_path": TransitPath,
+			},
+		}
+
+		provider, err := kms.GetProvider("openbao", config)
+		require.NoError(t, err)
+		defer provider.Close()
+
+		ctx := context.Background()
+		req := &kms.GenerateDataKeyRequest{
+			KeyID:   "test-key-1",
+			KeySpec: kms.KeySpecAES256,
+			EncryptionContext: map[string]string{
+				"test": "context",
+				"env":  "integration",
+			},
+		}
+
+		resp, err := provider.GenerateDataKey(ctx, req)
+		require.NoError(t, err)
+		require.NotNil(t, resp)
+
+		assert.Equal(t, "test-key-1", resp.KeyID)
+		assert.Len(t, resp.Plaintext, 32) // 256 bits
+		assert.NotEmpty(t, resp.CiphertextBlob)
+
+		// Verify the response is in standardized envelope format
+		envelope, err := kms.ParseEnvelope(resp.CiphertextBlob)
+		assert.NoError(t, err)
+		assert.Equal(t, "openbao", envelope.Provider)
+		assert.Equal(t, "test-key-1", envelope.KeyID)
+		assert.True(t, strings.HasPrefix(envelope.Ciphertext, "vault:")) // Raw OpenBao format inside envelope
+	})
+
+	t.Run("DecryptDataKey", func(t *testing.T) {
+		config := &testConfig{
+			config: map[string]interface{}{
+				"address":      OpenBaoAddress,
+				"token":        OpenBaoToken,
+				"transit_path": TransitPath,
+			},
+		}
+
+		provider, err := kms.GetProvider("openbao", config)
+		require.NoError(t, err)
+		defer provider.Close()
+
+		ctx := context.Background()
+
+		// First generate a data key
+		genReq := &kms.GenerateDataKeyRequest{
+			KeyID:   "test-key-1",
+			KeySpec: kms.KeySpecAES256,
+			EncryptionContext: map[string]string{
+				"test": "decrypt",
+				"env":  "integration",
+			},
+		}
+
+		genResp, err := provider.GenerateDataKey(ctx, genReq)
+		require.NoError(t, err)
+
+		// Now decrypt it
+		decReq := &kms.DecryptRequest{
+			CiphertextBlob: genResp.CiphertextBlob,
+			EncryptionContext: map[string]string{
+				"openbao:key:name": "test-key-1",
+				"test":             "decrypt",
+				"env":              "integration",
+			},
+		}
+
+		decResp, err := provider.Decrypt(ctx, decReq)
+		require.NoError(t, err)
+		require.NotNil(t, decResp)
+
+		assert.Equal(t, "test-key-1", decResp.KeyID)
+		assert.Equal(t, genResp.Plaintext, decResp.Plaintext)
+	})
+
+	t.Run("DescribeKey", func(t *testing.T) {
+		config := &testConfig{
+			config: map[string]interface{}{
+				"address":      OpenBaoAddress,
+				"token":        OpenBaoToken,
+				"transit_path": TransitPath,
+			},
+		}
+
+		provider, err := kms.GetProvider("openbao", config)
+		require.NoError(t, err)
+		defer provider.Close()
+
+		ctx := context.Background()
+		req := &kms.DescribeKeyRequest{
+			KeyID: "test-key-1",
+		}
+
+		resp, err := provider.DescribeKey(ctx, req)
+		require.NoError(t, err)
+		require.NotNil(t, resp)
+
+		assert.Equal(t, "test-key-1", resp.KeyID)
+		assert.Contains(t, resp.ARN, "openbao:")
+		assert.Equal(t, kms.KeyStateEnabled, resp.KeyState)
+		assert.Equal(t, kms.KeyUsageEncryptDecrypt, resp.KeyUsage)
+	})
+
+	t.Run("NonExistentKey", func(t *testing.T) {
+		config := &testConfig{
+			config: map[string]interface{}{
+				"address":      OpenBaoAddress,
+				"token":        OpenBaoToken,
+				"transit_path": TransitPath,
+			},
+		}
+
+		provider, err := kms.GetProvider("openbao", config)
+		require.NoError(t, err)
+		defer provider.Close()
+
+		ctx := context.Background()
+		req := &kms.DescribeKeyRequest{
+			KeyID: "non-existent-key",
+		}
+
+		_, err = provider.DescribeKey(ctx, req)
+		require.Error(t, err)
+
+		kmsErr, ok := err.(*kms.KMSError)
+		require.True(t, ok)
+		assert.Equal(t, kms.ErrCodeNotFoundException, kmsErr.Code)
+	})
+
+	t.Run("MultipleKeys", func(t *testing.T) {
+		config := &testConfig{
+			config: map[string]interface{}{
+				"address":      OpenBaoAddress,
+				"token":        OpenBaoToken,
+				"transit_path": TransitPath,
+			},
+		}
+
+		provider, err := kms.GetProvider("openbao", config)
+		require.NoError(t, err)
+		defer provider.Close()
+
+		ctx := context.Background()
+
+		// Test with multiple keys
+		testKeys := []string{"test-key-1", "test-key-2", "seaweedfs-test-key"}
+
+		for _, keyName := range testKeys {
+			t.Run(fmt.Sprintf("Key_%s", keyName), func(t *testing.T) {
+				// Generate data key
+				genReq := &kms.GenerateDataKeyRequest{
+					KeyID:   keyName,
+					KeySpec: kms.KeySpecAES256,
+					EncryptionContext: map[string]string{
+						"key": keyName,
+					},
+				}
+
+				genResp, err := provider.GenerateDataKey(ctx, genReq)
+				require.NoError(t, err)
+				assert.Equal(t, keyName, genResp.KeyID)
+
+				// Decrypt data key
+				decReq := &kms.DecryptRequest{
+					CiphertextBlob: genResp.CiphertextBlob,
+					EncryptionContext: map[string]string{
+						"openbao:key:name": keyName,
+						"key":              keyName,
+					},
+				}
+
+				decResp, err := provider.Decrypt(ctx, decReq)
+				require.NoError(t, err)
+				assert.Equal(t, genResp.Plaintext, decResp.Plaintext)
+			})
+		}
+	})
+}
+
+func TestOpenBaoKMSProvider_ErrorHandling(t *testing.T) {
+	// Start OpenBao server
+	_, cleanup := setupOpenBao(t)
+	defer cleanup()
+
+	setupTransitEngine(t)
+
+	t.Run("InvalidToken", func(t *testing.T) {
+		t.Skip("Skipping invalid token test - OpenBao dev mode may be too permissive")
+
+		config := &testConfig{
+			config: map[string]interface{}{
+				"address":      OpenBaoAddress,
+				"token":        "invalid-token",
+				"transit_path": TransitPath,
+			},
+		}
+
+		provider, err := kms.GetProvider("openbao", config)
+		require.NoError(t, err) // Provider creation doesn't validate token
+		defer provider.Close()
+
+		ctx := context.Background()
+		req := &kms.GenerateDataKeyRequest{
+			KeyID:   "test-key-1",
+			KeySpec: kms.KeySpecAES256,
+		}
+
+		_, err = provider.GenerateDataKey(ctx, req)
+		require.Error(t, err)
+
+		// Check that it's a KMS error (could be access denied or other auth error)
+		kmsErr, ok := err.(*kms.KMSError)
+		require.True(t, ok, "Expected KMSError but got: %T", err)
+		// OpenBao might return different error codes for invalid tokens
+		assert.Contains(t, []string{kms.ErrCodeAccessDenied, kms.ErrCodeKMSInternalFailure}, kmsErr.Code)
+	})
+
+}
+
+func TestKMSManager_WithOpenBao(t *testing.T) {
+	// Start OpenBao server
+	_, cleanup := setupOpenBao(t)
+	defer cleanup()
+
+	setupTransitEngine(t)
+
+	t.Run("KMSManagerIntegration", func(t *testing.T) {
+		manager := kms.InitializeKMSManager()
+
+		// Add OpenBao provider to manager
+		kmsConfig := &kms.KMSConfig{
+			Provider: "openbao",
+			Config: map[string]interface{}{
+				"address":      OpenBaoAddress,
+				"token":        OpenBaoToken,
+				"transit_path": TransitPath,
+			},
+			CacheEnabled: true,
+			CacheTTL:     time.Hour,
+		}
+
+		err := manager.AddKMSProvider("openbao-test", kmsConfig)
+		require.NoError(t, err)
+
+		// Set as default provider
+		err = manager.SetDefaultKMSProvider("openbao-test")
+		require.NoError(t, err)
+
+		// Test bucket-specific assignment
+		err = manager.SetBucketKMSProvider("test-bucket", "openbao-test")
+		require.NoError(t, err)
+
+		// Test key operations through manager
+		ctx := context.Background()
+		resp, err := manager.GenerateDataKeyForBucket(ctx, "test-bucket", "test-key-1", kms.KeySpecAES256, map[string]string{
+			"bucket": "test-bucket",
+		})
+		require.NoError(t, err)
+		require.NotNil(t, resp)
+
+		assert.Equal(t, "test-key-1", resp.KeyID)
+		assert.Len(t, resp.Plaintext, 32)
+
+		// Test decryption through manager
+		decResp, err := manager.DecryptForBucket(ctx, "test-bucket", resp.CiphertextBlob, map[string]string{
+			"bucket": "test-bucket",
+		})
+		require.NoError(t, err)
+		assert.Equal(t, resp.Plaintext, decResp.Plaintext)
+
+		// Test health check
+		health := manager.GetKMSHealth(ctx)
+		assert.Contains(t, health, "openbao-test")
+		assert.NoError(t, health["openbao-test"]) // Should be healthy
+
+		// Cleanup
+		manager.Close()
+	})
+}
+
+// Benchmark tests for performance
+func BenchmarkOpenBaoKMS_GenerateDataKey(b *testing.B) {
+	if testing.Short() {
+		b.Skip("Skipping benchmark in short mode")
+	}
+
+	// Start OpenBao server
+	_, cleanup := setupOpenBao(&testing.T{})
+	defer cleanup()
+
+	setupTransitEngine(&testing.T{})
+
+	config := &testConfig{
+		config: map[string]interface{}{
+			"address":      OpenBaoAddress,
+			"token":        OpenBaoToken,
+			"transit_path": TransitPath,
+		},
+	}
+
+	provider, err := kms.GetProvider("openbao", config)
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer provider.Close()
+
+	ctx := context.Background()
+	req := &kms.GenerateDataKeyRequest{
+		KeyID:   "test-key-1",
+		KeySpec: kms.KeySpecAES256,
+	}
+
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			_, err := provider.GenerateDataKey(ctx, req)
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+}
+
+func BenchmarkOpenBaoKMS_Decrypt(b *testing.B) {
+	if testing.Short() {
+		b.Skip("Skipping benchmark in short mode")
+	}
+
+	// Start OpenBao server
+	_, cleanup := setupOpenBao(&testing.T{})
+	defer cleanup()
+
+	setupTransitEngine(&testing.T{})
+
+	config := &testConfig{
+		config: map[string]interface{}{
+			"address":      OpenBaoAddress,
+			"token":        OpenBaoToken,
+			"transit_path": TransitPath,
+		},
+	}
+
+	provider, err := kms.GetProvider("openbao", config)
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer provider.Close()
+
+	ctx := context.Background()
+
+	// Generate a data key for decryption testing
+	genResp, err := provider.GenerateDataKey(ctx, &kms.GenerateDataKeyRequest{
+		KeyID:   "test-key-1",
+		KeySpec: kms.KeySpecAES256,
+	})
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	decReq := &kms.DecryptRequest{
+		CiphertextBlob: genResp.CiphertextBlob,
+		EncryptionContext: map[string]string{
+			"openbao:key:name": "test-key-1",
+		},
+	}
+
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			_, err := provider.Decrypt(ctx, decReq)
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+}

test/kms/s3-config-openbao-template.json

@@ -0,0 +1,27 @@
+{
+  "identities": [
+    {
+      "name": "admin",
+      "credentials": [
+        {
+          "accessKey": "ACCESS_KEY_PLACEHOLDER",
+          "secretKey": "SECRET_KEY_PLACEHOLDER"
+        }
+      ],
+      "actions": ["Admin", "Read", "List", "Tagging", "Write"]
+    }
+  ],
+  "kms": {
+    "default_provider": "openbao",
+    "providers": {
+      "openbao": {
+        "type": "openbao",
+        "address": "OPENBAO_ADDR_PLACEHOLDER",
+        "token": "OPENBAO_TOKEN_PLACEHOLDER",
+        "transit_path": "transit",
+        "cache_enabled": true,
+        "cache_ttl": "20m"
+      }
+    }
+  }
+}

test/kms/setup_openbao.sh

@@ -0,0 +1,145 @@
+#!/bin/bash
+
+# Setup script for OpenBao KMS integration testing
+set -e
+
+OPENBAO_ADDR=${OPENBAO_ADDR:-"http://127.0.0.1:8200"}
+OPENBAO_TOKEN=${OPENBAO_TOKEN:-"root-token-for-testing"}
+TRANSIT_PATH=${TRANSIT_PATH:-"transit"}
+
+echo "🚀 Setting up OpenBao for KMS integration testing..."
+echo "OpenBao Address: $OPENBAO_ADDR"
+echo "Transit Path: $TRANSIT_PATH"
+
+# Wait for OpenBao to be ready
+echo "⏳ Waiting for OpenBao to be ready..."
+for i in {1..30}; do
+    if curl -s "$OPENBAO_ADDR/v1/sys/health" >/dev/null 2>&1; then
+        echo "[OK] OpenBao is ready!"
+        break
+    fi
+    echo "   Attempt $i/30: OpenBao not ready yet, waiting..."
+    sleep 2
+done
+
+# Check if we can connect
+if ! curl -s -H "X-Vault-Token: $OPENBAO_TOKEN" "$OPENBAO_ADDR/v1/sys/health" >/dev/null; then
+    echo "[FAIL] Cannot connect to OpenBao at $OPENBAO_ADDR"
+    exit 1
+fi
+
+echo "🔧 Setting up transit secrets engine..."
+
+# Enable transit secrets engine (ignore if already enabled)
+curl -s -X POST \
+    -H "X-Vault-Token: $OPENBAO_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d '{"type":"transit","description":"Transit engine for KMS testing"}' \
+    "$OPENBAO_ADDR/v1/sys/mounts/$TRANSIT_PATH" || true
+
+echo "🔑 Creating test encryption keys..."
+
+# Define test keys
+declare -a TEST_KEYS=(
+    "test-key-1:aes256-gcm96:Test key 1 for basic operations"
+    "test-key-2:aes256-gcm96:Test key 2 for multi-key scenarios" 
+    "seaweedfs-test-key:aes256-gcm96:SeaweedFS integration test key"
+    "bucket-default-key:aes256-gcm96:Default key for bucket encryption"
+    "high-security-key:aes256-gcm96:High security test key"
+    "performance-key:aes256-gcm96:Performance testing key"
+    "aws-compat-key:aes256-gcm96:AWS compatibility test key"
+    "multipart-key:aes256-gcm96:Multipart upload test key"
+)
+
+# Create each test key
+for key_spec in "${TEST_KEYS[@]}"; do
+    IFS=':' read -r key_name key_type key_desc <<< "$key_spec"
+    
+    echo "   Creating key: $key_name ($key_type)"
+    
+    # Create the encryption key
+    curl -s -X POST \
+        -H "X-Vault-Token: $OPENBAO_TOKEN" \
+        -H "Content-Type: application/json" \
+        -d "{\"type\":\"$key_type\",\"description\":\"$key_desc\"}" \
+        "$OPENBAO_ADDR/v1/$TRANSIT_PATH/keys/$key_name" || {
+        echo "   ⚠️  Key $key_name might already exist"
+    }
+    
+    # Verify the key was created
+    if curl -s -H "X-Vault-Token: $OPENBAO_TOKEN" "$OPENBAO_ADDR/v1/$TRANSIT_PATH/keys/$key_name" >/dev/null; then
+        echo "   [OK] Key $key_name verified"
+    else
+        echo "   [FAIL] Failed to create/verify key $key_name"
+        exit 1
+    fi
+done
+
+echo "🧪 Testing basic encryption/decryption..."
+
+# Test basic encrypt/decrypt operation
+TEST_PLAINTEXT="Hello, SeaweedFS KMS Integration!"
+PLAINTEXT_B64=$(echo -n "$TEST_PLAINTEXT" | base64)
+
+echo "   Testing with key: test-key-1"
+
+# Encrypt
+ENCRYPT_RESPONSE=$(curl -s -X POST \
+    -H "X-Vault-Token: $OPENBAO_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d "{\"plaintext\":\"$PLAINTEXT_B64\"}" \
+    "$OPENBAO_ADDR/v1/$TRANSIT_PATH/encrypt/test-key-1")
+
+CIPHERTEXT=$(echo "$ENCRYPT_RESPONSE" | jq -r '.data.ciphertext')
+
+if [[ "$CIPHERTEXT" == "null" || -z "$CIPHERTEXT" ]]; then
+    echo "   [FAIL] Encryption test failed"
+    echo "   Response: $ENCRYPT_RESPONSE"
+    exit 1
+fi
+
+echo "   [OK] Encryption successful: ${CIPHERTEXT:0:50}..."
+
+# Decrypt
+DECRYPT_RESPONSE=$(curl -s -X POST \
+    -H "X-Vault-Token: $OPENBAO_TOKEN" \
+    -H "Content-Type: application/json" \
+    -d "{\"ciphertext\":\"$CIPHERTEXT\"}" \
+    "$OPENBAO_ADDR/v1/$TRANSIT_PATH/decrypt/test-key-1")
+
+DECRYPTED_B64=$(echo "$DECRYPT_RESPONSE" | jq -r '.data.plaintext')
+DECRYPTED_TEXT=$(echo "$DECRYPTED_B64" | base64 -d)
+
+if [[ "$DECRYPTED_TEXT" != "$TEST_PLAINTEXT" ]]; then
+    echo "   [FAIL] Decryption test failed"
+    echo "   Expected: $TEST_PLAINTEXT"
+    echo "   Got: $DECRYPTED_TEXT"
+    exit 1
+fi
+
+echo "   [OK] Decryption successful: $DECRYPTED_TEXT"
+
+echo "📊 OpenBao KMS setup summary:"
+echo "   Address: $OPENBAO_ADDR"
+echo "   Transit Path: $TRANSIT_PATH"
+echo "   Keys Created: ${#TEST_KEYS[@]}"
+echo "   Status: Ready for integration testing"
+
+echo ""
+echo "🎯 Ready to run KMS integration tests!"
+echo ""
+echo "Usage:"
+echo "   # Run Go integration tests"
+echo "   go test -v ./test/kms/..."
+echo ""
+echo "   # Run with Docker Compose"
+echo "   cd test/kms && docker-compose up -d"
+echo "   docker-compose exec openbao bao status"
+echo ""
+echo "   # Test S3 API with encryption"
+echo "   aws s3api put-bucket-encryption \\"
+echo "     --endpoint-url http://localhost:8333 \\"
+echo "     --bucket test-bucket \\"
+echo "     --server-side-encryption-configuration file://bucket-encryption.json"
+echo ""
+echo "[OK] OpenBao KMS setup complete!"

test/kms/test_s3_kms.sh

@@ -0,0 +1,217 @@
+#!/bin/bash
+
+# End-to-end S3 KMS integration tests
+set -e
+
+SEAWEEDFS_S3_ENDPOINT=${SEAWEEDFS_S3_ENDPOINT:-"http://127.0.0.1:8333"}
+ACCESS_KEY=${ACCESS_KEY:-"any"}
+SECRET_KEY=${SECRET_KEY:-"any"}
+
+echo "🧪 Running S3 KMS Integration Tests"
+echo "S3 Endpoint: $SEAWEEDFS_S3_ENDPOINT"
+
+# Test file content
+TEST_CONTENT="Hello, SeaweedFS KMS Integration! This is test data that should be encrypted."
+TEST_FILE="/tmp/seaweedfs-kms-test.txt"
+DOWNLOAD_FILE="/tmp/seaweedfs-kms-download.txt"
+
+# Create test file
+echo "$TEST_CONTENT" > "$TEST_FILE"
+
+# AWS CLI configuration
+export AWS_ACCESS_KEY_ID="$ACCESS_KEY"
+export AWS_SECRET_ACCESS_KEY="$SECRET_KEY"
+export AWS_DEFAULT_REGION="us-east-1"
+
+echo "📁 Creating test buckets..."
+
+# Create test buckets
+BUCKETS=("test-openbao" "test-vault" "test-local" "secure-data")
+
+for bucket in "${BUCKETS[@]}"; do
+    echo "   Creating bucket: $bucket"
+    aws s3 mb "s3://$bucket" --endpoint-url "$SEAWEEDFS_S3_ENDPOINT" || {
+        echo "   ⚠️  Bucket $bucket might already exist"
+    }
+done
+
+echo "🔐 Setting up bucket encryption..."
+
+# Test 1: OpenBao KMS Encryption
+echo "   Setting OpenBao encryption for test-openbao bucket..."
+cat > /tmp/openbao-encryption.json << EOF
+{
+    "Rules": [
+        {
+            "ApplyServerSideEncryptionByDefault": {
+                "SSEAlgorithm": "aws:kms",
+                "KMSMasterKeyID": "test-key-1"
+            },
+            "BucketKeyEnabled": false
+        }
+    ]
+}
+EOF
+
+aws s3api put-bucket-encryption \
+    --endpoint-url "$SEAWEEDFS_S3_ENDPOINT" \
+    --bucket test-openbao \
+    --server-side-encryption-configuration file:///tmp/openbao-encryption.json || {
+    echo "   ⚠️  Failed to set bucket encryption for test-openbao"
+}
+
+# Test 2: Verify bucket encryption
+echo "   Verifying bucket encryption configuration..."
+aws s3api get-bucket-encryption \
+    --endpoint-url "$SEAWEEDFS_S3_ENDPOINT" \
+    --bucket test-openbao | jq '.' || {
+    echo "   ⚠️  Failed to get bucket encryption for test-openbao"
+}
+
+echo "⬆️  Testing object uploads with KMS encryption..."
+
+# Test 3: Upload objects with default bucket encryption
+echo "   Uploading object with default bucket encryption..."
+aws s3 cp "$TEST_FILE" "s3://test-openbao/encrypted-object-1.txt" \
+    --endpoint-url "$SEAWEEDFS_S3_ENDPOINT"
+
+# Test 4: Upload object with explicit SSE-KMS
+echo "   Uploading object with explicit SSE-KMS headers..."
+aws s3 cp "$TEST_FILE" "s3://test-openbao/encrypted-object-2.txt" \
+    --endpoint-url "$SEAWEEDFS_S3_ENDPOINT" \
+    --sse aws:kms \
+    --sse-kms-key-id "test-key-2"
+
+# Test 5: Upload to unencrypted bucket
+echo "   Uploading object to unencrypted bucket..."
+aws s3 cp "$TEST_FILE" "s3://test-local/unencrypted-object.txt" \
+    --endpoint-url "$SEAWEEDFS_S3_ENDPOINT"
+
+echo "⬇️  Testing object downloads and decryption..."
+
+# Test 6: Download encrypted objects
+echo "   Downloading encrypted object 1..."
+aws s3 cp "s3://test-openbao/encrypted-object-1.txt" "$DOWNLOAD_FILE" \
+    --endpoint-url "$SEAWEEDFS_S3_ENDPOINT"
+
+# Verify content
+if cmp -s "$TEST_FILE" "$DOWNLOAD_FILE"; then
+    echo "   [OK] Encrypted object 1 downloaded and decrypted successfully"
+else
+    echo "   [FAIL] Encrypted object 1 content mismatch"
+    exit 1
+fi
+
+echo "   Downloading encrypted object 2..."
+aws s3 cp "s3://test-openbao/encrypted-object-2.txt" "$DOWNLOAD_FILE" \
+    --endpoint-url "$SEAWEEDFS_S3_ENDPOINT"
+
+# Verify content
+if cmp -s "$TEST_FILE" "$DOWNLOAD_FILE"; then
+    echo "   [OK] Encrypted object 2 downloaded and decrypted successfully"
+else
+    echo "   [FAIL] Encrypted object 2 content mismatch"
+    exit 1
+fi
+
+echo "📊 Testing object metadata..."
+
+# Test 7: Check encryption metadata
+echo "   Checking encryption metadata..."
+METADATA=$(aws s3api head-object \
+    --endpoint-url "$SEAWEEDFS_S3_ENDPOINT" \
+    --bucket test-openbao \
+    --key encrypted-object-1.txt)
+
+echo "$METADATA" | jq '.'
+
+# Verify SSE headers are present
+if echo "$METADATA" | grep -q "ServerSideEncryption"; then
+    echo "   [OK] SSE metadata found in object headers"
+else
+    echo "   ⚠️  No SSE metadata found (might be internal only)"
+fi
+
+echo "📋 Testing list operations..."
+
+# Test 8: List objects
+echo "   Listing objects in encrypted bucket..."
+aws s3 ls "s3://test-openbao/" --endpoint-url "$SEAWEEDFS_S3_ENDPOINT"
+
+echo "🔄 Testing multipart uploads with encryption..."
+
+# Test 9: Multipart upload with encryption
+LARGE_FILE="/tmp/large-test-file.txt"
+echo "   Creating large test file..."
+for i in {1..1000}; do
+    echo "Line $i: $TEST_CONTENT" >> "$LARGE_FILE"
+done
+
+echo "   Uploading large file with multipart and SSE-KMS..."
+aws s3 cp "$LARGE_FILE" "s3://test-openbao/large-encrypted-file.txt" \
+    --endpoint-url "$SEAWEEDFS_S3_ENDPOINT" \
+    --sse aws:kms \
+    --sse-kms-key-id "multipart-key"
+
+# Download and verify
+echo "   Downloading and verifying large encrypted file..."
+DOWNLOAD_LARGE_FILE="/tmp/downloaded-large-file.txt"
+aws s3 cp "s3://test-openbao/large-encrypted-file.txt" "$DOWNLOAD_LARGE_FILE" \
+    --endpoint-url "$SEAWEEDFS_S3_ENDPOINT"
+
+if cmp -s "$LARGE_FILE" "$DOWNLOAD_LARGE_FILE"; then
+    echo "   [OK] Large encrypted file uploaded and downloaded successfully"
+else
+    echo "   [FAIL] Large encrypted file content mismatch"
+    exit 1
+fi
+
+echo "🧹 Cleaning up test files..."
+rm -f "$TEST_FILE" "$DOWNLOAD_FILE" "$LARGE_FILE" "$DOWNLOAD_LARGE_FILE" /tmp/*-encryption.json
+
+echo "📈 Running performance test..."
+
+# Test 10: Performance test
+PERF_FILE="/tmp/perf-test.txt"
+for i in {1..100}; do
+    echo "Performance test line $i: $TEST_CONTENT" >> "$PERF_FILE"
+done
+
+echo "   Testing upload/download performance with encryption..."
+start_time=$(date +%s)
+
+aws s3 cp "$PERF_FILE" "s3://test-openbao/perf-test.txt" \
+    --endpoint-url "$SEAWEEDFS_S3_ENDPOINT" \
+    --sse aws:kms \
+    --sse-kms-key-id "performance-key"
+
+aws s3 cp "s3://test-openbao/perf-test.txt" "/tmp/perf-download.txt" \
+    --endpoint-url "$SEAWEEDFS_S3_ENDPOINT"
+
+end_time=$(date +%s)
+duration=$((end_time - start_time))
+
+echo "   ⏱️  Performance test completed in ${duration} seconds"
+
+rm -f "$PERF_FILE" "/tmp/perf-download.txt"
+
+echo ""
+echo "🎉 S3 KMS Integration Tests Summary:"
+echo "   [OK] Bucket creation and encryption configuration"
+echo "   [OK] Default bucket encryption"
+echo "   [OK] Explicit SSE-KMS encryption"
+echo "   [OK] Object upload and download"
+echo "   [OK] Encryption/decryption verification" 
+echo "   [OK] Metadata handling"
+echo "   [OK] Multipart upload with encryption"
+echo "   [OK] Performance test"
+echo ""
+echo "🔐 All S3 KMS integration tests passed successfully!"
+echo ""
+
+# Optional: Show bucket sizes and object counts
+echo "📊 Final Statistics:"
+for bucket in "${BUCKETS[@]}"; do
+    COUNT=$(aws s3 ls "s3://$bucket/" --endpoint-url "$SEAWEEDFS_S3_ENDPOINT" | wc -l)
+    echo "   Bucket $bucket: $COUNT objects"
+done

test/kms/wait_for_services.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Wait for services to be ready
+set -e
+
+OPENBAO_ADDR=${OPENBAO_ADDR:-"http://127.0.0.1:8200"}
+SEAWEEDFS_S3_ENDPOINT=${SEAWEEDFS_S3_ENDPOINT:-"http://127.0.0.1:8333"}
+MAX_WAIT=120 # 2 minutes
+
+echo "🕐 Waiting for services to be ready..."
+
+# Wait for OpenBao
+echo "   Waiting for OpenBao at $OPENBAO_ADDR..."
+for i in $(seq 1 $MAX_WAIT); do
+    if curl -s "$OPENBAO_ADDR/v1/sys/health" >/dev/null 2>&1; then
+        echo "   [OK] OpenBao is ready!"
+        break
+    fi
+    if [ $i -eq $MAX_WAIT ]; then
+        echo "   [FAIL] Timeout waiting for OpenBao"
+        exit 1
+    fi
+    sleep 1
+done
+
+# Wait for SeaweedFS Master
+echo "   Waiting for SeaweedFS Master at http://127.0.0.1:9333..."
+for i in $(seq 1 $MAX_WAIT); do
+    if curl -s "http://127.0.0.1:9333/cluster/status" >/dev/null 2>&1; then
+        echo "   [OK] SeaweedFS Master is ready!"
+        break
+    fi
+    if [ $i -eq $MAX_WAIT ]; then
+        echo "   [FAIL] Timeout waiting for SeaweedFS Master"
+        exit 1
+    fi
+    sleep 1
+done
+
+# Wait for SeaweedFS Volume Server
+echo "   Waiting for SeaweedFS Volume Server at http://127.0.0.1:8080..."
+for i in $(seq 1 $MAX_WAIT); do
+    if curl -s "http://127.0.0.1:8080/status" >/dev/null 2>&1; then
+        echo "   [OK] SeaweedFS Volume Server is ready!"
+        break
+    fi
+    if [ $i -eq $MAX_WAIT ]; then
+        echo "   [FAIL] Timeout waiting for SeaweedFS Volume Server"
+        exit 1
+    fi
+    sleep 1
+done
+
+# Wait for SeaweedFS S3 API
+echo "   Waiting for SeaweedFS S3 API at $SEAWEEDFS_S3_ENDPOINT..."
+for i in $(seq 1 $MAX_WAIT); do
+    if curl -s "$SEAWEEDFS_S3_ENDPOINT/" >/dev/null 2>&1; then
+        echo "   [OK] SeaweedFS S3 API is ready!"
+        break
+    fi
+    if [ $i -eq $MAX_WAIT ]; then
+        echo "   [FAIL] Timeout waiting for SeaweedFS S3 API"
+        exit 1
+    fi
+    sleep 1
+done
+
+echo "🎉 All services are ready!"
+
+# Show service status
+echo ""
+echo "📊 Service Status:"
+echo "   OpenBao:             $(curl -s $OPENBAO_ADDR/v1/sys/health | jq -r '.initialized // "Unknown"')"
+echo "   SeaweedFS Master:    $(curl -s http://127.0.0.1:9333/cluster/status | jq -r '.IsLeader // "Unknown"')"
+echo "   SeaweedFS Volume:    $(curl -s http://127.0.0.1:8080/status | jq -r '.Version // "Unknown"')"
+echo "   SeaweedFS S3 API:    Ready"
+echo ""