snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/s3/sse/s3_volume_encryption_test.go

@@ -0,0 +1,471 @@
+//go:build integration
+
+package sse
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+// TestS3VolumeEncryptionRoundtrip tests that data uploaded with encryptVolumeData
+// enabled can be read back correctly. This requires the S3 server to be started with
+// -encryptVolumeData=true for the test to verify encryption is working.
+//
+// To run this test:
+// 1. Start SeaweedFS: weed server -s3 -s3.encryptVolumeData=true
+// 2. Run: go test -v -run TestS3VolumeEncryptionRoundtrip
+func TestS3VolumeEncryptionRoundtrip(t *testing.T) {
+	svc := getS3Client(t)
+	bucket := fmt.Sprintf("volume-encryption-test-%d", time.Now().Unix())
+
+	// Create bucket
+	_, err := svc.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(bucket),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bucket: %v", err)
+	}
+	defer cleanupBucket(t, svc, bucket)
+
+	testCases := []struct {
+		name          string
+		key           string
+		content       string
+		binaryContent []byte
+		isBinary      bool
+		rangeReq      string // Optional range request
+	}{
+		{
+			name:    "small file",
+			key:     "small.txt",
+			content: "Hello, encrypted world!",
+		},
+		{
+			name:    "medium file",
+			key:     "medium.txt",
+			content: strings.Repeat("SeaweedFS volume encryption test content. ", 1000),
+		},
+		{
+			name: "binary content",
+			key:  "binary.bin",
+			binaryContent: func() []byte {
+				// Create a 1KB binary file with predictable pattern including null bytes
+				data := make([]byte, 1024)
+				for i := range data {
+					data[i] = byte(i % 256)
+				}
+				return data
+			}(),
+			isBinary: true,
+		},
+		{
+			name:     "range request",
+			key:      "range-test.txt",
+			content:  "0123456789ABCDEFGHIJ",
+			rangeReq: "bytes=5-10",
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc // Capture range variable for closure
+		t.Run(tc.name, func(t *testing.T) {
+			// Add small delay to avoid rapid consecutive requests during volume encryption
+			time.Sleep(50 * time.Millisecond)
+
+			// Upload - use appropriate reader for binary vs text content
+			var uploadBody io.ReadSeeker
+			var expectedData []byte
+			if tc.isBinary {
+				uploadBody = bytes.NewReader(tc.binaryContent)
+				expectedData = tc.binaryContent
+			} else {
+				uploadBody = strings.NewReader(tc.content)
+				expectedData = []byte(tc.content)
+			}
+
+			_, err := svc.PutObject(&s3.PutObjectInput{
+				Bucket: aws.String(bucket),
+				Key:    aws.String(tc.key),
+				Body:   uploadBody,
+			})
+			if err != nil {
+				t.Fatalf("PutObject failed: %v", err)
+			}
+
+			// Download
+			getInput := &s3.GetObjectInput{
+				Bucket: aws.String(bucket),
+				Key:    aws.String(tc.key),
+			}
+			if tc.rangeReq != "" {
+				getInput.Range = aws.String(tc.rangeReq)
+			}
+
+			result, err := svc.GetObject(getInput)
+			if err != nil {
+				t.Fatalf("GetObject failed: %v", err)
+			}
+			defer result.Body.Close()
+
+			data, err := io.ReadAll(result.Body)
+			if err != nil {
+				t.Fatalf("Failed to read response body: %v", err)
+			}
+
+			// Verify content
+			expected := expectedData
+			if tc.rangeReq != "" {
+				// For "bytes=5-10", we expect bytes at positions 5-10 (inclusive)
+				expected = expectedData[5:11]
+			}
+
+			if !bytes.Equal(data, expected) {
+				t.Errorf("Content mismatch:\n  expected: %v\n  got: %v", expected, data)
+			} else {
+				t.Logf("Successfully uploaded and downloaded %s (%d bytes)", tc.key, len(data))
+			}
+		})
+	}
+}
+
+// TestS3VolumeEncryptionMultiChunk tests large files that span multiple chunks
+// to ensure encryption works correctly across chunk boundaries.
+func TestS3VolumeEncryptionMultiChunk(t *testing.T) {
+	svc := getS3Client(t)
+	bucket := fmt.Sprintf("volume-encryption-multichunk-%d", time.Now().Unix())
+
+	// Create bucket
+	_, err := svc.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(bucket),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bucket: %v", err)
+	}
+	defer cleanupBucket(t, svc, bucket)
+
+	// Create a file larger than default chunk size (8MB)
+	// Use 10MB to ensure multiple chunks
+	largeContent := make([]byte, 10*1024*1024)
+	for i := range largeContent {
+		largeContent[i] = byte(i % 256)
+	}
+
+	key := "large-file.bin"
+
+	// Upload
+	_, err = svc.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+		Body:   bytes.NewReader(largeContent),
+	})
+	if err != nil {
+		t.Fatalf("PutObject failed for large file: %v", err)
+	}
+
+	// Download full file
+	result, err := svc.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+	if err != nil {
+		t.Fatalf("GetObject failed: %v", err)
+	}
+	defer result.Body.Close()
+
+	downloadedData, err := io.ReadAll(result.Body)
+	if err != nil {
+		t.Fatalf("Failed to read response body: %v", err)
+	}
+
+	if len(downloadedData) != len(largeContent) {
+		t.Errorf("Size mismatch: expected %d, got %d", len(largeContent), len(downloadedData))
+	}
+
+	if !bytes.Equal(downloadedData, largeContent) {
+		t.Errorf("Content mismatch in multi-chunk file")
+		// Find first mismatch
+		for i := 0; i < len(downloadedData) && i < len(largeContent); i++ {
+			if downloadedData[i] != largeContent[i] {
+				t.Errorf("First mismatch at byte %d: expected %02x, got %02x", i, largeContent[i], downloadedData[i])
+				break
+			}
+		}
+	} else {
+		t.Logf("Successfully uploaded and downloaded %d byte multi-chunk file", len(downloadedData))
+	}
+
+	// Test range request spanning chunk boundary (around 8MB)
+	rangeStart := int64(8*1024*1024 - 1000)
+	rangeEnd := int64(8*1024*1024 + 1000)
+	rangeResult, err := svc.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+		Range:  aws.String(fmt.Sprintf("bytes=%d-%d", rangeStart, rangeEnd)),
+	})
+	if err != nil {
+		t.Fatalf("Range GetObject failed: %v", err)
+	}
+	defer rangeResult.Body.Close()
+
+	rangeData, err := io.ReadAll(rangeResult.Body)
+	if err != nil {
+		t.Fatalf("Failed to read range response: %v", err)
+	}
+
+	expectedRange := largeContent[rangeStart : rangeEnd+1]
+	if !bytes.Equal(rangeData, expectedRange) {
+		t.Errorf("Range request content mismatch at chunk boundary")
+	} else {
+		t.Logf("Successfully retrieved range spanning chunk boundary (%d bytes)", len(rangeData))
+	}
+}
+
+// TestS3VolumeEncryptionMultiChunkRangeRead tests range reads that span multiple chunks:
+// - Part of chunk 1
+// - Whole chunk 2
+// - Part of chunk 3
+// This is critical for verifying that cipher decryption works correctly when
+// reading across chunk boundaries with the cipherKey from each chunk.
+func TestS3VolumeEncryptionMultiChunkRangeRead(t *testing.T) {
+	svc := getS3Client(t)
+	bucket := fmt.Sprintf("volume-encryption-multirange-%d", time.Now().Unix())
+
+	// Create bucket
+	_, err := svc.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(bucket),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bucket: %v", err)
+	}
+	defer cleanupBucket(t, svc, bucket)
+
+	// Default chunk size is 8MB. Create a file with 3+ chunks (25MB)
+	// to ensure we have multiple complete chunks
+	const chunkSize = 8 * 1024 * 1024 // 8MB
+	const fileSize = 25 * 1024 * 1024 // 25MB = 3 chunks + partial 4th
+
+	largeContent := make([]byte, fileSize)
+	for i := range largeContent {
+		// Use recognizable pattern: each byte encodes its position
+		largeContent[i] = byte(i % 256)
+	}
+
+	key := "multi-chunk-range-test.bin"
+
+	// Upload
+	_, err = svc.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+		Body:   bytes.NewReader(largeContent),
+	})
+	if err != nil {
+		t.Fatalf("PutObject failed: %v", err)
+	}
+
+	t.Logf("Uploaded %d byte file (%d chunks of %d bytes)", fileSize, (fileSize+chunkSize-1)/chunkSize, chunkSize)
+
+	// Test cases for range reads spanning multiple chunks
+	testCases := []struct {
+		name       string
+		rangeStart int64
+		rangeEnd   int64
+		desc       string
+	}{
+		{
+			name:       "part_chunk1_whole_chunk2_part_chunk3",
+			rangeStart: chunkSize - 1000,   // 1000 bytes before end of chunk 1
+			rangeEnd:   2*chunkSize + 1000, // 1000 bytes into chunk 3
+			desc:       "Spans from end of chunk 1 through entire chunk 2 into beginning of chunk 3",
+		},
+		{
+			name:       "last_byte_chunk1_through_first_byte_chunk3",
+			rangeStart: chunkSize - 1, // Last byte of chunk 1
+			rangeEnd:   2 * chunkSize, // First byte of chunk 3
+			desc:       "Minimal span: last byte of chunk 1, entire chunk 2, first byte of chunk 3",
+		},
+		{
+			name:       "middle_chunk1_to_middle_chunk3",
+			rangeStart: chunkSize / 2,             // Middle of chunk 1
+			rangeEnd:   2*chunkSize + chunkSize/2, // Middle of chunk 3
+			desc:       "From middle of chunk 1 to middle of chunk 3",
+		},
+		{
+			name:       "half_chunk1_whole_chunk2_half_chunk3",
+			rangeStart: chunkSize / 2,                 // Half of chunk 1
+			rangeEnd:   2*chunkSize + chunkSize/2 - 1, // Half of chunk 3
+			desc:       "Half of each boundary chunk, whole middle chunk",
+		},
+		{
+			name:       "single_byte_each_boundary",
+			rangeStart: chunkSize - 1, // 1 byte from chunk 1
+			rangeEnd:   2 * chunkSize, // 1 byte from chunk 3
+			desc:       "1 byte from chunk 1, entire chunk 2, 1 byte from chunk 3",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Logf("Test: %s", tc.desc)
+			t.Logf("Range: bytes=%d-%d (spanning bytes at offsets across chunk boundaries)", tc.rangeStart, tc.rangeEnd)
+
+			result, err := svc.GetObject(&s3.GetObjectInput{
+				Bucket: aws.String(bucket),
+				Key:    aws.String(key),
+				Range:  aws.String(fmt.Sprintf("bytes=%d-%d", tc.rangeStart, tc.rangeEnd)),
+			})
+			if err != nil {
+				t.Fatalf("Range GetObject failed: %v", err)
+			}
+			defer result.Body.Close()
+
+			rangeData, err := io.ReadAll(result.Body)
+			if err != nil {
+				t.Fatalf("Failed to read range response: %v", err)
+			}
+
+			expectedLen := tc.rangeEnd - tc.rangeStart + 1
+			if int64(len(rangeData)) != expectedLen {
+				t.Errorf("Size mismatch: expected %d bytes, got %d", expectedLen, len(rangeData))
+			}
+
+			expectedRange := largeContent[tc.rangeStart : tc.rangeEnd+1]
+			if !bytes.Equal(rangeData, expectedRange) {
+				t.Errorf("Content mismatch in multi-chunk range read")
+				// Find first mismatch for debugging
+				for i := 0; i < len(rangeData) && i < len(expectedRange); i++ {
+					if rangeData[i] != expectedRange[i] {
+						globalOffset := tc.rangeStart + int64(i)
+						chunkNum := globalOffset / chunkSize
+						offsetInChunk := globalOffset % chunkSize
+						t.Errorf("First mismatch at byte %d (chunk %d, offset %d in chunk): expected %02x, got %02x",
+							i, chunkNum, offsetInChunk, expectedRange[i], rangeData[i])
+						break
+					}
+				}
+			} else {
+				t.Logf("✓ Successfully read %d bytes spanning chunks (offsets %d-%d)", len(rangeData), tc.rangeStart, tc.rangeEnd)
+			}
+		})
+	}
+}
+
+// TestS3VolumeEncryptionCopy tests that copying encrypted objects works correctly.
+func TestS3VolumeEncryptionCopy(t *testing.T) {
+	svc := getS3Client(t)
+	bucket := fmt.Sprintf("volume-encryption-copy-%d", time.Now().Unix())
+
+	// Create bucket
+	_, err := svc.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(bucket),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bucket: %v", err)
+	}
+	defer cleanupBucket(t, svc, bucket)
+
+	srcKey := "source-object.txt"
+	dstKey := "copied-object.txt"
+	content := "Content to be copied with volume-level encryption"
+
+	// Upload source
+	_, err = svc.PutObject(&s3.PutObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(srcKey),
+		Body:   strings.NewReader(content),
+	})
+	if err != nil {
+		t.Fatalf("PutObject failed: %v", err)
+	}
+
+	// Copy object
+	_, err = svc.CopyObject(&s3.CopyObjectInput{
+		Bucket:     aws.String(bucket),
+		Key:        aws.String(dstKey),
+		CopySource: aws.String(fmt.Sprintf("%s/%s", bucket, srcKey)),
+	})
+	if err != nil {
+		t.Fatalf("CopyObject failed: %v", err)
+	}
+
+	// Read copied object
+	result, err := svc.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(dstKey),
+	})
+	if err != nil {
+		t.Fatalf("GetObject failed for copied object: %v", err)
+	}
+	defer result.Body.Close()
+
+	data, err := io.ReadAll(result.Body)
+	if err != nil {
+		t.Fatalf("Failed to read copied object: %v", err)
+	}
+
+	if string(data) != content {
+		t.Errorf("Copied content mismatch:\n  expected: %q\n  got: %q", content, string(data))
+	} else {
+		t.Logf("Successfully copied encrypted object")
+	}
+}
+
+// Helper functions
+
+func getS3Client(t *testing.T) *s3.S3 {
+	// Use credentials that match the Makefile configuration
+	// ACCESS_KEY ?= some_access_key1
+	// SECRET_KEY ?= some_secret_key1
+	sess, err := session.NewSession(&aws.Config{
+		Region:           aws.String("us-east-1"),
+		Endpoint:         aws.String("http://localhost:8333"),
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+		Credentials:      credentials.NewStaticCredentials("some_access_key1", "some_secret_key1", ""),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create session: %v", err)
+	}
+	return s3.New(sess)
+}
+
+func cleanupBucket(t *testing.T, svc *s3.S3, bucket string) {
+	ctx := context.Background()
+	_ = ctx
+
+	// List and delete all objects
+	listResult, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
+		Bucket: aws.String(bucket),
+	})
+	if err != nil {
+		t.Logf("Warning: failed to list objects for cleanup: %v", err)
+		return
+	}
+
+	for _, obj := range listResult.Contents {
+		_, err := svc.DeleteObject(&s3.DeleteObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    obj.Key,
+		})
+		if err != nil {
+			t.Logf("Warning: failed to delete object %s: %v", *obj.Key, err)
+		}
+	}
+
+	// Delete bucket
+	_, err = svc.DeleteBucket(&s3.DeleteBucketInput{
+		Bucket: aws.String(bucket),
+	})
+	if err != nil {
+		t.Logf("Warning: failed to delete bucket %s: %v", bucket, err)
+	}
+}