upstream-seaweedfs/test/s3/versioning/s3_velero_integration_test.go

package s3api

import (
	"context"
	"io"
	"strings"
	"testing"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// TestVeleroKopiaVersionedObjectListing tests the exact access patterns used by Velero/Kopia
// when backing up to a versioned S3 bucket. This test validates the fix for:
// https://github.com/seaweedfs/seaweedfs/discussions/7573
//
// The bug was that when listing versioned objects with nested paths like
// "kopia/logpaste/kopia.blobcfg", the returned Key would be doubled:
// "kopia/logpaste/kopia/logpaste/kopia.blobcfg" instead of the correct path.
//
// This was caused by getLatestVersionEntryForListOperation setting the entry's
// Name to the full path instead of just the base filename.
func TestVeleroKopiaVersionedObjectListing(t *testing.T) {
	client := getS3Client(t)
	bucketName := getNewBucketName()

	// Step 1: Create bucket
	createBucket(t, client, bucketName)
	defer deleteBucket(t, client, bucketName)

	// Step 2: Enable versioning (required to reproduce the bug)
	enableVersioning(t, client, bucketName)

	// Step 3: Create objects matching Velero/Kopia's typical path structure
	// These are the actual paths Kopia uses when storing repository data
	testObjects := map[string]string{
		"kopia/logpaste/kopia.blobcfg":    "blob-config-content",
		"kopia/logpaste/kopia.repository": "repository-content",
		"kopia/logpaste/.storageconfig":   "storage-config-content",
		"backups/namespace1/backup.tar":   "backup-content",
		"restic/locks/lock-001.json":      "lock-content",
		"restic/index/index-001.json":     "index-content",
		"data/a/b/c/deeply-nested.dat":    "nested-content",
		"single-file.txt":                 "single-file-content",
	}

	// Upload all test objects
	for key, content := range testObjects {
		_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
			Bucket: aws.String(bucketName),
			Key:    aws.String(key),
			Body:   strings.NewReader(content),
		})
		require.NoError(t, err, "Failed to upload object %s", key)
	}

	// Step 4: Test listing with various prefixes (mimicking Kopia's ListObjects calls)
	testCases := []struct {
		name         string
		prefix       string
		expectedKeys []string
		description  string
	}{
		{
			name:   "kopia repository prefix",
			prefix: "kopia/logpaste/",
			expectedKeys: []string{
				"kopia/logpaste/.storageconfig",
				"kopia/logpaste/kopia.blobcfg",
				"kopia/logpaste/kopia.repository",
			},
			description: "Kopia lists its repository directory to find config files",
		},
		{
			name:         "exact config file",
			prefix:       "kopia/logpaste/kopia.blobcfg",
			expectedKeys: []string{"kopia/logpaste/kopia.blobcfg"},
			description:  "Kopia checks for specific config files",
		},
		{
			name:   "restic index directory",
			prefix: "restic/index/",
			expectedKeys: []string{
				"restic/index/index-001.json",
			},
			description: "Restic lists index files (similar pattern to Kopia)",
		},
		{
			name:   "deeply nested path",
			prefix: "data/a/b/c/",
			expectedKeys: []string{
				"data/a/b/c/deeply-nested.dat",
			},
			description: "Tests deeply nested paths don't get doubled",
		},
		{
			name:   "backup directory",
			prefix: "backups/",
			expectedKeys: []string{
				"backups/namespace1/backup.tar",
			},
			description: "Velero lists backup directories",
		},
		{
			name:   "root level single file",
			prefix: "single-file",
			expectedKeys: []string{
				"single-file.txt",
			},
			description: "Files at bucket root should work correctly",
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			// List objects with prefix
			listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
				Bucket: aws.String(bucketName),
				Prefix: aws.String(tc.prefix),
			})
			require.NoError(t, err)

			// Extract returned keys
			var actualKeys []string
			for _, obj := range listResp.Contents {
				key := *obj.Key
				actualKeys = append(actualKeys, key)

				// Critical assertion: verify no path doubling
				// The bug would cause "kopia/logpaste/kopia.blobcfg" to become
				// "kopia/logpaste/kopia/logpaste/kopia.blobcfg"
				assert.False(t, hasDoubledPath(key),
					"Path doubling detected! Key '%s' should not have repeated path segments", key)

				// Also verify we can actually GET the object using the listed key
				getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
					Bucket: aws.String(bucketName),
					Key:    aws.String(key),
				})
				require.NoError(t, err, "Should be able to GET object using listed key: %s", key)
				getResp.Body.Close()
			}

			// Verify we got the expected keys
			assert.ElementsMatch(t, tc.expectedKeys, actualKeys,
				"%s: Listed keys should match expected", tc.description)
		})
	}
}

// TestVeleroKopiaGetAfterList simulates Kopia's pattern of listing and then getting objects
// This specifically tests the bug where listed paths couldn't be retrieved
func TestVeleroKopiaGetAfterList(t *testing.T) {
	client := getS3Client(t)
	bucketName := getNewBucketName()

	createBucket(t, client, bucketName)
	defer deleteBucket(t, client, bucketName)
	enableVersioning(t, client, bucketName)

	// Create a nested object like Kopia does
	objectKey := "kopia/appwrite/kopia.repository"
	objectContent := "kopia-repository-config-data"

	_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
		Bucket: aws.String(bucketName),
		Key:    aws.String(objectKey),
		Body:   strings.NewReader(objectContent),
	})
	require.NoError(t, err)

	// List objects (this is where the bug manifested - returned wrong Key)
	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
		Bucket: aws.String(bucketName),
		Prefix: aws.String("kopia/appwrite/"),
	})
	require.NoError(t, err)
	require.Len(t, listResp.Contents, 1, "Should list exactly one object")

	listedKey := *listResp.Contents[0].Key

	// The bug caused listedKey to be "kopia/appwrite/kopia/appwrite/kopia.repository"
	// instead of "kopia/appwrite/kopia.repository"
	assert.Equal(t, objectKey, listedKey,
		"Listed key should match the original key without path doubling")

	// Now try to GET the object using the listed key
	// With the bug, this would fail because the doubled path doesn't exist
	getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
		Bucket: aws.String(bucketName),
		Key:    aws.String(listedKey),
	})
	require.NoError(t, err, "Should be able to GET object using the key returned from ListObjects")
	defer getResp.Body.Close()

	// Verify content matches
	body, err := io.ReadAll(getResp.Body)
	require.NoError(t, err)
	assert.Equal(t, objectContent, string(body),
		"Retrieved content should match original")
}

// TestVeleroMultipleVersionsWithNestedPaths tests listing when objects have multiple versions
func TestVeleroMultipleVersionsWithNestedPaths(t *testing.T) {
	client := getS3Client(t)
	bucketName := getNewBucketName()

	createBucket(t, client, bucketName)
	defer deleteBucket(t, client, bucketName)
	enableVersioning(t, client, bucketName)

	objectKey := "kopia/logpaste/kopia.blobcfg"

	// Create multiple versions of the same object (Kopia updates config files)
	versions := []string{"version-1", "version-2", "version-3"}
	for _, content := range versions {
		_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
			Bucket: aws.String(bucketName),
			Key:    aws.String(objectKey),
			Body:   strings.NewReader(content),
		})
		require.NoError(t, err)
	}

	// List objects - should return only the latest version in regular listing
	listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
		Bucket: aws.String(bucketName),
		Prefix: aws.String("kopia/logpaste/"),
	})
	require.NoError(t, err)

	// Should only see one object (the latest version)
	require.Len(t, listResp.Contents, 1, "Should list only one object (latest version)")

	listedKey := *listResp.Contents[0].Key
	assert.Equal(t, objectKey, listedKey,
		"Listed key should match original without path doubling")
	assert.False(t, hasDoubledPath(listedKey),
		"Path should not be doubled")

	// Verify we can GET the latest version
	getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
		Bucket: aws.String(bucketName),
		Key:    aws.String(listedKey),
	})
	require.NoError(t, err)
	defer getResp.Body.Close()

	body, err := io.ReadAll(getResp.Body)
	require.NoError(t, err)
	assert.Equal(t, "version-3", string(body),
		"Should get the latest version content")
}

// TestVeleroListVersionsWithNestedPaths tests ListObjectVersions with nested paths
func TestVeleroListVersionsWithNestedPaths(t *testing.T) {
	client := getS3Client(t)
	bucketName := getNewBucketName()

	createBucket(t, client, bucketName)
	defer deleteBucket(t, client, bucketName)
	enableVersioning(t, client, bucketName)

	objectKey := "backups/velero/test-backup/backup.json"

	// Create multiple versions
	for i := 1; i <= 3; i++ {
		_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
			Bucket: aws.String(bucketName),
			Key:    aws.String(objectKey),
			Body:   strings.NewReader(strings.Repeat("x", i*100)),
		})
		require.NoError(t, err)
	}

	// List all versions
	listResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
		Bucket: aws.String(bucketName),
		Prefix: aws.String("backups/velero/"),
	})
	require.NoError(t, err)

	// Should have 3 versions
	require.Len(t, listResp.Versions, 3, "Should have 3 versions")

	// Verify all version keys are correct (not doubled)
	for _, version := range listResp.Versions {
		key := *version.Key
		assert.Equal(t, objectKey, key,
			"Version key should match original")
		assert.False(t, hasDoubledPath(key),
			"Version key should not have doubled path: %s", key)

		// Verify we can GET each version by its version ID
		getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
			Bucket:    aws.String(bucketName),
			Key:       aws.String(key),
			VersionId: version.VersionId,
		})
		require.NoError(t, err, "Should be able to GET version %s", *version.VersionId)
		func() {
			defer getResp.Body.Close()
		}()
	}
}

// hasDoubledPath checks if a path has a repeated pair of segments, which indicates
// the path doubling bug. For example: "kopia/logpaste/kopia/logpaste/file.txt" would return true.
func hasDoubledPath(path string) bool {
	parts := strings.Split(path, "/")
	if len(parts) < 4 {
		return false
	}

	// Check for repeated pairs of segments, e.g., a/b/.../a/b.
	for i := 0; i < len(parts)-3; i++ {
		for j := i + 2; j < len(parts)-1; j++ {
			if parts[i] == parts[j] && parts[i+1] == parts[j+1] {
				return true
			}
		}
	}

	return false
}