upstream-seaweedfs/test/s3/versioning/s3_lock_key_verb_parity_test.go

package s3api

import (
	"bytes"
	"context"
	"errors"
	"io"
	"strings"
	"testing"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	smithyhttp "github.com/aws/smithy-go/transport/http"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// Backup clients arbitrate ownership by writing and retracting small keys under a
// fixed prefix, then re-probing them. Each probe uses a different verb, and the
// client trusts them to agree: a key reported present by one and absent by another
// makes it either spin or conclude the repository is corrupt. The keys are written
// and immediately deleted by version, which is the cycle that leaves versioning
// residue behind, so parity has to hold on both sides of the delete.

var lockArbitrationKeys = []string{
	"Metadata/Lock/create.checkpoint/lock",
	"Metadata/Lock/create.checkpoint/try",
	"Metadata/Lock/create.checkpoint/declare",
	"Metadata/Lock/delete.checkpoint/lock",
}

// existence reports what each verb believes about a key, so a disagreement names
// the verbs rather than just failing.
type existence struct {
	get         bool
	head        bool
	listObjects bool
	listVersion bool
}

func (e existence) agree() bool {
	return e.get == e.head && e.head == e.listObjects && e.listObjects == e.listVersion
}

func notFound(t *testing.T, err error) bool {
	t.Helper()
	if err == nil {
		return false
	}
	var respErr *smithyhttp.ResponseError
	if errors.As(err, &respErr) {
		require.Less(t, respErr.HTTPStatusCode(), 500, "probing a lock key must never fault the server: %v", err)
		return respErr.HTTPStatusCode() == 404
	}
	t.Fatalf("unexpected error probing lock key: %v", err)
	return false
}

func probe(t *testing.T, client *s3.Client, bucket, key string) existence {
	t.Helper()
	var e existence

	getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
		Bucket: aws.String(bucket), Key: aws.String(key),
	})
	if err == nil {
		// Drain before closing so the connection goes back to the pool; these
		// probes run in tight loops and a discarded body forces a new connection.
		_, _ = io.Copy(io.Discard, getResp.Body)
		getResp.Body.Close()
		e.get = true
	} else {
		require.True(t, notFound(t, err), "GET must answer present or 404, got %v", err)
	}

	if _, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
		Bucket: aws.String(bucket), Key: aws.String(key),
	}); err == nil {
		e.head = true
	} else {
		require.True(t, notFound(t, err), "HEAD must answer present or 404, got %v", err)
	}

	listed, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
		Bucket: aws.String(bucket), Prefix: aws.String(key),
	})
	require.NoError(t, err)
	for _, obj := range listed.Contents {
		if obj.Key != nil && *obj.Key == key {
			e.listObjects = true
		}
	}

	versions, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
		Bucket: aws.String(bucket), Prefix: aws.String(key),
	})
	require.NoError(t, err)
	for _, v := range versions.Versions {
		// A version is only "the object exists" if it is the current one; a
		// superseded version coexists with a delete marker on a live key.
		if v.Key != nil && *v.Key == key && v.IsLatest != nil && *v.IsLatest {
			e.listVersion = true
		}
	}

	return e
}

func TestLockKeyVerbParityWhilePresent(t *testing.T) {
	client := getS3Client(t)
	bucketName := getNewBucketName()

	createBucketWithObjectLock(t, client, bucketName)
	defer deleteBucket(t, client, bucketName)

	for _, key := range lockArbitrationKeys {
		_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
			Bucket: aws.String(bucketName),
			Key:    aws.String(key),
			Body:   bytes.NewReader([]byte("held")),
		})
		require.NoError(t, err)
	}

	for _, key := range lockArbitrationKeys {
		t.Run(strings.ReplaceAll(key, "/", "_"), func(t *testing.T) {
			e := probe(t, client, bucketName, key)
			assert.True(t, e.agree(), "verbs disagree on a present lock key: %+v", e)
			assert.True(t, e.get, "lock key should be present: %+v", e)
		})
	}
}

// The retract half of the cycle: write, then delete by version id — the sequence
// that empties a version container. Every verb must then agree the key is gone.
func TestLockKeyVerbParityAfterVersionDelete(t *testing.T) {
	client := getS3Client(t)
	bucketName := getNewBucketName()

	createBucketWithObjectLock(t, client, bucketName)
	defer deleteBucket(t, client, bucketName)

	for _, key := range lockArbitrationKeys {
		put, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
			Bucket: aws.String(bucketName),
			Key:    aws.String(key),
			Body:   bytes.NewReader([]byte("held")),
		})
		require.NoError(t, err)
		require.NotNil(t, put.VersionId)

		_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
			Bucket:    aws.String(bucketName),
			Key:       aws.String(key),
			VersionId: put.VersionId,
		})
		require.NoError(t, err)
	}

	for _, key := range lockArbitrationKeys {
		t.Run(strings.ReplaceAll(key, "/", "_"), func(t *testing.T) {
			e := probe(t, client, bucketName, key)
			assert.True(t, e.agree(), "verbs disagree on a retracted lock key: %+v", e)
			assert.False(t, e.get, "retracted lock key should be absent: %+v", e)
		})
	}
}

// Re-acquiring after a retract is the steady-state loop. Residue left by the
// previous cycle must not make the new key invisible to any verb.
func TestLockKeyVerbParityAcrossReacquireCycles(t *testing.T) {
	client := getS3Client(t)
	bucketName := getNewBucketName()

	createBucketWithObjectLock(t, client, bucketName)
	defer deleteBucket(t, client, bucketName)

	for _, key := range lockArbitrationKeys {
		t.Run(strings.ReplaceAll(key, "/", "_"), func(t *testing.T) {
			for cycle := 0; cycle < 3; cycle++ {
				put, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
					Bucket: aws.String(bucketName),
					Key:    aws.String(key),
					Body:   bytes.NewReader([]byte("held")),
				})
				require.NoError(t, err)
				require.NotNil(t, put.VersionId)

				held := probe(t, client, bucketName, key)
				require.True(t, held.agree(), "cycle %d: verbs disagree while held: %+v", cycle, held)
				require.True(t, held.get, "cycle %d: key should be present while held: %+v", cycle, held)

				_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
					Bucket:    aws.String(bucketName),
					Key:       aws.String(key),
					VersionId: put.VersionId,
				})
				require.NoError(t, err)

				released := probe(t, client, bucketName, key)
				require.True(t, released.agree(), "cycle %d: verbs disagree after release: %+v", cycle, released)
				require.False(t, released.get, "cycle %d: key should be absent after release: %+v", cycle, released)
			}
		})
	}
}