snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/s3api/filer_util_delete_test.go

@@ -0,0 +1,338 @@
+package s3api
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	grpc "google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type deleteObjectEntryTestClient struct {
+	filer_pb.SeaweedFilerClient
+
+	deleteResp *filer_pb.DeleteEntryResponse
+	deleteErr  error
+	lookupResp *filer_pb.LookupDirectoryEntryResponse
+	lookupErr  error
+	updateErr  error
+
+	// deleteCallErrs scripts the RPC itself per attempt, deleteRespErrs the
+	// filer's own message: the two ways a delete fails, told apart because
+	// only the first is a transport that dropped a reply.
+	deleteCallErrs []error
+	deleteRespErrs []string
+
+	deleteAttempts int
+	deleteReqs     []*filer_pb.DeleteEntryRequest
+	deleteReq      *filer_pb.DeleteEntryRequest
+	lookupReq      *filer_pb.LookupDirectoryEntryRequest
+	updateReq      *filer_pb.UpdateEntryRequest
+}
+
+func (c *deleteObjectEntryTestClient) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest, _ ...grpc.CallOption) (*filer_pb.DeleteEntryResponse, error) {
+	attempt := c.deleteAttempts
+	c.deleteAttempts++
+	c.deleteReq = req
+	c.deleteReqs = append(c.deleteReqs, req)
+
+	if err := ctx.Err(); err != nil {
+		return nil, status.FromContextError(err).Err()
+	}
+	if attempt < len(c.deleteCallErrs) && c.deleteCallErrs[attempt] != nil {
+		return nil, c.deleteCallErrs[attempt]
+	}
+	if attempt < len(c.deleteRespErrs) && c.deleteRespErrs[attempt] != "" {
+		return &filer_pb.DeleteEntryResponse{Error: c.deleteRespErrs[attempt]}, nil
+	}
+	if c.deleteResp == nil {
+		return &filer_pb.DeleteEntryResponse{}, c.deleteErr
+	}
+	return c.deleteResp, c.deleteErr
+}
+
+func (c *deleteObjectEntryTestClient) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	c.lookupReq = req
+	if c.lookupResp == nil {
+		return &filer_pb.LookupDirectoryEntryResponse{}, c.lookupErr
+	}
+	return c.lookupResp, c.lookupErr
+}
+
+func (c *deleteObjectEntryTestClient) UpdateEntry(_ context.Context, req *filer_pb.UpdateEntryRequest, _ ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
+	c.updateReq = req
+	return &filer_pb.UpdateEntryResponse{}, c.updateErr
+}
+
+func TestDeleteObjectEntryDemotesNonEmptyDirectoryMarker(t *testing.T) {
+	client := &deleteObjectEntryTestClient{
+		deleteResp: &filer_pb.DeleteEntryResponse{
+			Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/test/photos",
+		},
+		lookupResp: &filer_pb.LookupDirectoryEntryResponse{
+			Entry: &filer_pb.Entry{
+				Name:        "photos",
+				IsDirectory: true,
+				Attributes: &filer_pb.FuseAttributes{
+					Mime:     "application/octet-stream",
+					Md5:      []byte{1, 2, 3, 4},
+					FileSize: 4,
+				},
+				Content: []byte("test"),
+				Extended: map[string][]byte{
+					s3_constants.ExtETagKey:                   []byte("etag"),
+					s3_constants.ExtAmzOwnerKey:               []byte("owner"),
+					s3_constants.AmzUserMetaPrefix + "Color":  []byte("blue"),
+					s3_constants.AmzObjectTaggingPrefix + "k": []byte("v"),
+					"xattr-keep":           []byte("keep-me"),
+					"x-seaweedfs-internal": []byte("keep-me-too"),
+				},
+			},
+		},
+	}
+
+	err := deleteObjectEntry(context.Background(), client, "/buckets/test", "photos", true, false)
+	require.NoError(t, err)
+	require.NotNil(t, client.lookupReq)
+	require.NotNil(t, client.updateReq)
+
+	updated := client.updateReq.Entry
+	require.NotNil(t, updated)
+	assert.False(t, updated.IsDirectoryKeyObject())
+	assert.Equal(t, "", updated.Attributes.Mime)
+	assert.Empty(t, updated.Attributes.Md5)
+	assert.Zero(t, updated.Attributes.FileSize)
+	assert.Nil(t, updated.Content)
+	assert.Nil(t, updated.Chunks)
+	assert.Equal(t, map[string][]byte{
+		"xattr-keep":           []byte("keep-me"),
+		"x-seaweedfs-internal": []byte("keep-me-too"),
+	}, updated.Extended)
+}
+
+// A prefix object is demoted the same way, and the mark has to go with the data:
+// the path is a plain directory again, and no longer a key of its own.
+func TestDeleteObjectEntryDemotesPrefixObject(t *testing.T) {
+	client := &deleteObjectEntryTestClient{
+		deleteResp: &filer_pb.DeleteEntryResponse{
+			Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/test/photos",
+		},
+		lookupResp: &filer_pb.LookupDirectoryEntryResponse{
+			Entry: &filer_pb.Entry{
+				Name:        "photos",
+				IsDirectory: true,
+				Attributes:  &filer_pb.FuseAttributes{},
+				Extended: map[string][]byte{
+					s3_constants.SeaweedFSPrefixObject: []byte("true"),
+					s3_constants.ExtETagKey:            []byte("etag"),
+				},
+			},
+		},
+	}
+
+	require.NoError(t, deleteObjectEntry(context.Background(), client, "/buckets/test", "photos", true, false))
+	require.NotNil(t, client.updateReq)
+
+	updated := client.updateReq.Entry
+	require.NotNil(t, updated)
+	assert.False(t, updated.IsPrefixObject())
+	assert.False(t, updated.IsDirectoryKeyObject())
+	assert.Empty(t, updated.Extended)
+}
+
+func TestDeleteObjectEntryTreatsImplicitDirectoryAsSuccessfulNoop(t *testing.T) {
+	client := &deleteObjectEntryTestClient{
+		deleteResp: &filer_pb.DeleteEntryResponse{
+			Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/test/photos",
+		},
+		lookupResp: &filer_pb.LookupDirectoryEntryResponse{
+			Entry: &filer_pb.Entry{
+				Name:        "photos",
+				IsDirectory: true,
+				Attributes:  &filer_pb.FuseAttributes{},
+			},
+		},
+	}
+
+	err := deleteObjectEntry(context.Background(), client, "/buckets/test", "photos", true, false)
+	require.NoError(t, err)
+	require.NotNil(t, client.lookupReq)
+	assert.Nil(t, client.updateReq)
+}
+
+func TestDeleteObjectEntryIgnoresConcurrentUpdateNotFound(t *testing.T) {
+	client := &deleteObjectEntryTestClient{
+		deleteResp: &filer_pb.DeleteEntryResponse{
+			Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/test/photos",
+		},
+		lookupResp: &filer_pb.LookupDirectoryEntryResponse{
+			Entry: &filer_pb.Entry{
+				Name:        "photos",
+				IsDirectory: true,
+				Attributes: &filer_pb.FuseAttributes{
+					Mime: "application/octet-stream",
+				},
+			},
+		},
+		updateErr: status.Error(codes.NotFound, "already removed"),
+	}
+
+	err := deleteObjectEntry(context.Background(), client, "/buckets/test", "photos", true, false)
+	require.NoError(t, err)
+	require.NotNil(t, client.lookupReq)
+	require.NotNil(t, client.updateReq)
+}
+
+// The key is the client's and the filer echoes it in the message it sends back,
+// so a key named after the marker must not turn a real failure into the demote
+// no-op, which would answer a failed delete with a 204.
+func TestDeleteObjectEntryIgnoresMarkerSpoofedByKey(t *testing.T) {
+	name := filer.MsgFailDelNonEmptyFolder
+	client := &deleteObjectEntryTestClient{
+		deleteResp: &filer_pb.DeleteEntryResponse{
+			Error: "delete file /buckets/test/" + name + ": filer store delete: disk full",
+		},
+	}
+
+	err := deleteObjectEntry(context.Background(), client, "/buckets/test", name, true, false)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "disk full")
+	assert.Nil(t, client.lookupReq)
+	assert.Nil(t, client.updateReq)
+}
+
+func TestDeleteObjectEntryPropagatesNonDirectoryDeleteErrors(t *testing.T) {
+	client := &deleteObjectEntryTestClient{
+		deleteErr: errors.New("boom"),
+	}
+
+	err := deleteObjectEntry(context.Background(), client, "/buckets/test", "photos", true, false)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "boom")
+	assert.Nil(t, client.lookupReq)
+	assert.Nil(t, client.updateReq)
+}
+
+// A delete is idempotent at the filer, so a reply the transport dropped is
+// reissued rather than surfaced. Issue #7204 saw the surfaced version as a 500
+// on the bucket delete, which boto3 resent and then answered NoSuchBucket;
+// issue #7224 saw it as a per-key InternalError inside a 200, which no SDK
+// retries, so the object silently stayed.
+func TestDeleteUnversionedObjectReplaysADroppedReply(t *testing.T) {
+	s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
+	client := &deleteObjectEntryTestClient{
+		deleteCallErrs: []error{status.Error(codes.Unavailable, "transport is closing"), nil},
+	}
+
+	err := s3a.deleteUnversionedObjectWithClient(testRetryCtx(10*time.Millisecond), client, "mybucket", "a/b/c.txt", false)
+
+	require.NoError(t, err)
+	assert.Equal(t, 2, client.deleteAttempts)
+	// the replay must ask for the same delete, not a differently shaped one
+	require.Len(t, client.deleteReqs, 2)
+	assert.Equal(t, client.deleteReqs[0].String(), client.deleteReqs[1].String())
+}
+
+// The filer formats the deleted path, and for a recursive delete the child it
+// stopped on, into the message it sends back, so both are the client's text.
+// Nothing here reads that text, and this pins it: the same failure under a key
+// named after a transport condition is replayed exactly as often as under a
+// plain one.
+func TestDeleteUnversionedObjectReplayIgnoresTheObjectName(t *testing.T) {
+	attemptsFor := func(key string) int {
+		s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
+		client := &deleteObjectEntryTestClient{
+			deleteRespErrs: []string{"delete file /buckets/b/" + key + ": permission denied", ""},
+		}
+		require.NoError(t, s3a.deleteUnversionedObjectWithClient(testRetryCtx(10*time.Millisecond), client, "b", key, false))
+		return client.deleteAttempts
+	}
+
+	plain := attemptsFor("plain.log")
+	for _, key := range []string{"transport.log", "unavailable.txt", "slowdown.csv", filer_pb.ErrNotFound.Error()} {
+		assert.Equal(t, plain, attemptsFor(key), "key %q must not steer the replay", key)
+	}
+}
+
+// A folder the filer refused because it still has children is an answer about
+// the tree, not a hiccup: it is handed straight to the demote path instead of
+// being replayed until the allowance runs out.
+func TestDeleteUnversionedObjectDoesNotReplayANonEmptyFolder(t *testing.T) {
+	s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
+	client := &deleteObjectEntryTestClient{
+		deleteResp: &filer_pb.DeleteEntryResponse{
+			Error: filer.MsgFailDelNonEmptyFolder + ": /buckets/b/photos",
+		},
+	}
+
+	require.NoError(t, s3a.deleteUnversionedObjectWithClient(context.Background(), client, "b", "photos", false))
+	assert.Equal(t, 1, client.deleteAttempts)
+	assert.NotNil(t, client.lookupReq, "the demote must still run")
+}
+
+// A caller that has gone away is not worth replaying for.
+func TestDeleteUnversionedObjectDoesNotReplayForACancelledCaller(t *testing.T) {
+	s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
+	client := &deleteObjectEntryTestClient{}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	err := s3a.deleteUnversionedObjectWithClient(ctx, client, "b", "k", false)
+
+	require.Error(t, err)
+	assert.Equal(t, 1, client.deleteAttempts)
+}
+
+// The client picks how many keys a multi-object delete carries, so the backoff
+// is drawn from one allowance held by the request rather than one per key.
+func TestDeleteUnversionedObjectDrawsFromTheRequestAllowance(t *testing.T) {
+	s3a := &S3ApiServer{option: &S3ApiServerOption{BucketsPath: "/buckets"}}
+	ctx := testRetryCtx(20 * time.Millisecond)
+
+	start := time.Now()
+	var attempts int
+	for i := 0; i < 50; i++ {
+		client := &deleteObjectEntryTestClient{deleteErr: status.Error(codes.Unavailable, "transport is closing")}
+		require.Error(t, s3a.deleteUnversionedObjectWithClient(ctx, client, "b", fmt.Sprintf("k%d", i), false))
+		attempts += client.deleteAttempts
+	}
+
+	assert.Less(t, time.Since(start), time.Second, "one allowance, not one per key")
+	assert.Greater(t, attempts, 50, "the allowance must still buy a replay")
+}
+
+// testRetryCtx caps the backoff so a test exercises the replay without waiting
+// out the real per-request allowance.
+func testRetryCtx(allowance time.Duration) context.Context {
+	return withFilerRetryBudget(context.Background(), allowance)
+}
+
+func TestIsRetryableFilerErr(t *testing.T) {
+	assert.False(t, isRetryableFilerErr(nil))
+	assert.True(t, isRetryableFilerErr(status.Error(codes.Unavailable, "transport is closing")))
+	assert.True(t, isRetryableFilerErr(fmt.Errorf("delete entry /b/k: %w", status.Error(codes.Unavailable, "x"))))
+	// the filer's own message carries no status, and a store error may well clear
+	assert.True(t, isRetryableFilerErr(filer.DeleteEntryError("delete file /b/k: filer store delete: disk full")))
+
+	assert.False(t, isRetryableFilerErr(filer.DeleteEntryError(filer.MsgFailDelNonEmptyFolder+": /b/photos")))
+	assert.False(t, isRetryableFilerErr(filer_pb.ErrNotFound))
+	assert.False(t, isRetryableFilerErr(status.Error(codes.NotFound, "gone")))
+	assert.False(t, isRetryableFilerErr(context.Canceled))
+	assert.False(t, isRetryableFilerErr(status.Error(codes.Canceled, "context canceled")))
+	assert.False(t, isRetryableFilerErr(status.Error(codes.DeadlineExceeded, "context deadline exceeded")))
+
+	// the message is never read, so the same code classifies the same way
+	// whatever the client named the key
+	assert.Equal(t,
+		isRetryableFilerErr(status.Error(codes.PermissionDenied, "denied for /b/plain.log")),
+		isRetryableFilerErr(status.Error(codes.PermissionDenied, "denied for /b/transport.log")))
+}