snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/s3api/s3api_versioning_reconciler_test.go

@@ -0,0 +1,245 @@
+package s3api
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// TestVersionsHealQueue_DedupOnEnqueue ensures multiple enqueues of the
+// same bucket/object collapse into a single pending entry, so a hot
+// failure path doesn't bloat the queue.
+func TestVersionsHealQueue_DedupOnEnqueue(t *testing.T) {
+	q := newVersionsHealQueue()
+	for i := 0; i < 5; i++ {
+		q.Enqueue("b", "obj")
+	}
+	assert.Equal(t, 1, q.Len(), "duplicate enqueues collapse")
+}
+
+// TestVersionsHealQueue_CapacityCap ensures the queue refuses growth
+// past the static cap and logs at V(1) instead of OOM-ing.
+func TestVersionsHealQueue_CapacityCap(t *testing.T) {
+	q := newVersionsHealQueue()
+	for i := 0; i < versionsHealQueueCapacity+50; i++ {
+		q.Enqueue("b", string(rune(i)))
+	}
+	assert.Equal(t, versionsHealQueueCapacity, q.Len(), "queue clamps at capacity")
+}
+
+// TestVersionsHealQueue_PopReadyOnlyDueItems checks that nextRetry
+// gating keeps not-yet-ready candidates in the queue.
+func TestVersionsHealQueue_PopReadyOnlyDueItems(t *testing.T) {
+	q := newVersionsHealQueue()
+	q.Enqueue("b", "due")
+
+	// Inject a deferred candidate directly so we control its nextRetry.
+	q.pending[versionsHealKey("b", "later")] = &versionsHealCandidate{
+		bucket:    "b",
+		object:    "later",
+		enqueued:  time.Now(),
+		nextRetry: time.Now().Add(10 * time.Minute),
+	}
+
+	due := q.popReady(time.Now())
+	require.Len(t, due, 1, "only the due candidate pops")
+	assert.Equal(t, "due", due[0].object)
+	assert.Equal(t, 1, q.Len(), "deferred candidate still queued")
+}
+
+// TestVersionsHealQueue_RequeueWithBackoff verifies failed candidates
+// re-enter the queue with an extended nextRetry.
+func TestVersionsHealQueue_RequeueWithBackoff(t *testing.T) {
+	q := newVersionsHealQueue()
+	c := &versionsHealCandidate{bucket: "b", object: "obj", attempts: 1}
+
+	q.requeue(c, 500*time.Millisecond)
+	assert.Equal(t, 1, q.Len())
+
+	now := time.Now()
+	due := q.popReady(now)
+	assert.Empty(t, due, "not yet due immediately after requeue")
+
+	due = q.popReady(now.Add(time.Second))
+	assert.Len(t, due, 1, "due after backoff window passes")
+}
+
+// TestVersionsHealQueue_GiveUpAfterMaxAttempts ensures we don't loop
+// forever against a deterministically broken state.
+func TestVersionsHealQueue_GiveUpAfterMaxAttempts(t *testing.T) {
+	q := newVersionsHealQueue()
+	c := &versionsHealCandidate{bucket: "b", object: "obj", attempts: versionsHealMaxRetries}
+	q.requeue(c, time.Millisecond)
+	assert.Equal(t, 0, q.Len(), "candidate at max attempts is dropped, read-path heal still covers it")
+}
+
+// TestRetryFilerOp_SucceedsBeforeExhaustion confirms a flaky op that
+// eventually succeeds is reported as success without surfacing prior
+// errors.
+func TestRetryFilerOp_SucceedsBeforeExhaustion(t *testing.T) {
+	calls := 0
+	err := retryFilerOp(context.Background(), "test", func() error {
+		calls++
+		if calls < 3 {
+			return errors.New("transient")
+		}
+		return nil
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, 3, calls, "stops calling once the op succeeds")
+}
+
+// TestRetryFilerOp_PropagatesAfterExhaustion confirms a deterministic
+// failure is wrapped with the attempt count so operators can tell at
+// a glance whether the underlying issue is transient.
+func TestRetryFilerOp_PropagatesAfterExhaustion(t *testing.T) {
+	calls := 0
+	err := retryFilerOp(context.Background(), "test", func() error {
+		calls++
+		return errors.New("permanent")
+	})
+	require.Error(t, err)
+	assert.Equal(t, updateLatestRetryAttempts, calls, "ran the full retry budget")
+	assert.Contains(t, err.Error(), "exhausted")
+	assert.Contains(t, err.Error(), "permanent", "underlying error preserved")
+}
+
+// TestRetryFilerOp_ContextCancelInterruptsBackoff confirms that a ctx
+// canceled mid-backoff returns ctx.Err() immediately instead of
+// blocking until the worst-case ~6.3s retry budget elapses. Server
+// shutdown / client disconnect relies on this to drain in-flight
+// retries promptly.
+func TestRetryFilerOp_ContextCancelInterruptsBackoff(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	// Cancel after the first failed attempt has scheduled its backoff
+	// but well before the timer fires.
+	go func() {
+		time.Sleep(30 * time.Millisecond)
+		cancel()
+	}()
+	calls := 0
+	start := time.Now()
+	err := retryFilerOp(ctx, "test", func() error {
+		calls++
+		return errors.New("transient")
+	})
+	elapsed := time.Since(start)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, context.Canceled, "must surface ctx.Err verbatim")
+	assert.Less(t, elapsed, 500*time.Millisecond, "ctx cancel must interrupt the backoff sleep")
+	assert.Less(t, calls, updateLatestRetryAttempts, "ctx cancel must short-circuit before exhausting the retry budget")
+}
+
+// TestRetryFilerOp_TerminalErrorsShortCircuit confirms that errors which
+// won't change on retry (NotFound, context cancellation) are returned
+// immediately and unwrapped, without burning the retry budget.
+func TestRetryFilerOp_TerminalErrorsShortCircuit(t *testing.T) {
+	cases := []struct {
+		name string
+		err  error
+	}{
+		{"filer_pb.ErrNotFound", filer_pb.ErrNotFound},
+		{"wrapped filer_pb.ErrNotFound", fmt.Errorf("wrap: %w", filer_pb.ErrNotFound)},
+		{"grpc NotFound status", status.Error(codes.NotFound, "missing")},
+		{"context canceled", context.Canceled},
+		{"context deadline exceeded", context.DeadlineExceeded},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			start := time.Now()
+			calls := 0
+			err := retryFilerOp(context.Background(), "test", func() error {
+				calls++
+				return tc.err
+			})
+			elapsed := time.Since(start)
+			require.Error(t, err)
+			assert.Equal(t, 1, calls, "terminal error must not be retried")
+			assert.Less(t, elapsed, 50*time.Millisecond, "terminal error must not delay")
+			assert.NotContains(t, err.Error(), "exhausted", "terminal error must not be wrapped with retry-budget prefix")
+		})
+	}
+}
+
+// TestFilerRetryBudget_ClampsToWhatIsLeft covers the allowance arithmetic:
+// a reservation larger than the remainder is trimmed, and once nothing is
+// left the caller is told to stop rather than handed a zero wait.
+func TestFilerRetryBudget_ClampsToWhatIsLeft(t *testing.T) {
+	b := &filerRetryBudget{remaining: 150 * time.Millisecond}
+
+	granted, ok := b.take(100 * time.Millisecond)
+	require.True(t, ok)
+	assert.Equal(t, 100*time.Millisecond, granted)
+
+	granted, ok = b.take(200 * time.Millisecond)
+	require.True(t, ok)
+	assert.Equal(t, 50*time.Millisecond, granted, "trimmed to the remainder")
+
+	_, ok = b.take(time.Millisecond)
+	assert.False(t, ok, "allowance spent")
+}
+
+// TestFilerRetryRequestBudget_MatchesOneOpWorstCase pins the request-wide
+// allowance to what a single retryFilerOp can sleep for, so a batch delete
+// adds the wait of one key rather than one per key.
+func TestFilerRetryRequestBudget_MatchesOneOpWorstCase(t *testing.T) {
+	var singleOp time.Duration
+	for attempt := 1; attempt < updateLatestRetryAttempts; attempt++ {
+		singleOp += retryFilerBackoff(attempt)
+	}
+	assert.Equal(t, singleOp, filerRetryRequestBudget)
+	assert.Equal(t, 3100*time.Millisecond, filerRetryRequestBudget)
+}
+
+// TestRetryFilerOp_SharedBudgetAcrossBatch is the batch-delete shape: many
+// keys, each driving its own retryFilerOp against a filer that always fails
+// retryably. Without a shared allowance every key pays the full per-op
+// backoff, so the wait scales with a key count the client picks. The
+// allowance here is deliberately small so the test never sleeps for the
+// production budget.
+func TestRetryFilerOp_SharedBudgetAcrossBatch(t *testing.T) {
+	const keys = 200
+	const allowance = 250 * time.Millisecond
+
+	ctx := withFilerRetryBudget(context.Background(), allowance)
+	calls := 0
+	start := time.Now()
+	for i := 0; i < keys; i++ {
+		err := retryFilerOp(ctx, "test", func() error {
+			calls++
+			return errors.New("transient")
+		})
+		require.Error(t, err)
+	}
+	elapsed := time.Since(start)
+
+	assert.LessOrEqual(t, calls, keys+updateLatestRetryAttempts, "only the keys that fit the allowance retry")
+	assert.Less(t, elapsed, 2*allowance, "the whole batch stays inside one allowance")
+}
+
+// TestRetryFilerOp_SpentBudgetStopsAfterFirstAttempt confirms a key that
+// arrives after the allowance is gone fails immediately, reporting why, and
+// still gets its one real attempt at the filer.
+func TestRetryFilerOp_SpentBudgetStopsAfterFirstAttempt(t *testing.T) {
+	ctx := withFilerRetryBudget(context.Background(), 0)
+	calls := 0
+	start := time.Now()
+	err := retryFilerOp(ctx, "test", func() error {
+		calls++
+		return errors.New("transient")
+	})
+
+	require.Error(t, err)
+	assert.Equal(t, 1, calls, "the op still runs once")
+	assert.Less(t, time.Since(start), 50*time.Millisecond, "no backoff once the allowance is spent")
+	assert.Contains(t, err.Error(), "retry allowance spent")
+	assert.Contains(t, err.Error(), "transient", "underlying error preserved")
+}