snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/s3/lifecycle/s3_lifecycle_admin_dispatch_test.go

@@ -0,0 +1,155 @@
+package lifecycle
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/stretchr/testify/require"
+)
+
+// Drives the real admin->worker dispatch path via the run endpoint and
+// asserts the dispatch reaches the filer (no dial error) and deletes
+// the backdated object. The Makefile pins filer.port.grpc off the
+// FILER_PORT+10000 convention so a raw-address forwarding regression
+// resurfaces here.
+func TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort(t *testing.T) {
+	adminEndpoint := envOr("ADMIN_ENDPOINT", defaultAdminEndpoint)
+
+	c := s3Client(t)
+	fc, fcClose := filerClient(t)
+	defer fcClose()
+
+	bucket := uniqueBucket("admin-dispatch")
+	mustCreateBucket(t, c, bucket)
+	putExpirationLifecycle(t, c, bucket, "expire/", 1)
+	const oldKey = "expire/old.txt"
+	putObject(t, c, bucket, oldKey, "old")
+	backdateMtime(t, fc, bucket, oldKey, 30)
+
+	waitForLifecycleWorkerReady(t, adminEndpoint)
+
+	payload := runLifecycleJob(t, adminEndpoint, 30)
+
+	require.GreaterOrEqual(t, jsonNumber(t, payload, "detected_count"), 1)
+	require.Equal(t, 0, jsonNumber(t, payload, "error_count"),
+		"dispatched job errored — likely filer_grpc_address was raw host:httpPort.grpcPort")
+
+	require.Eventuallyf(t, func() bool {
+		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
+			Bucket: aws.String(bucket), Key: aws.String(oldKey),
+		})
+		return err != nil
+	}, 30*time.Second, 500*time.Millisecond,
+		"expected %s/%s to be deleted after admin-dispatched lifecycle run", bucket, oldKey)
+}
+
+// A worker-dispatched pass with rules in place but nothing due: the
+// invariant is that a pass returns on its own, so the admin never has to
+// cancel it and the executor slot is free for the next one.
+//
+// Not a regression test for the wedge this scenario comes from. Before
+// the subscription was bounded a pass ended only when some write landed
+// past its boundary, and on a shared test cluster something usually
+// does — verified: the whole suite passes on the unfixed build. The
+// deterministic guards are the dailyrun unit tests. This covers the
+// worker path end-to-end and would catch a pass that hangs
+// unconditionally.
+func TestLifecycleAdminDispatchCompletesWithNothingDue(t *testing.T) {
+	adminEndpoint := envOr("ADMIN_ENDPOINT", defaultAdminEndpoint)
+
+	c := s3Client(t)
+	bucket := uniqueBucket("admin-idle")
+	mustCreateBucket(t, c, bucket)
+	putExpirationLifecycle(t, c, bucket, "expire/", 1)
+	// Deliberately not backdated: the rule matches the prefix but nothing
+	// is due, so the pass dispatches nothing.
+	putObject(t, c, bucket, "expire/fresh.txt", "fresh")
+
+	waitForLifecycleWorkerReady(t, adminEndpoint)
+
+	// Let the writes above settle below the pass boundary, or one of them
+	// ends the pass and the wedge is masked.
+	time.Sleep(2 * time.Second)
+
+	payload := runLifecycleJob(t, adminEndpoint, 30)
+
+	require.Equal(t, 0, jsonNumber(t, payload, "canceled_count"),
+		"pass did not return on its own; the admin cancelled it at the run timeout")
+	require.GreaterOrEqual(t, jsonNumber(t, payload, "success_count"), 1,
+		"expected the dispatched pass to complete")
+	require.Equal(t, 0, jsonNumber(t, payload, "skipped_active_count"),
+		"a previous pass still held the executor slot")
+}
+
+// runLifecycleJob drives the admin's run endpoint and returns its decoded
+// response. timeoutSeconds bounds the run admin-side: a pass that never
+// returns comes back as canceled_count=1 after that long.
+func runLifecycleJob(t *testing.T, adminEndpoint string, timeoutSeconds int) map[string]any {
+	t.Helper()
+	body, err := json.Marshal(map[string]any{"timeout_seconds": timeoutSeconds})
+	require.NoError(t, err)
+	req, err := http.NewRequestWithContext(
+		context.Background(), http.MethodPost,
+		adminEndpoint+"/api/plugin/job-types/s3_lifecycle/run",
+		bytes.NewReader(body),
+	)
+	require.NoError(t, err)
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	var payload map[string]any
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&payload))
+	t.Logf("admin /api/plugin/job-types/s3_lifecycle/run response: %v", payload)
+	require.Equal(t, http.StatusOK, resp.StatusCode, "admin run endpoint failed: %v", payload)
+	return payload
+}
+
+func jsonNumber(t *testing.T, payload map[string]any, key string) int {
+	t.Helper()
+	raw, ok := payload[key]
+	require.Truef(t, ok, "response missing key %q: %v", key, payload)
+	switch v := raw.(type) {
+	case float64:
+		return int(v)
+	case json.Number:
+		n, err := v.Int64()
+		require.NoErrorf(t, err, "key %q is not an int: %v", key, raw)
+		return int(n)
+	default:
+		t.Fatalf("key %q has unexpected type %T (%v)", key, raw, raw)
+		return 0
+	}
+}
+
+func waitForLifecycleWorkerReady(t *testing.T, adminEndpoint string) {
+	t.Helper()
+	deadline := time.Now().Add(30 * time.Second)
+	for time.Now().Before(deadline) {
+		req, err := http.NewRequest(http.MethodGet, adminEndpoint+"/api/plugin/scheduler-states", nil)
+		require.NoError(t, err)
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			time.Sleep(250 * time.Millisecond)
+			continue
+		}
+		var payload any
+		_ = json.NewDecoder(resp.Body).Decode(&payload)
+		resp.Body.Close()
+		if resp.StatusCode == http.StatusOK && strings.Contains(fmt.Sprintf("%v", payload), "s3_lifecycle") {
+			return
+		}
+		time.Sleep(250 * time.Millisecond)
+	}
+	t.Fatalf("admin never reported an s3_lifecycle-capable worker")
+}