upstream-seaweedfs/test/s3/lifecycle/s3_lifecycle_admin_dispatch_test.go

package lifecycle

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/stretchr/testify/require"
)

// Drives the real admin->worker dispatch path via the run endpoint and
// asserts the dispatch reaches the filer (no dial error) and deletes
// the backdated object. The Makefile pins filer.port.grpc off the
// FILER_PORT+10000 convention so a raw-address forwarding regression
// resurfaces here.
func TestLifecycleAdminDispatchSucceedsWithCustomFilerGrpcPort(t *testing.T) {
	adminEndpoint := envOr("ADMIN_ENDPOINT", defaultAdminEndpoint)

	c := s3Client(t)
	fc, fcClose := filerClient(t)
	defer fcClose()

	bucket := uniqueBucket("admin-dispatch")
	mustCreateBucket(t, c, bucket)
	putExpirationLifecycle(t, c, bucket, "expire/", 1)
	const oldKey = "expire/old.txt"
	putObject(t, c, bucket, oldKey, "old")
	backdateMtime(t, fc, bucket, oldKey, 30)

	waitForLifecycleWorkerReady(t, adminEndpoint)

	payload := runLifecycleJob(t, adminEndpoint, 30)

	require.GreaterOrEqual(t, jsonNumber(t, payload, "detected_count"), 1)
	require.Equal(t, 0, jsonNumber(t, payload, "error_count"),
		"dispatched job errored — likely filer_grpc_address was raw host:httpPort.grpcPort")

	require.Eventuallyf(t, func() bool {
		_, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{
			Bucket: aws.String(bucket), Key: aws.String(oldKey),
		})
		return err != nil
	}, 30*time.Second, 500*time.Millisecond,
		"expected %s/%s to be deleted after admin-dispatched lifecycle run", bucket, oldKey)
}

// A worker-dispatched pass with rules in place but nothing due: the
// invariant is that a pass returns on its own, so the admin never has to
// cancel it and the executor slot is free for the next one.
//
// Not a regression test for the wedge this scenario comes from. Before
// the subscription was bounded a pass ended only when some write landed
// past its boundary, and on a shared test cluster something usually
// does — verified: the whole suite passes on the unfixed build. The
// deterministic guards are the dailyrun unit tests. This covers the
// worker path end-to-end and would catch a pass that hangs
// unconditionally.
func TestLifecycleAdminDispatchCompletesWithNothingDue(t *testing.T) {
	adminEndpoint := envOr("ADMIN_ENDPOINT", defaultAdminEndpoint)

	c := s3Client(t)
	bucket := uniqueBucket("admin-idle")
	mustCreateBucket(t, c, bucket)
	putExpirationLifecycle(t, c, bucket, "expire/", 1)
	// Deliberately not backdated: the rule matches the prefix but nothing
	// is due, so the pass dispatches nothing.
	putObject(t, c, bucket, "expire/fresh.txt", "fresh")

	waitForLifecycleWorkerReady(t, adminEndpoint)

	// Let the writes above settle below the pass boundary, or one of them
	// ends the pass and the wedge is masked.
	time.Sleep(2 * time.Second)

	payload := runLifecycleJob(t, adminEndpoint, 30)

	require.Equal(t, 0, jsonNumber(t, payload, "canceled_count"),
		"pass did not return on its own; the admin cancelled it at the run timeout")
	require.GreaterOrEqual(t, jsonNumber(t, payload, "success_count"), 1,
		"expected the dispatched pass to complete")
	require.Equal(t, 0, jsonNumber(t, payload, "skipped_active_count"),
		"a previous pass still held the executor slot")
}

// runLifecycleJob drives the admin's run endpoint and returns its decoded
// response. timeoutSeconds bounds the run admin-side: a pass that never
// returns comes back as canceled_count=1 after that long.
func runLifecycleJob(t *testing.T, adminEndpoint string, timeoutSeconds int) map[string]any {
	t.Helper()
	body, err := json.Marshal(map[string]any{"timeout_seconds": timeoutSeconds})
	require.NoError(t, err)
	req, err := http.NewRequestWithContext(
		context.Background(), http.MethodPost,
		adminEndpoint+"/api/plugin/job-types/s3_lifecycle/run",
		bytes.NewReader(body),
	)
	require.NoError(t, err)
	req.Header.Set("Content-Type", "application/json")

	resp, err := http.DefaultClient.Do(req)
	require.NoError(t, err)
	defer resp.Body.Close()

	var payload map[string]any
	require.NoError(t, json.NewDecoder(resp.Body).Decode(&payload))
	t.Logf("admin /api/plugin/job-types/s3_lifecycle/run response: %v", payload)
	require.Equal(t, http.StatusOK, resp.StatusCode, "admin run endpoint failed: %v", payload)
	return payload
}

func jsonNumber(t *testing.T, payload map[string]any, key string) int {
	t.Helper()
	raw, ok := payload[key]
	require.Truef(t, ok, "response missing key %q: %v", key, payload)
	switch v := raw.(type) {
	case float64:
		return int(v)
	case json.Number:
		n, err := v.Int64()
		require.NoErrorf(t, err, "key %q is not an int: %v", key, raw)
		return int(n)
	default:
		t.Fatalf("key %q has unexpected type %T (%v)", key, raw, raw)
		return 0
	}
}

func waitForLifecycleWorkerReady(t *testing.T, adminEndpoint string) {
	t.Helper()
	deadline := time.Now().Add(30 * time.Second)
	for time.Now().Before(deadline) {
		req, err := http.NewRequest(http.MethodGet, adminEndpoint+"/api/plugin/scheduler-states", nil)
		require.NoError(t, err)
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			time.Sleep(250 * time.Millisecond)
			continue
		}
		var payload any
		_ = json.NewDecoder(resp.Body).Decode(&payload)
		resp.Body.Close()
		if resp.StatusCode == http.StatusOK && strings.Contains(fmt.Sprintf("%v", payload), "s3_lifecycle") {
			return
		}
		time.Sleep(250 * time.Millisecond)
	}
	t.Fatalf("admin never reported an s3_lifecycle-capable worker")
}