snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/s3api/s3lifecycle/scheduler/configload.go

@@ -0,0 +1,114 @@
+package scheduler
+
+import (
+	"context"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/lifecycle_xml"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
+)
+
+// The bucket lifecycle configuration lives here rather than in weed/s3api so
+// callers outside the gateway (the admin dashboard, the scheduler itself) and
+// the gateway share one definition of the extended-attribute keys and the
+// size cap. weed/s3api aliases these; nothing redeclares the values.
+const (
+	BucketLifecycleConfigurationXMLKey            = "s3-bucket-lifecycle-configuration-xml"
+	BucketLifecycleTransitionMinimumObjectSizeKey = "s3-bucket-lifecycle-transition-default-minimum-object-size"
+
+	// MaxBucketLifecycleConfigurationSize caps the serialized XML, mirroring
+	// the S3 API's limit.
+	MaxBucketLifecycleConfigurationSize = 1 << 20
+)
+
+// ParseError is returned alongside successfully-loaded inputs so callers can
+// surface malformed bucket configs rather than silently dropping them.
+type ParseError struct {
+	Bucket string
+	Err    error
+}
+
+// LoadCompileInputs walks the buckets directory and returns one
+// engine.CompileInput per bucket that carries a non-empty lifecycle
+// configuration. Pagination loops with startFrom so clusters with more
+// than one page of buckets don't drop the tail.
+func LoadCompileInputs(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketsPath string) ([]engine.CompileInput, []ParseError, error) {
+	var (
+		inputs      []engine.CompileInput
+		parseErrors []ParseError
+		startFrom   string
+	)
+	const pageSize uint32 = 1024
+	for {
+		pageCount := 0
+		var lastName string
+		err := filer_pb.SeaweedList(ctx, client, bucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
+			pageCount++
+			lastName = entry.Name
+			if !entry.IsDirectory {
+				return nil
+			}
+			xmlBytes, ok := entry.Extended[BucketLifecycleConfigurationXMLKey]
+			if !ok || len(xmlBytes) == 0 {
+				return nil
+			}
+			rules, err := lifecycle_xml.ParseCanonical(xmlBytes)
+			if err != nil {
+				parseErrors = append(parseErrors, ParseError{Bucket: entry.Name, Err: err})
+				return nil
+			}
+			if len(rules) == 0 {
+				return nil
+			}
+			inputs = append(inputs, engine.CompileInput{
+				Bucket:    entry.Name,
+				Rules:     rules,
+				Versioned: IsBucketVersioned(entry),
+			})
+			return nil
+		}, startFrom, false, pageSize)
+		if err != nil {
+			return nil, nil, err
+		}
+		if uint32(pageCount) < pageSize {
+			break
+		}
+		startFrom = lastName
+	}
+	return inputs, parseErrors, nil
+}
+
+// IsBucketVersioned returns true when the bucket's filer entry has the
+// versioning extended attribute set to Enabled or Suspended.
+func IsBucketVersioned(entry *filer_pb.Entry) bool {
+	v, ok := entry.Extended[s3_constants.ExtVersioningKey]
+	if !ok {
+		return false
+	}
+	s := strings.ToLower(strings.TrimSpace(string(v)))
+	return s == "enabled" || s == "suspended"
+}
+
+// AllActivePriorStates seeds every compiled action as bootstrap-complete +
+// event-driven so the scheduler dispatches whatever fires immediately.
+// Production bootstrap walks set this incrementally per bucket; the
+// scheduler runs out-of-band of that flow for now.
+func AllActivePriorStates(inputs []engine.CompileInput) map[s3lifecycle.ActionKey]engine.PriorState {
+	prior := map[s3lifecycle.ActionKey]engine.PriorState{}
+	for _, in := range inputs {
+		for _, rule := range in.Rules {
+			hash := s3lifecycle.RuleHash(rule)
+			for _, kind := range s3lifecycle.RuleActionKinds(rule) {
+				key := s3lifecycle.ActionKey{Bucket: in.Bucket, RuleHash: hash, ActionKind: kind}
+				prior[key] = engine.PriorState{
+					BootstrapComplete: true,
+					Mode:              engine.ModeEventDriven,
+				}
+			}
+		}
+	}
+	return prior
+}

weed/s3api/s3lifecycle/scheduler/configload_test.go

@@ -0,0 +1,236 @@
+package scheduler
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Tests for LoadCompileInputs / IsBucketVersioned / AllActivePriorStates.
+// LoadCompileInputs is the bridge between the filer's bucket directory
+// and the engine snapshot the scheduler compiles every refresh —
+// pagination, parse-error surfacing, and the empty/missing-XML skip
+// semantics matter because a missed bucket silently disables lifecycle
+// for that prefix until the next refresh.
+
+const testBucketsRoot = "/buckets"
+
+// minimalLifecycleXML is the smallest XML that ParseCanonical accepts as
+// a non-empty rule set; the actual rule details are exercised by the
+// lifecycle_xml package's own tests.
+const minimalLifecycleXML = `<LifecycleConfiguration>
+  <Rule>
+    <ID>r1</ID>
+    <Status>Enabled</Status>
+    <Filter><Prefix>logs/</Prefix></Filter>
+    <Expiration><Days>30</Days></Expiration>
+  </Rule>
+</LifecycleConfiguration>`
+
+func bucketEntry(name string, extended map[string][]byte) *filer_pb.Entry {
+	return dirEntry(name, extended)
+}
+
+func TestIsBucketVersioned_MissingExtendedReturnsFalse(t *testing.T) {
+	assert.False(t, IsBucketVersioned(&filer_pb.Entry{}))
+	assert.False(t, IsBucketVersioned(&filer_pb.Entry{Extended: map[string][]byte{}}))
+}
+
+func TestIsBucketVersioned_AcceptsEnabledAndSuspended(t *testing.T) {
+	// Casing and surrounding whitespace are normalized so the worker
+	// tolerates whatever casing the writer chose.
+	for _, raw := range []string{"Enabled", "ENABLED", "enabled", "  enabled  ", "Suspended", "suspended"} {
+		t.Run(raw, func(t *testing.T) {
+			e := &filer_pb.Entry{Extended: map[string][]byte{s3_constants.ExtVersioningKey: []byte(raw)}}
+			assert.True(t, IsBucketVersioned(e), "value %q must classify as versioned", raw)
+		})
+	}
+}
+
+func TestIsBucketVersioned_RejectsOtherValues(t *testing.T) {
+	for _, raw := range []string{"", "Disabled", "Off", "true", "1", "garbage"} {
+		t.Run(raw, func(t *testing.T) {
+			e := &filer_pb.Entry{Extended: map[string][]byte{s3_constants.ExtVersioningKey: []byte(raw)}}
+			assert.False(t, IsBucketVersioned(e), "value %q must NOT classify as versioned", raw)
+		})
+	}
+}
+
+func TestAllActivePriorStates_EmptyInputs(t *testing.T) {
+	assert.Empty(t, AllActivePriorStates(nil))
+	assert.Empty(t, AllActivePriorStates([]engine.CompileInput{}))
+}
+
+func TestAllActivePriorStates_SeedsAllActionsAsActive(t *testing.T) {
+	// Every (bucket, ruleHash, actionKind) tuple from the inputs must
+	// receive a PriorState with BootstrapComplete=true and
+	// Mode=ModeEventDriven so the scheduler dispatches matched events
+	// immediately rather than waiting for a per-action bootstrap walk.
+	rule := &s3lifecycle.Rule{
+		ID:             "r1",
+		Status:         s3lifecycle.StatusEnabled,
+		ExpirationDays: 30,
+	}
+	inputs := []engine.CompileInput{{Bucket: "b1", Rules: []*s3lifecycle.Rule{rule}}}
+	prior := AllActivePriorStates(inputs)
+
+	hash := s3lifecycle.RuleHash(rule)
+	kinds := s3lifecycle.RuleActionKinds(rule)
+	require.NotEmpty(t, kinds, "expiration-days rule must yield at least one action kind")
+	for _, kind := range kinds {
+		key := s3lifecycle.ActionKey{Bucket: "b1", RuleHash: hash, ActionKind: kind}
+		state, ok := prior[key]
+		require.True(t, ok, "prior state missing for action kind %v", kind)
+		assert.True(t, state.BootstrapComplete, "bootstrap complete for kind %v", kind)
+		assert.Equal(t, engine.ModeEventDriven, state.Mode, "mode for kind %v", kind)
+	}
+	assert.Len(t, prior, len(kinds), "exactly one prior-state entry per action kind")
+}
+
+func TestAllActivePriorStates_KeysSeparatedByBucket(t *testing.T) {
+	// Two buckets sharing identical rules must still get distinct
+	// PriorState entries, otherwise refresh would conflate their state.
+	rule := &s3lifecycle.Rule{ID: "r1", Status: s3lifecycle.StatusEnabled, ExpirationDays: 30}
+	inputs := []engine.CompileInput{
+		{Bucket: "alpha", Rules: []*s3lifecycle.Rule{rule}},
+		{Bucket: "beta", Rules: []*s3lifecycle.Rule{rule}},
+	}
+	prior := AllActivePriorStates(inputs)
+	hash := s3lifecycle.RuleHash(rule)
+	for _, kind := range s3lifecycle.RuleActionKinds(rule) {
+		assert.Contains(t, prior, s3lifecycle.ActionKey{Bucket: "alpha", RuleHash: hash, ActionKind: kind})
+		assert.Contains(t, prior, s3lifecycle.ActionKey{Bucket: "beta", RuleHash: hash, ActionKind: kind})
+	}
+}
+
+func TestLoadCompileInputs_EmptyBucketDir(t *testing.T) {
+	// No buckets at all -> no inputs, no parse errors, no transport error.
+	client := &fakeFilerClient{tree: map[string][]*filer_pb.Entry{testBucketsRoot: {}}}
+	inputs, perr, err := LoadCompileInputs(context.Background(), client, testBucketsRoot)
+	require.NoError(t, err)
+	assert.Empty(t, inputs)
+	assert.Empty(t, perr)
+}
+
+func TestLoadCompileInputs_FilesAtBucketLevelSkipped(t *testing.T) {
+	// A bare file at the buckets root isn't a bucket; the loader must
+	// skip it without touching the parser.
+	client := &fakeFilerClient{tree: map[string][]*filer_pb.Entry{
+		testBucketsRoot: {fileEntry("stray.txt")},
+	}}
+	inputs, perr, err := LoadCompileInputs(context.Background(), client, testBucketsRoot)
+	require.NoError(t, err)
+	assert.Empty(t, inputs)
+	assert.Empty(t, perr)
+}
+
+func TestLoadCompileInputs_BucketWithoutLifecycleConfigSkipped(t *testing.T) {
+	// Buckets that don't carry the lifecycle XML extended attribute are
+	// silently skipped — operators may have many buckets without rules.
+	client := &fakeFilerClient{tree: map[string][]*filer_pb.Entry{
+		testBucketsRoot: {bucketEntry("b1", nil)},
+	}}
+	inputs, perr, err := LoadCompileInputs(context.Background(), client, testBucketsRoot)
+	require.NoError(t, err)
+	assert.Empty(t, inputs)
+	assert.Empty(t, perr)
+}
+
+func TestLoadCompileInputs_BucketWithEmptyLifecycleXMLSkipped(t *testing.T) {
+	// Empty bytes under the lifecycle key shouldn't reach the parser.
+	client := &fakeFilerClient{tree: map[string][]*filer_pb.Entry{
+		testBucketsRoot: {bucketEntry("b1", map[string][]byte{
+			BucketLifecycleConfigurationXMLKey: nil,
+		})},
+	}}
+	inputs, perr, err := LoadCompileInputs(context.Background(), client, testBucketsRoot)
+	require.NoError(t, err)
+	assert.Empty(t, inputs)
+	assert.Empty(t, perr)
+}
+
+func TestLoadCompileInputs_ValidConfigBecomesInput(t *testing.T) {
+	client := &fakeFilerClient{tree: map[string][]*filer_pb.Entry{
+		testBucketsRoot: {bucketEntry("b1", map[string][]byte{
+			BucketLifecycleConfigurationXMLKey: []byte(minimalLifecycleXML),
+		})},
+	}}
+	inputs, perr, err := LoadCompileInputs(context.Background(), client, testBucketsRoot)
+	require.NoError(t, err)
+	assert.Empty(t, perr)
+	require.Len(t, inputs, 1)
+	assert.Equal(t, "b1", inputs[0].Bucket)
+	assert.False(t, inputs[0].Versioned, "no versioning attr -> not versioned")
+	assert.NotEmpty(t, inputs[0].Rules)
+}
+
+func TestLoadCompileInputs_VersioningAttrPropagates(t *testing.T) {
+	client := &fakeFilerClient{tree: map[string][]*filer_pb.Entry{
+		testBucketsRoot: {bucketEntry("b1", map[string][]byte{
+			BucketLifecycleConfigurationXMLKey: []byte(minimalLifecycleXML),
+			s3_constants.ExtVersioningKey:      []byte("Enabled"),
+		})},
+	}}
+	inputs, perr, err := LoadCompileInputs(context.Background(), client, testBucketsRoot)
+	require.NoError(t, err)
+	assert.Empty(t, perr)
+	require.Len(t, inputs, 1)
+	assert.True(t, inputs[0].Versioned)
+}
+
+func TestLoadCompileInputs_MalformedXMLBecomesParseError(t *testing.T) {
+	// Malformed XML must not abort the whole walk — surfacing it as a
+	// ParseError lets operators see the bucket-level failure while
+	// other buckets keep loading.
+	client := &fakeFilerClient{tree: map[string][]*filer_pb.Entry{
+		testBucketsRoot: {
+			bucketEntry("bad", map[string][]byte{
+				BucketLifecycleConfigurationXMLKey: []byte("<not-valid-xml"),
+			}),
+			bucketEntry("good", map[string][]byte{
+				BucketLifecycleConfigurationXMLKey: []byte(minimalLifecycleXML),
+			}),
+		},
+	}}
+	inputs, perr, err := LoadCompileInputs(context.Background(), client, testBucketsRoot)
+	require.NoError(t, err)
+	require.Len(t, perr, 1)
+	assert.Equal(t, "bad", perr[0].Bucket)
+	assert.Error(t, perr[0].Err)
+
+	require.Len(t, inputs, 1)
+	assert.Equal(t, "good", inputs[0].Bucket)
+}
+
+func TestLoadCompileInputs_PaginatesAcrossPages(t *testing.T) {
+	// pageSize is 1024 inside LoadCompileInputs; populate just over the
+	// boundary to force a second listing call. Every bucket has the same
+	// minimal config so the input count must equal the bucket count.
+	const total = 1030
+	entries := make([]*filer_pb.Entry, 0, total)
+	for i := 0; i < total; i++ {
+		entries = append(entries, bucketEntry(fmt.Sprintf("b%05d", i), map[string][]byte{
+			BucketLifecycleConfigurationXMLKey: []byte(minimalLifecycleXML),
+		}))
+	}
+	client := &fakeFilerClient{tree: map[string][]*filer_pb.Entry{
+		testBucketsRoot: entries,
+	}}
+	inputs, perr, err := LoadCompileInputs(context.Background(), client, testBucketsRoot)
+	require.NoError(t, err)
+	assert.Empty(t, perr)
+	require.Len(t, inputs, total, "pagination must surface every bucket")
+	// Order is preserved across pages, and the page boundary at 1023/1024
+	// (pageSize is 1024) must not skip or duplicate entries.
+	assert.Equal(t, "b00000", inputs[0].Bucket)
+	assert.Equal(t, "b01023", inputs[1023].Bucket, "last entry of first page")
+	assert.Equal(t, "b01024", inputs[1024].Bucket, "first entry of second page")
+	assert.Equal(t, fmt.Sprintf("b%05d", total-1), inputs[total-1].Bucket)
+}

weed/s3api/s3lifecycle/scheduler/testhelpers_test.go

@@ -0,0 +1,119 @@
+package scheduler
+
+import (
+	"context"
+	"io"
+	"sort"
+	"sync"
+	"sync/atomic"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+// fakeListStream implements grpc.ServerStreamingClient[filer_pb.ListEntriesResponse]
+// for configload tests.
+type fakeListStream struct {
+	responses []*filer_pb.ListEntriesResponse
+	index     int
+	ctx       context.Context
+}
+
+func (s *fakeListStream) Recv() (*filer_pb.ListEntriesResponse, error) {
+	if s.ctx != nil {
+		if err := s.ctx.Err(); err != nil {
+			return nil, err
+		}
+	}
+	if s.index >= len(s.responses) {
+		return nil, io.EOF
+	}
+	r := s.responses[s.index]
+	s.index++
+	return r, nil
+}
+
+func (s *fakeListStream) Header() (metadata.MD, error) { return metadata.MD{}, nil }
+func (s *fakeListStream) Trailer() metadata.MD         { return metadata.MD{} }
+func (s *fakeListStream) CloseSend() error             { return nil }
+func (s *fakeListStream) Context() context.Context {
+	if s.ctx != nil {
+		return s.ctx
+	}
+	return context.Background()
+}
+func (s *fakeListStream) SendMsg(any) error { return nil }
+func (s *fakeListStream) RecvMsg(any) error { return nil }
+
+// fakeFilerClient is the in-memory filer used by configload tests.
+type fakeFilerClient struct {
+	filer_pb.SeaweedFilerClient
+
+	mu      sync.Mutex
+	tree    map[string][]*filer_pb.Entry
+	listed  []string
+	listedN int32
+}
+
+func (c *fakeFilerClient) LookupDirectoryEntry(_ context.Context, in *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	for _, e := range c.tree[in.Directory] {
+		if e != nil && e.Name == in.Name {
+			return &filer_pb.LookupDirectoryEntryResponse{Entry: e}, nil
+		}
+	}
+	return nil, filer_pb.ErrNotFound
+}
+
+func (c *fakeFilerClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, _ ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
+	c.mu.Lock()
+	c.listed = append(c.listed, in.Directory)
+	src := c.tree[in.Directory]
+	c.mu.Unlock()
+	atomic.AddInt32(&c.listedN, 1)
+
+	filtered := make([]*filer_pb.Entry, 0, len(src))
+	for _, e := range src {
+		if e == nil {
+			continue
+		}
+		if in.StartFromFileName != "" {
+			if in.InclusiveStartFrom {
+				if e.Name < in.StartFromFileName {
+					continue
+				}
+			} else if e.Name <= in.StartFromFileName {
+				continue
+			}
+		}
+		filtered = append(filtered, e)
+	}
+	sort.SliceStable(filtered, func(i, j int) bool { return filtered[i].Name < filtered[j].Name })
+	if in.Limit > 0 && uint32(len(filtered)) > in.Limit {
+		filtered = filtered[:in.Limit]
+	}
+	resps := make([]*filer_pb.ListEntriesResponse, 0, len(filtered))
+	for _, e := range filtered {
+		resps = append(resps, &filer_pb.ListEntriesResponse{Entry: e})
+	}
+	return &fakeListStream{responses: resps, ctx: ctx}, nil
+}
+
+func dirEntry(name string, extended map[string][]byte) *filer_pb.Entry {
+	return &filer_pb.Entry{
+		Name:        name,
+		IsDirectory: true,
+		Attributes:  &filer_pb.FuseAttributes{},
+		Extended:    extended,
+	}
+}
+
+func fileEntry(name string) *filer_pb.Entry {
+	return &filer_pb.Entry{
+		Name:        name,
+		IsDirectory: false,
+		Attributes:  &filer_pb.FuseAttributes{},
+	}
+}