upstream-seaweedfs/weed/server/filer_server_storage_rule_ttl_test.go

package weed_server

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

const ttlRulePrefix = "/buckets/ttl/"

// addTtlRule gives ttlRulePrefix a 3 minute volume TTL, the fs.configure setting
// whose effect every write path below has to reproduce on the entries it stores.
func addTtlRule(t *testing.T, f *filer.Filer) {
	t.Helper()
	if err := f.FilerConf.AddLocationConf(&filer_pb.FilerConf_PathConf{
		LocationPrefix: ttlRulePrefix,
		Ttl:            "3m",
	}); err != nil {
		t.Fatalf("AddLocationConf: %v", err)
	}
}

// The native LMCache path allocates chunks and publishes the Filer Entry in
// separate RPCs. Both RPCs must resolve the same storage rule, or metadata can
// expire while its Volume remains permanent (or the reverse).
func TestNativeCacheWriteUsesOneTtlForAllocationAndEntry(t *testing.T) {
	store := newRenameTestStore()
	store.entries[ttlRulePrefix] = newDirectoryEntry(ttlRulePrefix, 10)
	server := &FilerServer{
		filer:          newRenameTestFiler(t, store),
		option:         &FilerOption{},
		entryLockTable: util.NewLockTable[util.FullPath](),
	}
	addTtlRule(t, server.filer)

	allocation, err := server.resolveAssignStorageOption(context.Background(), &filer_pb.AssignVolumeRequest{
		Path:       ttlRulePrefix + "cache-key",
		Collection: "lmcache",
	})
	if err != nil {
		t.Fatalf("resolveAssignStorageOption: %v", err)
	}
	if allocation.Collection != "lmcache" {
		t.Fatalf("allocation collection = %q, want lmcache", allocation.Collection)
	}
	if allocation.TtlSeconds != 180 {
		t.Fatalf("allocation TTL = %d, want 180", allocation.TtlSeconds)
	}

	if _, err := server.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
		Directory: "/buckets/ttl",
		Entry: &filer_pb.Entry{
			Name:       "cache-key",
			Attributes: &filer_pb.FuseAttributes{FileMode: 0644},
		},
	}); err != nil {
		t.Fatalf("CreateEntry: %v", err)
	}

	entry, err := store.FindEntry(context.Background(), ttlRulePrefix+"cache-key")
	if err != nil {
		t.Fatalf("FindEntry: %v", err)
	}
	if entry.TtlSec != allocation.TtlSeconds {
		t.Fatalf("entry TTL = %d, allocation TTL = %d", entry.TtlSec, allocation.TtlSeconds)
	}
}

// An object written through ObjectTransaction (the routed S3 write path) must
// pick up the path's TTL rule, the same as one written through CreateEntry.
func TestObjectTransactionPutAppliesRuleTtl(t *testing.T) {
	store := newRenameTestStore()
	store.entries[ttlRulePrefix] = newDirectoryEntry(ttlRulePrefix, 10)

	server := &FilerServer{
		filer:          newRenameTestFiler(t, store),
		option:         &FilerOption{},
		entryLockTable: util.NewLockTable[util.FullPath](),
	}
	addTtlRule(t, server.filer)

	if _, err := server.ObjectTransaction(context.Background(), &filer_pb.ObjectTransactionRequest{
		LockKey: ttlRulePrefix + "obj",
		Mutations: []*filer_pb.ObjectMutation{{
			Type:      filer_pb.ObjectMutation_PUT,
			Directory: "/buckets/ttl",
			Entry: &filer_pb.Entry{
				Name:       "obj",
				Attributes: &filer_pb.FuseAttributes{FileMode: 0644},
			},
		}},
	}); err != nil {
		t.Fatalf("ObjectTransaction: %v", err)
	}

	entry, err := store.FindEntry(context.Background(), ttlRulePrefix+"obj")
	if err != nil {
		t.Fatalf("FindEntry: %v", err)
	}
	if entry.TtlSec != 180 {
		t.Errorf("entry TtlSec = %d, want 180", entry.TtlSec)
	}
}

// A copy landing in a TTL path re-uploads its chunks under that path's rule, so
// the entry has to carry the rule's TTL too - whatever TTL the source had.
func TestCopyAppliesRuleTtl(t *testing.T) {
	for _, sourceTtlSec := range []int32{0, 600} {
		t.Run(fmt.Sprintf("source ttl %d", sourceTtlSec), func(t *testing.T) {
			store := newRenameTestStore()
			source := newFileEntry("/src.txt", 11)
			source.Content = []byte("hello")
			source.TtlSec = sourceTtlSec
			source.Crtime = time.Now() // a TTL entry older than its TTL reads back as expired
			store.entries["/src.txt"] = source
			store.entries[ttlRulePrefix] = newDirectoryEntry(ttlRulePrefix, 10)

			server := &FilerServer{
				filer:          newRenameTestFiler(t, store),
				option:         &FilerOption{},
				entryLockTable: util.NewLockTable[util.FullPath](),
			}
			addTtlRule(t, server.filer)

			req := httptest.NewRequest(http.MethodPost, ttlRulePrefix+"dst.txt?cp.from=/src.txt", http.NoBody)
			rec := httptest.NewRecorder()
			server.PostHandler(rec, req, 0)
			if rec.Code != http.StatusNoContent {
				t.Fatalf("copy = %d, want %d; body=%q", rec.Code, http.StatusNoContent, rec.Body.String())
			}

			entry, err := store.FindEntry(context.Background(), ttlRulePrefix+"dst.txt")
			if err != nil {
				t.Fatalf("FindEntry: %v", err)
			}
			if entry.TtlSec != 180 {
				t.Errorf("entry TtlSec = %d, want 180", entry.TtlSec)
			}
		})
	}
}

// A remote-backed entry copied into a TTL path must not expire locally: the
// remote storage owns its lifecycle, so a local expiry would drop the pointer
// to an object that is still there.
func TestCopyKeepsRemoteEntryUnexpiring(t *testing.T) {
	store := newRenameTestStore()
	source := newFileEntry("/src.txt", 11)
	source.Remote = &filer_pb.RemoteEntry{StorageName: "s3-remote", RemoteSize: 5}
	store.entries["/src.txt"] = source
	store.entries[ttlRulePrefix] = newDirectoryEntry(ttlRulePrefix, 10)

	server := &FilerServer{
		filer:          newRenameTestFiler(t, store),
		option:         &FilerOption{},
		entryLockTable: util.NewLockTable[util.FullPath](),
	}
	addTtlRule(t, server.filer)

	req := httptest.NewRequest(http.MethodPost, ttlRulePrefix+"dst.txt?cp.from=/src.txt", http.NoBody)
	rec := httptest.NewRecorder()
	server.PostHandler(rec, req, 0)
	if rec.Code != http.StatusNoContent {
		t.Fatalf("copy = %d, want %d; body=%q", rec.Code, http.StatusNoContent, rec.Body.String())
	}

	entry, err := store.FindEntry(context.Background(), ttlRulePrefix+"dst.txt")
	if err != nil {
		t.Fatalf("FindEntry: %v", err)
	}
	if entry.TtlSec != 0 {
		t.Errorf("remote entry TtlSec = %d, want 0", entry.TtlSec)
	}
}

// Clients percent-encode non-ASCII path segments on the wire, so a rule has to
// be matched against the decoded path the entry is written to, not the raw
// request-target: a read-only rule on "/data/只读/" must still refuse a POST to
// "/data/%E5%8F%AA%E8%AF%BB/".
func TestPostHandlerMatchesStorageRuleOnDecodedPath(t *testing.T) {
	const readOnlyPrefix = "/data/只读/"
	store := newRenameTestStore()
	source := newFileEntry("/src.txt", 11)
	source.Content = []byte("hello")
	store.entries["/src.txt"] = source
	store.entries[readOnlyPrefix] = newDirectoryEntry(readOnlyPrefix, 10)

	server := &FilerServer{
		filer:          newRenameTestFiler(t, store),
		option:         &FilerOption{},
		entryLockTable: util.NewLockTable[util.FullPath](),
	}
	if err := server.filer.FilerConf.AddLocationConf(&filer_pb.FilerConf_PathConf{
		LocationPrefix: readOnlyPrefix,
		ReadOnly:       true,
	}); err != nil {
		t.Fatalf("AddLocationConf: %v", err)
	}

	req := httptest.NewRequest(http.MethodPost, "/data/%E5%8F%AA%E8%AF%BB/dst.txt?cp.from=/src.txt", http.NoBody)
	rec := httptest.NewRecorder()
	server.PostHandler(rec, req, 0)
	if rec.Code != http.StatusInsufficientStorage {
		t.Fatalf("copy into read-only path = %d, want %d; body=%q", rec.Code, http.StatusInsufficientStorage, rec.Body.String())
	}
	if _, err := store.FindEntry(context.Background(), readOnlyPrefix+"dst.txt"); err == nil {
		t.Fatal("copy landed in the read-only path")
	}
}

// A completed TUS upload uploads its chunks under the target path's rule, so the
// entry it lands has to expire with them.
func TestCompleteTusUploadAppliesRuleTtl(t *testing.T) {
	fs, store := newTusTestServer(t, nil)
	addTtlRule(t, fs.filer)

	targetPath := ttlRulePrefix + "upload.bin"
	seedTusSession(t, fs, store, TusSession{ID: tusTestUploadID, TargetPath: targetPath, Size: 8})
	seedTusChunk(t, fs, store, tusTestUploadID, 0, 8, "3,01637037d6")

	req := tusRequest(http.MethodPatch, "/.tus/.uploads/"+tusTestUploadID, map[string]string{
		"Authorization": "Bearer " + signFilerToken(t, tusTestWriteKey, nil, nil),
		"Content-Type":  "application/offset+octet-stream",
		"Upload-Offset": "8",
	}, "")
	rec := httptest.NewRecorder()
	fs.tusHandler(rec, req)
	if rec.Code != http.StatusNoContent {
		t.Fatalf("PATCH = %d, want %d; body=%q", rec.Code, http.StatusNoContent, rec.Body.String())
	}

	entry, err := store.FindEntry(context.Background(), util.FullPath(targetPath))
	if err != nil {
		t.Fatalf("FindEntry: %v", err)
	}
	if entry.TtlSec != 180 {
		t.Errorf("entry TtlSec = %d, want 180", entry.TtlSec)
	}
}