snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/filer/abstract_sql/abstract_sql_store.go

@@ -0,0 +1,479 @@
+package abstract_sql
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type SqlGenerator interface {
+	GetSqlInsert(tableName string) string
+	GetSqlUpdate(tableName string) string
+	GetSqlFind(tableName string) string
+	GetSqlDelete(tableName string) string
+	GetSqlDeleteFolderChildren(tableName string) string
+	GetSqlListExclusive(tableName string) string
+	GetSqlListInclusive(tableName string) string
+	GetSqlCreateTable(tableName string) string
+	GetSqlDropTable(tableName string) string
+}
+
+type AbstractSqlStore struct {
+	SqlGenerator
+	DB                     *sql.DB
+	KvDB                   *sql.DB
+	SupportBucketTable     bool
+	SkipDDL                bool
+	dbs                    map[string]bool
+	dbsLock                sync.Mutex
+	RetryableErrorCallback func(err error) bool
+}
+
+var _ filer.BucketAware = (*AbstractSqlStore)(nil)
+
+func (store *AbstractSqlStore) CanDropWholeBucket() bool {
+	return store.SupportBucketTable && !store.SkipDDL
+}
+func (store *AbstractSqlStore) OnBucketCreation(bucket string) {
+	store.dbsLock.Lock()
+	defer store.dbsLock.Unlock()
+
+	store.CreateTable(context.Background(), bucket)
+
+	if store.dbs == nil {
+		return
+	}
+	store.dbs[bucket] = true
+}
+func (store *AbstractSqlStore) OnBucketDeletion(bucket string) {
+	store.dbsLock.Lock()
+	defer store.dbsLock.Unlock()
+
+	store.deleteTable(context.Background(), bucket)
+
+	if store.dbs == nil {
+		return
+	}
+	delete(store.dbs, bucket)
+}
+
+const (
+	DEFAULT_TABLE = "filemeta"
+)
+
+type TxOrDB interface {
+	ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
+	QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row
+	QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
+}
+
+func (store *AbstractSqlStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	tx, err := store.DB.BeginTx(ctx, &sql.TxOptions{
+		Isolation: sql.LevelReadCommitted,
+		ReadOnly:  false,
+	})
+	if err != nil {
+		return ctx, err
+	}
+
+	return context.WithValue(ctx, "tx", tx), nil
+}
+func (store *AbstractSqlStore) CommitTransaction(ctx context.Context) error {
+	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
+		return tx.Commit()
+	}
+	return nil
+}
+func (store *AbstractSqlStore) RollbackTransaction(ctx context.Context) error {
+	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
+		return tx.Rollback()
+	}
+	return nil
+}
+
+// A listing holds its connection for the whole row iteration while its callback
+// reads a hard link through KvGet, so both cannot come out of one bounded pool:
+// the listings fill it and then wait for a connection none of them will release.
+// Give the key-value reads their own slice of connection_max_open. A cap of 1 is
+// the exception -- it has to become 2, or a single listing cannot finish.
+func splitPoolForKv(maxOpen int) (mainOpen, kvOpen int) {
+	if maxOpen <= 0 {
+		return maxOpen, 0
+	}
+	kvOpen = min(max(maxOpen/4, 1), maxKvPoolSize)
+	return max(maxOpen-kvOpen, 1), kvOpen
+}
+
+const maxKvPoolSize = 8
+
+// UseConnectionPools sizes the store's pool and, when it is bounded, opens the
+// separate pool the key-value reads run on.
+func (store *AbstractSqlStore) UseConnectionPools(db *sql.DB, openKv func() (*sql.DB, error), maxIdle, maxOpen, maxLifetimeSeconds int) error {
+
+	lifetime := time.Duration(maxLifetimeSeconds) * time.Second
+	mainOpen, kvOpen := splitPoolForKv(maxOpen)
+
+	db.SetMaxIdleConns(maxIdle)
+	db.SetMaxOpenConns(mainOpen)
+	db.SetConnMaxLifetime(lifetime)
+	store.DB = db
+
+	if kvOpen == 0 {
+		return nil
+	}
+
+	kvDB, err := openKv()
+	if err != nil {
+		return err
+	}
+	kvDB.SetMaxIdleConns(kvOpen)
+	kvDB.SetMaxOpenConns(kvOpen)
+	kvDB.SetConnMaxLifetime(lifetime)
+	store.KvDB = kvDB
+
+	return nil
+}
+
+func (store *AbstractSqlStore) getTxOrDB(ctx context.Context, fullpath util.FullPath, isForChildren bool) (txOrDB TxOrDB, bucket string, shortPath util.FullPath, err error) {
+
+	shortPath = fullpath
+	bucket = DEFAULT_TABLE
+
+	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
+		txOrDB = tx
+	} else {
+		txOrDB = store.DB
+	}
+
+	if !store.SupportBucketTable {
+		return
+	}
+
+	if !strings.HasPrefix(string(fullpath), "/buckets/") {
+		return
+	}
+
+	// detect bucket
+	bucketAndObjectKey := string(fullpath)[len("/buckets/"):]
+	t := strings.Index(bucketAndObjectKey, "/")
+	if t < 0 && !isForChildren {
+		return
+	}
+	bucket = bucketAndObjectKey
+	shortPath = "/"
+	if t > 0 {
+		bucket = bucketAndObjectKey[:t]
+		shortPath = util.FullPath(bucketAndObjectKey[t:])
+	}
+
+	// Dot-prefixed entries directly under /buckets (e.g. .system) are internal
+	// folders, not S3 buckets; keep them in the default table by full path.
+	if strings.HasPrefix(bucket, ".") {
+		bucket = DEFAULT_TABLE
+		shortPath = fullpath
+		return
+	}
+
+	if isValidBucket(bucket) {
+		store.dbsLock.Lock()
+		defer store.dbsLock.Unlock()
+
+		if store.dbs == nil {
+			store.dbs = make(map[string]bool)
+		}
+
+		if _, found := store.dbs[bucket]; !found {
+			if err = store.CreateTable(ctx, bucket); err == nil {
+				store.dbs[bucket] = true
+			}
+		}
+
+	} else {
+		err = fmt.Errorf("invalid bucket name %s", bucket)
+	}
+
+	return
+}
+
+func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	// define the work to be done
+	var doInsert func() error
+	doInsert = func() error {
+		db, bucket, shortPath, err := store.getTxOrDB(ctx, entry.FullPath, false)
+		if err != nil {
+			return fmt.Errorf("findDB %s : %w", entry.FullPath, err)
+		}
+
+		dir, name := shortPath.DirAndName()
+		meta, err := entry.EncodeAttributesAndChunks()
+		if err != nil {
+			return fmt.Errorf("encode %s: %w", entry.FullPath, err)
+		}
+
+		if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+			meta = util.MaybeGzipData(meta)
+		}
+		sqlInsert := "insert"
+		res, err := db.ExecContext(ctx, store.GetSqlInsert(bucket), util.HashStringToLong(dir), name, dir, meta)
+		if err != nil && strings.Contains(strings.ToLower(err.Error()), "duplicate entry") {
+			// now the insert failed possibly due to duplication constraints
+			sqlInsert = "falls back to update"
+			glog.V(1).InfofCtx(ctx, "insert %s %s: %v", entry.FullPath, sqlInsert, err)
+			res, err = db.ExecContext(ctx, store.GetSqlUpdate(bucket), meta, util.HashStringToLong(dir), name, dir)
+		}
+		if err != nil {
+			return fmt.Errorf("%s %s: %w", sqlInsert, entry.FullPath, err)
+		}
+
+		_, err = res.RowsAffected()
+		if err != nil {
+			return fmt.Errorf("%s %s but no rows affected: %w", sqlInsert, entry.FullPath, err)
+		}
+
+		return nil
+	}
+
+	if store.RetryableErrorCallback != nil {
+		if ctx.Value("tx") != nil {
+			return doInsert()
+		}
+		return util.RetryUntil("InsertEntry", doInsert, store.RetryableErrorCallback)
+	}
+	return doInsert()
+}
+
+func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	var doUpdate func() error
+	doUpdate = func() error {
+		db, bucket, shortPath, err := store.getTxOrDB(ctx, entry.FullPath, false)
+		if err != nil {
+			return fmt.Errorf("findDB %s : %w", entry.FullPath, err)
+		}
+
+		dir, name := shortPath.DirAndName()
+		meta, err := entry.EncodeAttributesAndChunks()
+		if err != nil {
+			return fmt.Errorf("encode %s: %w", entry.FullPath, err)
+		}
+
+		res, err := db.ExecContext(ctx, store.GetSqlUpdate(bucket), meta, util.HashStringToLong(dir), name, dir)
+		if err != nil {
+			return fmt.Errorf("update %s: %w", entry.FullPath, err)
+		}
+
+		_, err = res.RowsAffected()
+		if err != nil {
+			return fmt.Errorf("update %s but no rows affected: %w", entry.FullPath, err)
+		}
+		return nil
+	}
+
+	if store.RetryableErrorCallback != nil {
+		if ctx.Value("tx") != nil {
+			return doUpdate()
+		}
+		return util.RetryUntil("UpdateEntry", doUpdate, store.RetryableErrorCallback)
+	}
+	return doUpdate()
+}
+
+func (store *AbstractSqlStore) FindEntry(ctx context.Context, fullpath util.FullPath) (*filer.Entry, error) {
+
+	db, bucket, shortPath, err := store.getTxOrDB(ctx, fullpath, false)
+	if err != nil {
+		return nil, fmt.Errorf("findDB %s : %v", fullpath, err)
+	}
+
+	dir, name := shortPath.DirAndName()
+	row := db.QueryRowContext(ctx, store.GetSqlFind(bucket), util.HashStringToLong(dir), name, dir)
+
+	var data []byte
+	if err := row.Scan(&data); err != nil {
+		if err == sql.ErrNoRows {
+			return nil, filer_pb.ErrNotFound
+		}
+		return nil, fmt.Errorf("find %s: %v", fullpath, err)
+	}
+
+	entry := &filer.Entry{
+		FullPath: fullpath,
+	}
+	if err := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *AbstractSqlStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {
+
+	var doDelete func() error
+	doDelete = func() error {
+		db, bucket, shortPath, err := store.getTxOrDB(ctx, fullpath, false)
+		if err != nil {
+			return fmt.Errorf("findDB %s : %w", fullpath, err)
+		}
+
+		dir, name := shortPath.DirAndName()
+
+		res, err := db.ExecContext(ctx, store.GetSqlDelete(bucket), util.HashStringToLong(dir), name, dir)
+		if err != nil {
+			return fmt.Errorf("delete %s: %w", fullpath, err)
+		}
+
+		_, err = res.RowsAffected()
+		if err != nil {
+			return fmt.Errorf("delete %s but no rows affected: %w", fullpath, err)
+		}
+		return nil
+	}
+
+	if store.RetryableErrorCallback != nil {
+		if ctx.Value("tx") != nil {
+			return doDelete()
+		}
+		return util.RetryUntil("DeleteEntry", doDelete, store.RetryableErrorCallback)
+	}
+	return doDelete()
+}
+
+func (store *AbstractSqlStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
+
+	var doDeleteFolderChildren func() error
+	doDeleteFolderChildren = func() error {
+		db, bucket, shortPath, err := store.getTxOrDB(ctx, fullpath, true)
+		if err != nil {
+			return fmt.Errorf("findDB %s : %w", fullpath, err)
+		}
+
+		if isValidBucket(bucket) && shortPath == "/" && store.CanDropWholeBucket() {
+			if err = store.deleteTable(ctx, bucket); err == nil {
+				store.dbsLock.Lock()
+				delete(store.dbs, bucket)
+				store.dbsLock.Unlock()
+				return nil
+			} else {
+				return err
+			}
+		}
+
+		glog.V(4).InfofCtx(ctx, "delete %s SQL %s %d", string(shortPath), store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)))
+		res, err := db.ExecContext(ctx, store.GetSqlDeleteFolderChildren(bucket), util.HashStringToLong(string(shortPath)), string(shortPath))
+		if err != nil {
+			return fmt.Errorf("deleteFolderChildren %s: %w", fullpath, err)
+		}
+
+		_, err = res.RowsAffected()
+		if err != nil {
+			return fmt.Errorf("deleteFolderChildren %s but no rows affected: %w", fullpath, err)
+		}
+		return nil
+	}
+
+	if store.RetryableErrorCallback != nil {
+		if ctx.Value("tx") != nil {
+			return doDeleteFolderChildren()
+		}
+		return util.RetryUntil("DeleteFolderChildren", doDeleteFolderChildren, store.RetryableErrorCallback)
+	}
+	return doDeleteFolderChildren()
+}
+
+func (store *AbstractSqlStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+
+	db, bucket, shortPath, err := store.getTxOrDB(ctx, dirPath, true)
+	if err != nil {
+		return lastFileName, fmt.Errorf("findDB %s : %v", dirPath, err)
+	}
+
+	sqlText := store.GetSqlListExclusive(bucket)
+	if includeStartFile {
+		sqlText = store.GetSqlListInclusive(bucket)
+	}
+
+	rows, err := db.QueryContext(ctx, sqlText, util.HashStringToLong(string(shortPath)), startFileName, string(shortPath), prefix+"%", limit+1)
+	if err != nil {
+		return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var name string
+		var data []byte
+		if err = rows.Scan(&name, &data); err != nil {
+			glog.V(0).InfofCtx(ctx, "scan %s : %v", dirPath, err)
+			return lastFileName, fmt.Errorf("scan %s: %v", dirPath, err)
+		}
+		lastFileName = name
+
+		entry := &filer.Entry{
+			FullPath: util.NewFullPath(string(dirPath), name),
+		}
+		if err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); err != nil {
+			glog.V(0).InfofCtx(ctx, "scan decode %s : %v", entry.FullPath, err)
+			return lastFileName, fmt.Errorf("scan decode %s : %v", entry.FullPath, err)
+		}
+
+		resEachEntryFunc, resEachEntryFuncErr := eachEntryFunc(entry)
+		if resEachEntryFuncErr != nil {
+			err = fmt.Errorf("failed to process eachEntryFunc: %w", resEachEntryFuncErr)
+			break
+		}
+
+		if !resEachEntryFunc {
+			break
+		}
+
+	}
+
+	return lastFileName, err
+}
+
+func (store *AbstractSqlStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc)
+}
+
+func (store *AbstractSqlStore) Shutdown() {
+	store.DB.Close()
+	if store.KvDB != nil {
+		store.KvDB.Close()
+	}
+}
+
+func isValidBucket(bucket string) bool {
+	if s3bucket.VerifyS3BucketName(bucket) != nil {
+		return false
+	}
+	return bucket != DEFAULT_TABLE && bucket != ""
+}
+
+func (store *AbstractSqlStore) CreateTable(ctx context.Context, bucket string) error {
+	if !store.SupportBucketTable || store.SkipDDL {
+		return nil
+	}
+	sql := store.SqlGenerator.GetSqlCreateTable(bucket)
+	if sql == "" {
+		return nil
+	}
+	_, err := store.DB.ExecContext(ctx, sql)
+	return err
+}
+
+func (store *AbstractSqlStore) deleteTable(ctx context.Context, bucket string) error {
+	if !store.SupportBucketTable || store.SkipDDL {
+		return nil
+	}
+	_, err := store.DB.ExecContext(ctx, store.SqlGenerator.GetSqlDropTable(bucket))
+	return err
+}

weed/filer/abstract_sql/abstract_sql_store_kv.go

@@ -0,0 +1,109 @@
+package abstract_sql
+
+import (
+	"context"
+	"database/sql"
+	"encoding/base64"
+	"fmt"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// Key-value reads run outside the listing pool: a listing calls KvGet from
+// inside its own callback, still holding the connection its rows are on.
+func (store *AbstractSqlStore) kvDB(ctx context.Context) TxOrDB {
+	if tx, ok := ctx.Value("tx").(*sql.Tx); ok {
+		return tx
+	}
+	if store.KvDB != nil {
+		return store.KvDB
+	}
+	return store.DB
+}
+
+func (store *AbstractSqlStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+
+	db := store.kvDB(ctx)
+
+	dirStr, dirHash, name := GenDirAndName(key)
+
+	_, err = db.ExecContext(ctx, store.GetSqlInsert(DEFAULT_TABLE), dirHash, name, dirStr, value)
+	if err == nil {
+		return
+	}
+
+	if !strings.Contains(strings.ToLower(err.Error()), "duplicate") {
+		// return fmt.Errorf("kv insert: %s", err)
+		// skip this since the error can be in a different language
+	}
+
+	// now the insert failed possibly due to duplication constraints
+	glog.V(1).InfofCtx(ctx, "kv insert falls back to update: %s", err)
+
+	res, err := db.ExecContext(ctx, store.GetSqlUpdate(DEFAULT_TABLE), value, dirHash, name, dirStr)
+	if err != nil {
+		return fmt.Errorf("kv upsert: %s", err)
+	}
+
+	_, err = res.RowsAffected()
+	if err != nil {
+		return fmt.Errorf("kv upsert no rows affected: %s", err)
+	}
+	return nil
+
+}
+
+func (store *AbstractSqlStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+
+	db := store.kvDB(ctx)
+
+	dirStr, dirHash, name := GenDirAndName(key)
+	row := db.QueryRowContext(ctx, store.GetSqlFind(DEFAULT_TABLE), dirHash, name, dirStr)
+
+	err = row.Scan(&value)
+
+	if err == sql.ErrNoRows {
+		return nil, filer.ErrKvNotFound
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("kv get: %w", err)
+	}
+
+	return
+}
+
+func (store *AbstractSqlStore) KvDelete(ctx context.Context, key []byte) (err error) {
+
+	db := store.kvDB(ctx)
+
+	dirStr, dirHash, name := GenDirAndName(key)
+
+	res, err := db.ExecContext(ctx, store.GetSqlDelete(DEFAULT_TABLE), dirHash, name, dirStr)
+	if err != nil {
+		return fmt.Errorf("kv delete: %s", err)
+	}
+
+	_, err = res.RowsAffected()
+	if err != nil {
+		return fmt.Errorf("kv delete no rows affected: %s", err)
+	}
+
+	return nil
+
+}
+
+func GenDirAndName(key []byte) (dirStr string, dirHash int64, name string) {
+	for len(key) < 8 {
+		key = append(key, 0)
+	}
+
+	dirHash = int64(util.BytesToUint64(key[:8]))
+	dirStr = base64.StdEncoding.EncodeToString(key[:8])
+	name = base64.StdEncoding.EncodeToString(key[8:])
+
+	return
+}

weed/filer/abstract_sql/abstract_sql_store_test.go

@@ -0,0 +1,73 @@
+package abstract_sql
+
+import (
+	"context"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// The bucket owner index lives at /buckets/.system/owners/<owner>/<bucket>.
+// .system is an internal folder, not an S3 bucket, so getTxOrDB must route it
+// to the default table rather than reject it as an invalid bucket name.
+func TestGetTxOrDBInternalSystemFolder(t *testing.T) {
+	store := &AbstractSqlStore{SupportBucketTable: true}
+	ctx := context.Background()
+
+	cases := []struct {
+		path          string
+		isForChildren bool
+	}{
+		{"/buckets/.system/owners", false},
+		{"/buckets/.system/owners/foo/bucket1", false},
+		{"/buckets/.system/owners/foo", true},
+	}
+	for _, c := range cases {
+		_, bucket, shortPath, err := store.getTxOrDB(ctx, util.FullPath(c.path), c.isForChildren)
+		if err != nil {
+			t.Errorf("getTxOrDB(%s): unexpected error: %v", c.path, err)
+		}
+		if bucket != DEFAULT_TABLE {
+			t.Errorf("getTxOrDB(%s): bucket = %q, want %q", c.path, bucket, DEFAULT_TABLE)
+		}
+		if string(shortPath) != c.path {
+			t.Errorf("getTxOrDB(%s): shortPath = %q, want %q", c.path, shortPath, c.path)
+		}
+	}
+}
+
+func TestGetTxOrDBRealBucket(t *testing.T) {
+	store := &AbstractSqlStore{SupportBucketTable: true, dbs: map[string]bool{"mybucket": true}}
+	ctx := context.Background()
+
+	_, bucket, shortPath, err := store.getTxOrDB(ctx, util.FullPath("/buckets/mybucket/dir/file.txt"), false)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if bucket != "mybucket" {
+		t.Errorf("bucket = %q, want mybucket", bucket)
+	}
+	if string(shortPath) != "/dir/file.txt" {
+		t.Errorf("shortPath = %q, want /dir/file.txt", shortPath)
+	}
+}
+
+func TestSplitPoolForKv(t *testing.T) {
+	cases := []struct {
+		maxOpen, mainOpen, kvOpen int
+	}{
+		{0, 0, 0},   // unbounded: nothing waits, one pool is enough
+		{-1, -1, 0}, // same, however the operator spelled it
+		{50, 42, 8},
+		{16, 12, 4},
+		{4, 3, 1},
+		{2, 1, 1},
+		{1, 1, 1}, // a listing needs two, so a cap of 1 has to become 2
+	}
+	for _, c := range cases {
+		mainOpen, kvOpen := splitPoolForKv(c.maxOpen)
+		if mainOpen != c.mainOpen || kvOpen != c.kvOpen {
+			t.Errorf("splitPoolForKv(%d) = %d, %d, want %d, %d", c.maxOpen, mainOpen, kvOpen, c.mainOpen, c.kvOpen)
+		}
+	}
+}

weed/filer/arangodb/arangodb_store.go

@@ -0,0 +1,356 @@
+package arangodb
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/arangodb/go-driver"
+	"github.com/arangodb/go-driver/http"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &ArangodbStore{})
+}
+
+var (
+	BUCKET_PREFIX      = "/buckets"
+	DEFAULT_COLLECTION = "seaweed_no_bucket"
+	KVMETA_COLLECTION  = "seaweed_kvmeta"
+)
+
+type ArangodbStore struct {
+	connect      driver.Connection
+	client       driver.Client
+	database     driver.Database
+	kvCollection driver.Collection
+
+	buckets map[string]driver.Collection
+	mu      sync.RWMutex
+
+	databaseName string
+}
+
+type Model struct {
+	Key       string `json:"_key"`
+	Directory string `json:"directory,omitempty"`
+	Name      string `json:"name,omitempty"`
+	Ttl       string `json:"ttl,omitempty"`
+
+	//arangodb does not support binary blobs
+	//we encode byte slice into uint64 slice
+	//see helpers.go
+	Meta []uint64 `json:"meta"`
+}
+
+func (store *ArangodbStore) GetName() string {
+	return "arangodb"
+}
+
+func (store *ArangodbStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	store.buckets = make(map[string]driver.Collection, 3)
+	store.databaseName = configuration.GetString(prefix + "db_name")
+	return store.connection(configuration.GetStringSlice(prefix+"servers"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetBool(prefix+"insecure_skip_verify"),
+	)
+}
+
+func (store *ArangodbStore) connection(uris []string, user string, pass string, insecure bool) (err error) {
+	ctx, cn := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cn()
+	store.connect, err = http.NewConnection(http.ConnectionConfig{
+		Endpoints: uris,
+		TLSConfig: &tls.Config{
+			InsecureSkipVerify: insecure,
+		},
+	})
+	if err != nil {
+		return err
+	}
+	store.client, err = driver.NewClient(driver.ClientConfig{
+		Connection:     store.connect,
+		Authentication: driver.BasicAuthentication(user, pass),
+	})
+	if err != nil {
+		return err
+	}
+	ok, err := store.client.DatabaseExists(ctx, store.databaseName)
+	if err != nil {
+		return err
+	}
+	if ok {
+		store.database, err = store.client.Database(ctx, store.databaseName)
+	} else {
+		store.database, err = store.client.CreateDatabase(ctx, store.databaseName, &driver.CreateDatabaseOptions{})
+	}
+	if err != nil {
+		return err
+	}
+	if store.kvCollection, err = store.ensureCollection(ctx, KVMETA_COLLECTION); err != nil {
+		return err
+	}
+	return err
+}
+
+type key int
+
+const (
+	transactionKey key = 0
+)
+
+func (store *ArangodbStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	keys := make([]string, 0, len(store.buckets)+1)
+	for k := range store.buckets {
+		keys = append(keys, k)
+	}
+	keys = append(keys, store.kvCollection.Name())
+	txn, err := store.database.BeginTransaction(ctx, driver.TransactionCollections{
+		Exclusive: keys,
+	}, &driver.BeginTransactionOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return context.WithValue(driver.WithTransactionID(ctx, txn), transactionKey, txn), nil
+}
+
+func (store *ArangodbStore) CommitTransaction(ctx context.Context) error {
+	val := ctx.Value(transactionKey)
+	cast, ok := val.(driver.TransactionID)
+	if !ok {
+		return fmt.Errorf("txn cast fail %s:", val)
+	}
+	err := store.database.CommitTransaction(ctx, cast, &driver.CommitTransactionOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (store *ArangodbStore) RollbackTransaction(ctx context.Context) error {
+	val := ctx.Value(transactionKey)
+	cast, ok := val.(driver.TransactionID)
+	if !ok {
+		return fmt.Errorf("txn cast fail %s:", val)
+	}
+	err := store.database.AbortTransaction(ctx, cast, &driver.AbortTransactionOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (store *ArangodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	dir, name := entry.FullPath.DirAndName()
+	meta, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
+	}
+
+	if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+		meta = util.MaybeGzipData(meta)
+	}
+	model := &Model{
+		Key:       hashString(string(entry.FullPath)),
+		Directory: dir,
+		Name:      name,
+		Meta:      bytesToArray(meta),
+	}
+	if entry.TtlSec > 0 {
+		model.Ttl = time.Now().Add(time.Second * time.Duration(entry.TtlSec)).Format(time.RFC3339)
+	} else {
+		model.Ttl = ""
+	}
+
+	targetCollection, err := store.extractBucketCollection(ctx, entry.FullPath)
+	if err != nil {
+		return err
+	}
+	_, err = targetCollection.CreateDocument(ctx, model)
+	if driver.IsConflict(err) {
+		return store.UpdateEntry(ctx, entry)
+	}
+
+	if err != nil {
+		return fmt.Errorf("InsertEntry %s: %v", entry.FullPath, err)
+	}
+
+	return nil
+
+}
+
+func (store *ArangodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	dir, name := entry.FullPath.DirAndName()
+	meta, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
+	}
+
+	if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+		meta = util.MaybeGzipData(meta)
+	}
+	model := &Model{
+		Key:       hashString(string(entry.FullPath)),
+		Directory: dir,
+		Name:      name,
+		Meta:      bytesToArray(meta),
+	}
+	if entry.TtlSec > 0 {
+		model.Ttl = time.Now().Add(time.Duration(entry.TtlSec) * time.Second).Format(time.RFC3339)
+	} else {
+		model.Ttl = "none"
+	}
+	targetCollection, err := store.extractBucketCollection(ctx, entry.FullPath)
+	if err != nil {
+		return err
+	}
+	_, err = targetCollection.UpdateDocument(ctx, model.Key, model)
+	if err != nil {
+		return fmt.Errorf("UpdateEntry %s: %v", entry.FullPath, err)
+	}
+
+	return nil
+}
+
+func (store *ArangodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+	var data Model
+	targetCollection, err := store.extractBucketCollection(ctx, fullpath)
+	if err != nil {
+		return nil, err
+	}
+	_, err = targetCollection.ReadDocument(ctx, hashString(string(fullpath)), &data)
+	if err != nil {
+		if driver.IsNotFound(err) {
+			return nil, filer_pb.ErrNotFound
+		}
+		glog.ErrorfCtx(ctx, "find %s: %v", fullpath, err)
+		return nil, filer_pb.ErrNotFound
+	}
+	if len(data.Meta) == 0 {
+		return nil, filer_pb.ErrNotFound
+	}
+	entry = &filer.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(arrayToBytes(data.Meta)))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *ArangodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) {
+	targetCollection, err := store.extractBucketCollection(ctx, fullpath)
+	if err != nil {
+		return err
+	}
+	_, err = targetCollection.RemoveDocument(ctx, hashString(string(fullpath)))
+	if err != nil && !driver.IsNotFound(err) {
+		glog.ErrorfCtx(ctx, "find %s: %v", fullpath, err)
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+	return nil
+}
+
+// this runs in log time
+func (store *ArangodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
+	targetCollection, err := store.extractBucketCollection(ctx, fullpath)
+	if err != nil {
+		return err
+	}
+	query := `
+	for d in @@collection
+	filter starts_with(d.directory, @dirPrefix)  || d.directory == @dir
+	remove d._key in @@collection`
+	cur, err := store.database.Query(ctx, query, map[string]interface{}{
+		"@collection": targetCollection.Name(),
+		"dirPrefix":   string(fullpath) + "/",
+		"dir":         string(fullpath),
+	})
+	if err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+	defer cur.Close()
+	return nil
+}
+
+func (store *ArangodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc)
+}
+
+func (store *ArangodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	targetCollection, err := store.extractBucketCollection(ctx, dirPath+"/")
+	if err != nil {
+		return lastFileName, err
+	}
+	bindVars := map[string]interface{}{
+		"@collection": targetCollection.Name(),
+		"dir":         dirPath,
+		"startFile":   startFileName,
+	}
+	query := "for d in @@collection"
+	if includeStartFile {
+		query = query + " filter d.name >= @startFile "
+	} else {
+		query = query + " filter d.name > @startFile "
+	}
+	if prefix != "" {
+		query = query + "&& starts_with(d.name, @prefix)"
+		bindVars["prefix"] = prefix
+	}
+	query = query + `
+filter d.directory == @dir
+sort d.name asc
+`
+	if limit > 0 {
+		query = query + "limit " + strconv.Itoa(int(limit))
+	}
+	query = query + "\n return d"
+	cur, err := store.database.Query(ctx, query, bindVars)
+	if err != nil {
+		return lastFileName, fmt.Errorf("failed to list directory entries: find error: %w", err)
+	}
+	defer cur.Close()
+	for cur.HasMore() {
+		var data Model
+		_, err = cur.ReadDocument(ctx, &data)
+		if err != nil {
+			break
+		}
+		entry := &filer.Entry{
+			FullPath: util.NewFullPath(data.Directory, data.Name),
+		}
+		lastFileName = data.Name
+		converted := arrayToBytes(data.Meta)
+		if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(converted)); decodeErr != nil {
+			err = decodeErr
+			glog.V(0).InfofCtx(ctx, "list %s : %v", entry.FullPath, err)
+			break
+		}
+
+		resEachEntryFunc, resEachEntryFuncErr := eachEntryFunc(entry)
+		if resEachEntryFuncErr != nil {
+			err = fmt.Errorf("failed to process eachEntryFunc: %w", resEachEntryFuncErr)
+			break
+		}
+
+		if !resEachEntryFunc {
+			break
+		}
+
+	}
+	return lastFileName, err
+}
+
+func (store *ArangodbStore) Shutdown() {
+}

weed/filer/arangodb/arangodb_store_bucket.go

@@ -0,0 +1,42 @@
+package arangodb
+
+import (
+	"context"
+	"time"
+
+	"github.com/arangodb/go-driver"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+var _ filer.BucketAware = (*ArangodbStore)(nil)
+
+func (store *ArangodbStore) OnBucketCreation(bucket string) {
+	timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	// create the collection && add to cache
+	_, err := store.ensureBucket(timeout, bucket)
+	if err != nil {
+		glog.Errorf("bucket create %s: %v", bucket, err)
+	}
+}
+func (store *ArangodbStore) OnBucketDeletion(bucket string) {
+	timeout, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	collection, err := store.ensureBucket(timeout, bucket)
+	if err != nil {
+		glog.Errorf("bucket delete %s: %v", bucket, err)
+		return
+	}
+	err = collection.Remove(timeout)
+	if err != nil && !driver.IsNotFound(err) {
+		glog.Errorf("bucket delete %s: %v", bucket, err)
+		return
+	}
+	store.mu.Lock()
+	delete(store.buckets, bucket)
+	store.mu.Unlock()
+}
+func (store *ArangodbStore) CanDropWholeBucket() bool {
+	return true
+}

weed/filer/arangodb/arangodb_store_kv.go

@@ -0,0 +1,54 @@
+package arangodb
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/arangodb/go-driver"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+	model := &Model{
+		Key:       hashString(".kvstore." + string(key)),
+		Directory: ".kvstore." + string(key),
+		Meta:      bytesToArray(value),
+	}
+
+	exists, err := store.kvCollection.DocumentExists(ctx, model.Key)
+	if err != nil {
+		return fmt.Errorf("kv put: %w", err)
+	}
+	if exists {
+		_, err = store.kvCollection.UpdateDocument(ctx, model.Key, model)
+	} else {
+		_, err = store.kvCollection.CreateDocument(ctx, model)
+	}
+	if err != nil {
+		return fmt.Errorf("kv put: %w", err)
+	}
+
+	return nil
+}
+func (store *ArangodbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+	var model Model
+	_, err = store.kvCollection.ReadDocument(ctx, hashString(".kvstore."+string(key)), &model)
+	if driver.IsNotFound(err) {
+		return nil, filer.ErrKvNotFound
+	}
+	if err != nil {
+		glog.ErrorfCtx(ctx, "kv get: %s %v", string(key), err)
+		return nil, filer.ErrKvNotFound
+	}
+	return arrayToBytes(model.Meta), nil
+}
+
+func (store *ArangodbStore) KvDelete(ctx context.Context, key []byte) (err error) {
+	_, err = store.kvCollection.RemoveDocument(ctx, hashString(".kvstore."+string(key)))
+	if err != nil {
+		glog.ErrorfCtx(ctx, "kv del: %v", err)
+		return filer.ErrKvNotFound
+	}
+	return nil
+}

weed/filer/arangodb/arangodb_store_test.go

@@ -0,0 +1,195 @@
+package arangodb
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/arangodb/go-driver"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func newTestStore(t *testing.T) *ArangodbStore {
+	t.Helper()
+
+	if os.Getenv("RUN_ARANGODB_TESTS") != "1" {
+		t.Skip("arangodb tests are disabled. Start an arangodb server and set RUN_ARANGODB_TESTS=1 to enable, ARANGODB_ADDR defaults to http://127.0.0.1:8529.")
+	}
+
+	addr := os.Getenv("ARANGODB_ADDR")
+	if addr == "" {
+		addr = "http://127.0.0.1:8529"
+	}
+	user := os.Getenv("ARANGODB_USER")
+	if user == "" {
+		user = "root"
+	}
+
+	store := &ArangodbStore{databaseName: fmt.Sprintf("seaweed_test_%d", time.Now().UnixNano())}
+	store.buckets = make(map[string]driver.Collection, 3)
+	if err := store.connection([]string{addr}, user, os.Getenv("ARANGODB_PASSWORD"), true); err != nil {
+		t.Fatalf("connect to arangodb at %s: %v", addr, err)
+	}
+	t.Cleanup(func() {
+		if err := store.database.Remove(context.Background()); err != nil {
+			t.Errorf("drop test database: %v", err)
+		}
+	})
+	return store
+}
+
+func insertTestEntry(t *testing.T, store *ArangodbStore, path string) {
+	t.Helper()
+	if err := store.InsertEntry(context.Background(), &filer.Entry{FullPath: util.FullPath(path)}); err != nil {
+		t.Fatalf("insert %s: %v", path, err)
+	}
+}
+
+func countDocuments(t *testing.T, store *ArangodbStore, bucket string) int64 {
+	t.Helper()
+	collection, err := store.ensureBucket(context.Background(), bucket)
+	if err != nil {
+		t.Fatalf("ensure bucket %s: %v", bucket, err)
+	}
+	count, err := collection.Count(context.Background())
+	if err != nil {
+		t.Fatalf("count %s: %v", bucket, err)
+	}
+	return count
+}
+
+func listNames(t *testing.T, store *ArangodbStore, dir util.FullPath, prefix string) []string {
+	t.Helper()
+	var names []string
+	_, err := store.ListDirectoryPrefixedEntries(context.Background(), dir, "", true, 100, prefix,
+		func(entry *filer.Entry) (bool, error) {
+			names = append(names, entry.Name())
+			return true, nil
+		})
+	if err != nil {
+		t.Fatalf("list %s prefix %q: %v", dir, prefix, err)
+	}
+	return names
+}
+
+// AQL operators in a list prefix must be matched literally, never executed.
+func TestListDirectoryPrefixedEntriesAqlInjection(t *testing.T) {
+	store := newTestStore(t)
+
+	insertTestEntry(t, store, "/buckets/victim/secret")
+	insertTestEntry(t, store, "/buckets/tenant/regular")
+
+	injection := "x\" && false || (FOR q IN `victim` LIMIT 1 UPDATE q WITH {injected:\"yes\"} IN `victim` RETURN \"\") || \""
+	if names := listNames(t, store, util.FullPath("/buckets/tenant"), injection); len(names) != 0 {
+		t.Errorf("expected no match for injected prefix, got %v", names)
+	}
+
+	var victim Model
+	collection, err := store.ensureBucket(context.Background(), "victim")
+	if err != nil {
+		t.Fatalf("ensure bucket victim: %v", err)
+	}
+	if _, err := collection.ReadDocument(context.Background(), hashString("/buckets/victim/secret"), &victim); err != nil {
+		t.Fatalf("read victim document: %v", err)
+	}
+	if victim.Name != "secret" {
+		t.Errorf("victim document was modified across buckets: %+v", victim)
+	}
+}
+
+// A prefix full of AQL operators must not be evaluated as a query.
+func TestListDirectoryPrefixedEntriesPrefixNotEvaluated(t *testing.T) {
+	store := newTestStore(t)
+
+	insertTestEntry(t, store, "/buckets/tenant/a")
+	insertTestEntry(t, store, "/buckets/tenant/b")
+	insertTestEntry(t, store, "/buckets/tenant/c")
+
+	start := time.Now()
+	if names := listNames(t, store, util.FullPath("/buckets/tenant"), "x\" && false || TO_STRING(SLEEP(1)) && false || \""); len(names) != 0 {
+		t.Errorf("expected no match for injected prefix, got %v", names)
+	}
+	if elapsed := time.Since(start); elapsed > time.Second {
+		t.Errorf("SLEEP() in prefix was executed, listing took %v", elapsed)
+	}
+}
+
+// Quotes and newlines are ordinary object-name characters and must round-trip.
+func TestListDirectoryPrefixedEntriesQuotedNames(t *testing.T) {
+	store := newTestStore(t)
+
+	names := []string{"plain", "quo\"te", "new\nline", "back\\slash"}
+	for _, name := range names {
+		insertTestEntry(t, store, "/buckets/tenant/"+name)
+	}
+
+	for _, name := range names {
+		if got := listNames(t, store, util.FullPath("/buckets/tenant"), name); len(got) != 1 || got[0] != name {
+			t.Errorf("prefix %q listed %v, want exactly [%q]", name, got, name)
+		}
+	}
+
+	if got := listNames(t, store, util.FullPath("/buckets/tenant"), ""); len(got) != len(names) {
+		t.Errorf("empty prefix listed %v, want %d entries", got, len(names))
+	}
+}
+
+// A directory name full of AQL operators must only drop that directory's own children.
+func TestDeleteFolderChildrenAqlInjection(t *testing.T) {
+	store := newTestStore(t)
+
+	for i := 0; i < 5; i++ {
+		insertTestEntry(t, store, fmt.Sprintf("/buckets/tenant/keep%d", i))
+	}
+	injection := "x\" || true || \""
+	insertTestEntry(t, store, "/buckets/tenant/"+injection+"/child")
+
+	if err := store.DeleteFolderChildren(context.Background(), util.FullPath("/buckets/tenant/"+injection)); err != nil {
+		t.Fatalf("delete folder children: %v", err)
+	}
+
+	if got := countDocuments(t, store, "tenant"); got != 5 {
+		t.Errorf("tenant collection holds %d documents, want the 5 unrelated ones", got)
+	}
+}
+
+func TestDeleteFolderChildrenRemovesSubtree(t *testing.T) {
+	store := newTestStore(t)
+
+	insertTestEntry(t, store, "/buckets/tenant/dir/file")
+	insertTestEntry(t, store, "/buckets/tenant/dir/sub/deep")
+	insertTestEntry(t, store, "/buckets/tenant/dirX/sibling")
+	insertTestEntry(t, store, "/buckets/tenant/other")
+
+	if err := store.DeleteFolderChildren(context.Background(), util.FullPath("/buckets/tenant/dir")); err != nil {
+		t.Fatalf("delete folder children: %v", err)
+	}
+
+	if got := countDocuments(t, store, "tenant"); got != 2 {
+		t.Errorf("tenant collection holds %d documents, want /buckets/tenant/dirX/sibling and /buckets/tenant/other", got)
+	}
+}
+
+// startFileName is request-controlled too and must not break out of the filter.
+func TestListDirectoryEntriesQuotedStartFileName(t *testing.T) {
+	store := newTestStore(t)
+
+	insertTestEntry(t, store, "/buckets/tenant/a")
+	insertTestEntry(t, store, "/buckets/tenant/z")
+
+	var names []string
+	_, err := store.ListDirectoryEntries(context.Background(), util.FullPath("/buckets/tenant"), "z\" || true || \"", false, 100,
+		func(entry *filer.Entry) (bool, error) {
+			names = append(names, entry.Name())
+			return true, nil
+		})
+	if err != nil {
+		t.Fatalf("list with quoted start file: %v", err)
+	}
+	if len(names) != 0 {
+		t.Errorf("listed %v, want nothing sorted after the literal start name", names)
+	}
+}

weed/filer/arangodb/helpers.go

@@ -0,0 +1,159 @@
+package arangodb
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/binary"
+	"encoding/hex"
+	"io"
+	"strings"
+
+	"github.com/arangodb/go-driver"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// convert a string into arango-key safe hex bytes hash
+func hashString(dir string) string {
+	h := md5.New()
+	io.WriteString(h, dir)
+	b := h.Sum(nil)
+	return hex.EncodeToString(b)
+}
+
+// convert slice of bytes into slice of uint64
+// the first uint64 indicates the length in bytes
+func bytesToArray(bs []byte) []uint64 {
+	out := make([]uint64, 0, 2+len(bs)/8)
+	out = append(out, uint64(len(bs)))
+	for len(bs)%8 != 0 {
+		bs = append(bs, 0)
+	}
+	for i := 0; i < len(bs); i = i + 8 {
+		out = append(out, binary.BigEndian.Uint64(bs[i:]))
+	}
+	return out
+}
+
+// convert from slice of uint64 back to bytes
+// if input length is 0 or 1, will return nil
+func arrayToBytes(xs []uint64) []byte {
+	if len(xs) < 2 {
+		return nil
+	}
+	first := xs[0]
+	out := make([]byte, len(xs)*8) // i think this can actually be len(xs)*8-8, but i dont think an extra 8 bytes hurts...
+	for i := 1; i < len(xs); i = i + 1 {
+		binary.BigEndian.PutUint64(out[((i-1)*8):], xs[i])
+	}
+	return out[:first]
+}
+
+// gets the collection the bucket points to from filepath
+func (store *ArangodbStore) extractBucketCollection(ctx context.Context, fullpath util.FullPath) (c driver.Collection, err error) {
+	bucket, _ := extractBucket(fullpath)
+	if bucket == "" {
+		bucket = DEFAULT_COLLECTION
+	}
+	c, err = store.ensureBucket(ctx, bucket)
+	if err != nil {
+		return nil, err
+	}
+	return c, err
+}
+
+// called by extractBucketCollection
+func extractBucket(fullpath util.FullPath) (string, string) {
+	if !strings.HasPrefix(string(fullpath), BUCKET_PREFIX+"/") {
+		return "", string(fullpath)
+	}
+	if strings.Count(string(fullpath), "/") < 3 {
+		return "", string(fullpath)
+	}
+	bucketAndObjectKey := string(fullpath)[len(BUCKET_PREFIX+"/"):]
+	t := strings.Index(bucketAndObjectKey, "/")
+	bucket := bucketAndObjectKey
+	shortPath := "/"
+	if t > 0 {
+		bucket = bucketAndObjectKey[:t]
+		shortPath = string(util.FullPath(bucketAndObjectKey[t:]))
+	}
+	// Dot-prefixed entries directly under /buckets (e.g. .system) are internal
+	// folders, not S3 buckets; keep them in the default collection.
+	if strings.HasPrefix(bucket, ".") {
+		return "", string(fullpath)
+	}
+	return bucket, shortPath
+}
+
+// get bucket collection from cache. if not exist, creates the buckets collection and grab it
+func (store *ArangodbStore) ensureBucket(ctx context.Context, bucket string) (bc driver.Collection, err error) {
+	var ok bool
+	store.mu.RLock()
+	bc, ok = store.buckets[bucket]
+	store.mu.RUnlock()
+	if ok && bc != nil {
+		return bc, nil
+	}
+	store.mu.Lock()
+	defer store.mu.Unlock()
+	store.buckets[bucket], err = store.ensureCollection(ctx, bucket)
+	if err != nil {
+		return nil, err
+	}
+	return store.buckets[bucket], nil
+}
+
+// transform to an arango compliant name
+func bucketToCollectionName(s string) string {
+	if len(s) == 0 {
+		return ""
+	}
+	// replace all "." with _
+	s = strings.ReplaceAll(s, ".", "_")
+
+	// if starts with number or '.' then add a special prefix
+	if (s[0] >= '0' && s[0] <= '9') || (s[0] == '.' || s[0] == '_' || s[0] == '-') {
+		s = "xN--" + s
+	}
+	return s
+}
+
+// creates collection if not exist, ensures indices if not exist
+func (store *ArangodbStore) ensureCollection(ctx context.Context, bucket_name string) (c driver.Collection, err error) {
+	// convert the bucket to collection name
+	name := bucketToCollectionName(bucket_name)
+
+	ok, err := store.database.CollectionExists(ctx, name)
+	if err != nil {
+		return
+	}
+	if ok {
+		c, err = store.database.Collection(ctx, name)
+	} else {
+		c, err = store.database.CreateCollection(ctx, name, &driver.CreateCollectionOptions{})
+	}
+	if err != nil {
+		return
+	}
+	// ensure indices
+	if _, _, err = c.EnsurePersistentIndex(ctx, []string{"directory", "name"},
+		&driver.EnsurePersistentIndexOptions{
+			Name: "directory_name_multi", Unique: true,
+		}); err != nil {
+		return
+	}
+	if _, _, err = c.EnsurePersistentIndex(ctx, []string{"directory"},
+		&driver.EnsurePersistentIndexOptions{Name: "IDX_directory"}); err != nil {
+		return
+	}
+	if _, _, err = c.EnsureTTLIndex(ctx, "ttl", 1,
+		&driver.EnsureTTLIndexOptions{Name: "IDX_TTL"}); err != nil {
+		return
+	}
+	if _, _, err = c.EnsurePersistentIndex(ctx, []string{"name"}, &driver.EnsurePersistentIndexOptions{
+		Name: "IDX_name",
+	}); err != nil {
+		return
+	}
+	return c, nil
+}

weed/filer/arangodb/readme.md

@@ -0,0 +1,85 @@
+##arangodb
+
+database: https://github.com/arangodb/arangodb
+go driver: https://github.com/arangodb/go-driver
+
+options:
+
+```
+[arangodb]
+enabled=true
+db_name="seaweedfs"
+servers=["http://localhost:8529"]
+#basic auth
+user="root"
+pass="test"
+
+# tls settings
+insecure_skip_verify=true
+```
+
+i test using this dev database:
+`docker run -p 8529:8529 -e ARANGO_ROOT_PASSWORD=test arangodb/arangodb:3.9.0`
+
+
+
+## database structure
+
+
+arangodb has a few restrictions which require the use of a few tricks in order to losslessly store the data.
+
+### filer store
+
+arangodb does not support []byte, and will store such as a uint64 array. this would be a waste of space. to counteract this, we store the data as a length prefixed uint64 byteset.
+
+### filer kv
+
+same as above
+
+### filer buckets
+
+s3 buckets are implemented through arangodb collection. this allows us to do very fast bucket deletion by simply deleting the collection
+
+
+arangodb collection name rules is character set `azAZ09_-` with a 256 character max. however the first character must be a letter.
+
+
+s3 bucket name rule is the set `azAZ09.-` with a 63 characters max.
+
+the rules for collection names is then the following:
+
+1. if the bucket name is a valid arangodb collection name, then nothing is done.
+2. if the bucket name contains a ".", the "." is replaced with "_"
+3. if the bucket name now begins with a number or "_", the prefix "xN--" is prepended to the collection name
+
+this allows for these collection names to be used.
+
+
+## features i don't personally need but are missing
+ [ ] provide tls cert to arango
+ [ ] authentication that is not basic auth
+ [ ] synchronise endpoint interval config
+ [ ] automatic creation of custom index
+ [ ] configure default arangodb collection sharding rules
+ [ ] configure default arangodb collection replication rules
+
+
+## complexity
+
+ok, so if https://www.arangodb.com/docs/stable/indexing-index-basics.html#persistent-index is correct
+
+O(1)
+- InsertEntry
+- UpdateEntry
+- FindEntry
+- DeleteEntry
+- KvPut
+- KvGet
+- KvDelete
+
+O(log(BUCKET_SIZE))
+- DeleteFolderChildren
+
+O(log(DIRECTORY_SIZE))
+- ListDirectoryEntries
+- ListDirectoryPrefixedEntries

weed/filer/cassandra/README.txt

@@ -0,0 +1 @@
+Deprecated by cassandra2

weed/filer/cassandra/cassandra_store.go

@@ -0,0 +1,233 @@
+package cassandra
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	gocql "github.com/apache/cassandra-gocql-driver/v2"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &CassandraStore{})
+}
+
+type CassandraStore struct {
+	cluster                 *gocql.ClusterConfig
+	session                 *gocql.Session
+	superLargeDirectoryHash map[string]string
+}
+
+func (store *CassandraStore) GetName() string {
+	return "cassandra"
+}
+
+func (store *CassandraStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	return store.initialize(
+		configuration.GetString(prefix+"keyspace"),
+		configuration.GetStringSlice(prefix+"hosts"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetStringSlice(prefix+"superLargeDirectories"),
+		configuration.GetString(prefix+"localDC"),
+		configuration.GetInt(prefix+"connection_timeout_millisecond"),
+	)
+}
+
+func (store *CassandraStore) isSuperLargeDirectory(dir string) (dirHash string, isSuperLargeDirectory bool) {
+	dirHash, isSuperLargeDirectory = store.superLargeDirectoryHash[dir]
+	return
+}
+
+func (store *CassandraStore) initialize(keyspace string, hosts []string, username string, password string, superLargeDirectories []string, localDC string, timeout int) (err error) {
+	store.cluster = gocql.NewCluster(hosts...)
+	if username != "" && password != "" {
+		store.cluster.Authenticator = gocql.PasswordAuthenticator{Username: username, Password: password}
+	}
+	store.cluster.Keyspace = keyspace
+	store.cluster.Timeout = time.Duration(timeout) * time.Millisecond
+	glog.V(0).Infof("timeout = %d", timeout)
+	fallback := gocql.RoundRobinHostPolicy()
+	if localDC != "" {
+		fallback = gocql.DCAwareRoundRobinPolicy(localDC)
+	}
+	store.cluster.PoolConfig.HostSelectionPolicy = gocql.TokenAwareHostPolicy(fallback)
+	store.cluster.Consistency = gocql.LocalQuorum
+
+	store.session, err = store.cluster.CreateSession()
+	if err != nil {
+		glog.V(0).Infof("Failed to open cassandra store, hosts %v, keyspace %s", hosts, keyspace)
+	}
+
+	// set directory hash
+	store.superLargeDirectoryHash = make(map[string]string)
+	existingHash := make(map[string]string)
+	for _, dir := range superLargeDirectories {
+		// adding dir hash to avoid duplicated names
+		dirHash := util.Md5String([]byte(dir))[:4]
+		store.superLargeDirectoryHash[dir] = dirHash
+		if existingDir, found := existingHash[dirHash]; found {
+			glog.Fatalf("directory %s has the same hash as %s", dir, existingDir)
+		}
+		existingHash[dirHash] = dir
+	}
+	return
+}
+
+func (store *CassandraStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *CassandraStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *CassandraStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *CassandraStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	dir, name := entry.FullPath.DirAndName()
+	if dirHash, ok := store.isSuperLargeDirectory(dir); ok {
+		dir, name = dirHash+name, ""
+	}
+
+	meta, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
+	}
+
+	if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+		meta = util.MaybeGzipData(meta)
+	}
+
+	if err := store.session.Query(
+		"INSERT INTO filemeta (directory,name,meta) VALUES(?,?,?) USING TTL ? ",
+		dir, name, meta, entry.TtlSec).Exec(); err != nil {
+		return fmt.Errorf("insert %s: %s", entry.FullPath, err)
+	}
+
+	return nil
+}
+
+func (store *CassandraStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *CassandraStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+
+	dir, name := fullpath.DirAndName()
+	if dirHash, ok := store.isSuperLargeDirectory(dir); ok {
+		dir, name = dirHash+name, ""
+	}
+
+	var data []byte
+	if err := store.session.Query(
+		"SELECT meta FROM filemeta WHERE directory=? AND name=?",
+		dir, name).Scan(&data); err != nil {
+		if errors.Is(err, gocql.ErrNotFound) {
+			return nil, filer_pb.ErrNotFound
+		}
+		return nil, err
+	}
+
+	entry = &filer.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *CassandraStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {
+
+	dir, name := fullpath.DirAndName()
+	if dirHash, ok := store.isSuperLargeDirectory(dir); ok {
+		dir, name = dirHash+name, ""
+	}
+
+	if err := store.session.Query(
+		"DELETE FROM filemeta WHERE directory=? AND name=?",
+		dir, name).Exec(); err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *CassandraStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
+	if _, ok := store.isSuperLargeDirectory(string(fullpath)); ok {
+		return nil // filer.ErrUnsupportedSuperLargeDirectoryListing
+	}
+
+	if err := store.session.Query(
+		"DELETE FROM filemeta WHERE directory=?",
+		fullpath).Exec(); err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *CassandraStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed
+}
+
+func (store *CassandraStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+
+	if _, ok := store.isSuperLargeDirectory(string(dirPath)); ok {
+		return // nil, filer.ErrUnsupportedSuperLargeDirectoryListing
+	}
+
+	cqlStr := "SELECT NAME, meta FROM filemeta WHERE directory=? AND name>? ORDER BY NAME ASC LIMIT ?"
+	if includeStartFile {
+		cqlStr = "SELECT NAME, meta FROM filemeta WHERE directory=? AND name>=? ORDER BY NAME ASC LIMIT ?"
+	}
+
+	var data []byte
+	var name string
+	iter := store.session.Query(cqlStr, string(dirPath), startFileName, limit+1).Iter()
+	for iter.Scan(&name, &data) {
+		entry := &filer.Entry{
+			FullPath: util.NewFullPath(string(dirPath), name),
+		}
+		lastFileName = name
+		if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); decodeErr != nil {
+			err = decodeErr
+			glog.V(0).InfofCtx(ctx, "list %s : %v", entry.FullPath, err)
+			break
+		}
+
+		resEachEntryFunc, resEachEntryFuncErr := eachEntryFunc(entry)
+		if resEachEntryFuncErr != nil {
+			err = fmt.Errorf("failed to process eachEntryFunc for entry %q: %w", entry.FullPath, resEachEntryFuncErr)
+			glog.V(0).InfofCtx(ctx, "failed to process eachEntryFunc for entry %q: %v", entry.FullPath, resEachEntryFuncErr)
+			break
+		}
+
+		if !resEachEntryFunc {
+			break
+		}
+	}
+	if errClose := iter.Close(); errClose != nil {
+		glog.V(0).InfofCtx(ctx, "list iterator close: %v", errClose)
+		if err == nil {
+			return lastFileName, errClose
+		}
+	}
+
+	return lastFileName, err
+}
+
+func (store *CassandraStore) Shutdown() {
+	store.session.Close()
+}

weed/filer/cassandra/cassandra_store_kv.go

@@ -0,0 +1,65 @@
+package cassandra
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+
+	gocql "github.com/apache/cassandra-gocql-driver/v2"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+)
+
+func (store *CassandraStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+	dir, name := genDirAndName(key)
+
+	if err := store.session.Query(
+		"INSERT INTO filemeta (directory,name,meta) VALUES(?,?,?) USING TTL ? ",
+		dir, name, value, 0).Exec(); err != nil {
+		return fmt.Errorf("kv insert: %s", err)
+	}
+
+	return nil
+}
+
+func (store *CassandraStore) KvGet(ctx context.Context, key []byte) (data []byte, err error) {
+	dir, name := genDirAndName(key)
+
+	if err := store.session.Query(
+		"SELECT meta FROM filemeta WHERE directory=? AND name=?",
+		dir, name).Scan(&data); err != nil {
+		if errors.Is(err, gocql.ErrNotFound) {
+			return nil, filer.ErrKvNotFound
+		}
+		return nil, fmt.Errorf("kv get: %w", err)
+	}
+
+	if len(data) == 0 {
+		return nil, filer.ErrKvNotFound
+	}
+
+	return data, nil
+}
+
+func (store *CassandraStore) KvDelete(ctx context.Context, key []byte) (err error) {
+	dir, name := genDirAndName(key)
+
+	if err := store.session.Query(
+		"DELETE FROM filemeta WHERE directory=? AND name=?",
+		dir, name).Exec(); err != nil {
+		return fmt.Errorf("kv delete: %w", err)
+	}
+
+	return nil
+}
+
+func genDirAndName(key []byte) (dir string, name string) {
+	for len(key) < 8 {
+		key = append(key, 0)
+	}
+
+	dir = base64.StdEncoding.EncodeToString(key[:8])
+	name = base64.StdEncoding.EncodeToString(key[8:])
+
+	return
+}

weed/filer/cassandra2/README.txt

@@ -0,0 +1,15 @@
+1. create a keyspace
+
+CREATE KEYSPACE seaweedfs WITH replication = {'class':'SimpleStrategy', 'replication_factor' : 1};
+
+2. create filemeta table
+
+ USE seaweedfs;
+
+ CREATE TABLE filemeta (
+    dirhash bigint,
+    directory varchar,
+    name varchar,
+    meta blob,
+    PRIMARY KEY ((dirhash, directory), name)
+ ) WITH CLUSTERING ORDER BY (name ASC);

weed/filer/cassandra2/cassandra_store.go

@@ -0,0 +1,286 @@
+package cassandra2
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	gocql "github.com/apache/cassandra-gocql-driver/v2"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &Cassandra2Store{})
+}
+
+type Cassandra2Store struct {
+	cluster                 *gocql.ClusterConfig
+	session                 *gocql.Session
+	superLargeDirectoryHash map[string]string
+}
+
+func (store *Cassandra2Store) GetName() string {
+	return "cassandra2"
+}
+
+func (store *Cassandra2Store) Initialize(configuration util.Configuration, prefix string) (err error) {
+	// Without this, an env-var config that omits the key drops cluster.Timeout to 0.
+	configuration.SetDefault(prefix+"connection_timeout_millisecond", 600)
+	enableHostVerification := true
+	if val := configuration.GetString(prefix + "ssl_enable_host_verification"); val != "" {
+		enableHostVerification = configuration.GetBool(prefix + "ssl_enable_host_verification")
+	}
+
+	return store.initialize(
+		configuration.GetString(prefix+"keyspace"),
+		configuration.GetStringSlice(prefix+"hosts"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetString(prefix+"ssl_ca_path"),
+		configuration.GetString(prefix+"ssl_cert_path"),
+		configuration.GetString(prefix+"ssl_key_path"),
+		configuration.GetStringSlice(prefix+"superLargeDirectories"),
+		configuration.GetString(prefix+"localDC"),
+		configuration.GetInt(prefix+"connection_timeout_millisecond"),
+		enableHostVerification,
+	)
+}
+
+func (store *Cassandra2Store) isSuperLargeDirectory(dir string) (dirHash string, isSuperLargeDirectory bool) {
+	dirHash, isSuperLargeDirectory = store.superLargeDirectoryHash[dir]
+	return
+}
+
+func (store *Cassandra2Store) initialize(keyspace string, hosts []string, username string, password string, sslCaPath string, sslCertPath string, sslKeyPath string, superLargeDirectories []string, localDC string, timeout int, enableHostVerification bool) (err error) {
+	store.cluster = gocql.NewCluster(hosts...)
+	if username != "" && password != "" {
+		store.cluster.Authenticator = gocql.PasswordAuthenticator{Username: username, Password: password}
+	}
+	if sslCaPath != "" || sslCertPath != "" || sslKeyPath != "" {
+		if (sslCertPath != "" && sslKeyPath == "") || (sslCertPath == "" && sslKeyPath != "") {
+			return fmt.Errorf("both ssl_cert_path and ssl_key_path must be provided for mTLS, or neither")
+		}
+
+		for _, path := range []string{sslCaPath, sslCertPath, sslKeyPath} {
+			if path != "" {
+				if _, err := os.Stat(path); err != nil {
+					return fmt.Errorf("ssl file %s not found: %v", path, err)
+				}
+			}
+		}
+
+		store.cluster.SslOpts = &gocql.SslOptions{
+			CaPath:                 sslCaPath,
+			CertPath:               sslCertPath,
+			KeyPath:                sslKeyPath,
+			EnableHostVerification: enableHostVerification,
+		}
+
+		// check if port is already specified in hosts
+		hasPort := false
+		for _, host := range hosts {
+			if strings.Contains(host, ":") {
+				hasPort = true
+				break
+			}
+		}
+		if !hasPort {
+			// standard cassandra port is 9042, but AWS keyspaces uses 9142
+			store.cluster.Port = 9142
+		}
+
+		if sslCertPath != "" {
+			glog.V(0).Infof("TLS enabled: mTLS with cert %s", sslCertPath)
+		} else {
+			glog.V(0).Infof("TLS enabled: server-verification with ca %s", sslCaPath)
+		}
+	}
+	store.cluster.Keyspace = keyspace
+	store.cluster.Timeout = time.Duration(timeout) * time.Millisecond
+	glog.V(0).Infof("timeout = %d", timeout)
+	fallback := gocql.RoundRobinHostPolicy()
+	if localDC != "" {
+		fallback = gocql.DCAwareRoundRobinPolicy(localDC)
+	}
+	store.cluster.PoolConfig.HostSelectionPolicy = gocql.TokenAwareHostPolicy(fallback)
+	store.cluster.Consistency = gocql.LocalQuorum
+
+	store.session, err = store.cluster.CreateSession()
+	if err != nil {
+		glog.V(0).Infof("Failed to open cassandra2 store, hosts %v, keyspace %s", hosts, keyspace)
+	}
+
+	// set directory hash
+	store.superLargeDirectoryHash = make(map[string]string)
+	existingHash := make(map[string]string)
+	for _, dir := range superLargeDirectories {
+		// adding dir hash to avoid duplicated names
+		dirHash := util.Md5String([]byte(dir))[:4]
+		store.superLargeDirectoryHash[dir] = dirHash
+		if existingDir, found := existingHash[dirHash]; found {
+			glog.Fatalf("directory %s has the same hash as %s", dir, existingDir)
+		}
+		existingHash[dirHash] = dir
+	}
+	return
+}
+
+func (store *Cassandra2Store) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *Cassandra2Store) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *Cassandra2Store) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *Cassandra2Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	dir, name := entry.FullPath.DirAndName()
+	if dirHash, ok := store.isSuperLargeDirectory(dir); ok {
+		dir, name = dirHash+name, ""
+	}
+
+	meta, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
+	}
+
+	if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+		meta = util.MaybeGzipData(meta)
+	}
+
+	if err := store.session.Query(
+		"INSERT INTO filemeta (dirhash,directory,name,meta) VALUES(?,?,?,?) USING TTL ? ",
+		util.HashStringToLong(dir), dir, name, meta, entry.TtlSec).Exec(); err != nil {
+		return fmt.Errorf("insert %s: %s", entry.FullPath, err)
+	}
+
+	return nil
+}
+
+func (store *Cassandra2Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *Cassandra2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+
+	dir, name := fullpath.DirAndName()
+	if dirHash, ok := store.isSuperLargeDirectory(dir); ok {
+		dir, name = dirHash+name, ""
+	}
+
+	var data []byte
+	if err := store.session.Query(
+		"SELECT meta FROM filemeta WHERE dirhash=? AND directory=? AND name=?",
+		util.HashStringToLong(dir), dir, name).Scan(&data); err != nil {
+		if errors.Is(err, gocql.ErrNotFound) {
+			return nil, filer_pb.ErrNotFound
+		}
+		return nil, err
+	}
+
+	entry = &filer.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *Cassandra2Store) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {
+
+	dir, name := fullpath.DirAndName()
+	if dirHash, ok := store.isSuperLargeDirectory(dir); ok {
+		dir, name = dirHash+name, ""
+	}
+
+	if err := store.session.Query(
+		"DELETE FROM filemeta WHERE dirhash=? AND directory=? AND name=?",
+		util.HashStringToLong(dir), dir, name).Exec(); err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *Cassandra2Store) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
+	if _, ok := store.isSuperLargeDirectory(string(fullpath)); ok {
+		return nil // filer.ErrUnsupportedSuperLargeDirectoryListing
+	}
+
+	if err := store.session.Query(
+		"DELETE FROM filemeta WHERE dirhash=? AND directory=?",
+		util.HashStringToLong(string(fullpath)), fullpath).Exec(); err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *Cassandra2Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed
+}
+
+func (store *Cassandra2Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+
+	if _, ok := store.isSuperLargeDirectory(string(dirPath)); ok {
+		return // nil, filer.ErrUnsupportedSuperLargeDirectoryListing
+	}
+
+	cqlStr := "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND directory=? AND name>? ORDER BY NAME ASC LIMIT ?"
+	if includeStartFile {
+		cqlStr = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND directory=? AND name>=? ORDER BY NAME ASC LIMIT ?"
+	}
+
+	var data []byte
+	var name string
+	iter := store.session.Query(cqlStr, util.HashStringToLong(string(dirPath)), string(dirPath), startFileName, limit+1).Iter()
+	for iter.Scan(&name, &data) {
+		entry := &filer.Entry{
+			FullPath: util.NewFullPath(string(dirPath), name),
+		}
+		lastFileName = name
+		if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); decodeErr != nil {
+			err = decodeErr
+			glog.V(0).InfofCtx(ctx, "list %s : %v", entry.FullPath, err)
+			break
+		}
+
+		resEachEntryFunc, resEachEntryFuncErr := eachEntryFunc(entry)
+		if resEachEntryFuncErr != nil {
+			err = fmt.Errorf("failed to process eachEntryFunc for entry %q: %w", entry.FullPath, resEachEntryFuncErr)
+			glog.V(0).InfofCtx(ctx, "failed to process eachEntryFunc for entry %q: %v", entry.FullPath, resEachEntryFuncErr)
+			break
+		}
+
+		if !resEachEntryFunc {
+			break
+		}
+	}
+
+	if errClose := iter.Close(); errClose != nil {
+		glog.V(0).InfofCtx(ctx, "list iterator close: %v", errClose)
+		if err == nil {
+			return lastFileName, errClose
+		}
+	}
+
+	return lastFileName, err
+}
+
+func (store *Cassandra2Store) Shutdown() {
+	store.session.Close()
+}

weed/filer/cassandra2/cassandra_store_kv.go

@@ -0,0 +1,66 @@
+package cassandra2
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+
+	gocql "github.com/apache/cassandra-gocql-driver/v2"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func (store *Cassandra2Store) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+	dir, name := genDirAndName(key)
+
+	if err := store.session.Query(
+		"INSERT INTO filemeta (dirhash,directory,name,meta) VALUES(?,?,?,?) USING TTL ? ",
+		util.HashStringToLong(dir), dir, name, value, 0).Exec(); err != nil {
+		return fmt.Errorf("kv insert: %s", err)
+	}
+
+	return nil
+}
+
+func (store *Cassandra2Store) KvGet(ctx context.Context, key []byte) (data []byte, err error) {
+	dir, name := genDirAndName(key)
+
+	if err := store.session.Query(
+		"SELECT meta FROM filemeta WHERE dirhash=? AND directory=? AND name=?",
+		util.HashStringToLong(dir), dir, name).Scan(&data); err != nil {
+		if errors.Is(err, gocql.ErrNotFound) {
+			return nil, filer.ErrKvNotFound
+		}
+		return nil, fmt.Errorf("kv get: %w", err)
+	}
+
+	if len(data) == 0 {
+		return nil, filer.ErrKvNotFound
+	}
+
+	return data, nil
+}
+
+func (store *Cassandra2Store) KvDelete(ctx context.Context, key []byte) (err error) {
+	dir, name := genDirAndName(key)
+
+	if err := store.session.Query(
+		"DELETE FROM filemeta WHERE dirhash=? AND directory=? AND name=?",
+		util.HashStringToLong(dir), dir, name).Exec(); err != nil {
+		return fmt.Errorf("kv delete: %w", err)
+	}
+
+	return nil
+}
+
+func genDirAndName(key []byte) (dir string, name string) {
+	for len(key) < 8 {
+		key = append(key, 0)
+	}
+
+	dir = base64.StdEncoding.EncodeToString(key[:8])
+	name = base64.StdEncoding.EncodeToString(key[8:])
+
+	return
+}

weed/filer/configuration.go

@@ -0,0 +1,100 @@
+package filer
+
+import (
+	"os"
+	"reflect"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var (
+	Stores []FilerStore
+)
+
+func (f *Filer) LoadConfiguration(config *util.ViperProxy) (isFresh bool) {
+
+	validateOneEnabledStore(config)
+
+	// load configuration for default filer store
+	hasDefaultStoreConfigured := false
+	for _, store := range Stores {
+		if config.GetBool(store.GetName() + ".enabled") {
+			store = reflect.New(reflect.ValueOf(store).Elem().Type()).Interface().(FilerStore)
+			if err := store.Initialize(config, store.GetName()+"."); err != nil {
+				glog.Fatalf("failed to initialize store for %s: %+v", store.GetName(), err)
+			}
+			isFresh = f.SetStore(store)
+			glog.V(0).Infof("configured filer store to %s", store.GetName())
+			hasDefaultStoreConfigured = true
+			break
+		}
+	}
+
+	if !hasDefaultStoreConfigured {
+		println()
+		println("Supported filer stores are the following. If not found, check the full version.")
+		for _, store := range Stores {
+			println("    " + store.GetName())
+		}
+		os.Exit(-1)
+	}
+
+	// load path-specific filer store here
+	// f.Store.AddPathSpecificStore(path, store)
+	storeNames := make(map[string]FilerStore)
+	for _, store := range Stores {
+		storeNames[store.GetName()] = store
+	}
+	allKeys := config.AllKeys()
+	for _, key := range allKeys {
+		if !strings.HasSuffix(key, ".enabled") {
+			continue
+		}
+		key = key[:len(key)-len(".enabled")]
+		if !strings.Contains(key, ".") {
+			continue
+		}
+
+		parts := strings.Split(key, ".")
+		storeName, storeId := parts[0], parts[1]
+
+		store, found := storeNames[storeName]
+		if !found {
+			continue
+		}
+
+		if !config.GetBool(key + ".enabled") {
+			continue
+		}
+
+		store = reflect.New(reflect.ValueOf(store).Elem().Type()).Interface().(FilerStore)
+		if err := store.Initialize(config, key+"."); err != nil {
+			glog.Fatalf("Failed to initialize store for %s: %+v", key, err)
+		}
+		location := config.GetString(key + ".location")
+		if location == "" {
+			glog.Errorf("path-specific filer store needs %s", key+".location")
+			os.Exit(-1)
+		}
+		f.Store.AddPathSpecificStore(location, storeId, store)
+
+		glog.V(0).Infof("configure filer %s for %s", store.GetName(), location)
+	}
+
+	return
+}
+
+func validateOneEnabledStore(config *util.ViperProxy) {
+	enabledStore := ""
+	for _, store := range Stores {
+		if config.GetBool(store.GetName() + ".enabled") {
+			if enabledStore == "" {
+				enabledStore = store.GetName()
+			} else {
+				glog.Fatalf("Filer store is enabled for both %s and %s", enabledStore, store.GetName())
+			}
+		}
+	}
+}

weed/filer/copy_params.go

@@ -0,0 +1,17 @@
+package filer
+
+const (
+	CopyQueryParamFrom             = "cp.from"
+	CopyQueryParamOverwrite        = "overwrite"
+	CopyQueryParamDataOnly         = "dataOnly"
+	CopyQueryParamRequestID        = "copy.requestId"
+	CopyQueryParamSourceInode      = "copy.srcInode"
+	CopyQueryParamSourceMtime      = "copy.srcMtime"
+	CopyQueryParamSourceSize       = "copy.srcSize"
+	CopyQueryParamDestinationInode = "copy.dstInode"
+	CopyQueryParamDestinationMtime = "copy.dstMtime"
+	CopyQueryParamDestinationSize  = "copy.dstSize"
+
+	CopyResponseHeaderCommitted = "X-SeaweedFS-Copy-Committed"
+	CopyResponseHeaderRequestID = "X-SeaweedFS-Copy-Request-ID"
+)

weed/filer/elastic/v7/doc.go

@@ -0,0 +1,7 @@
+/*
+Package elastic is for elastic filer store.
+
+The referenced "github.com/olivere/elastic/v7" library is too big when compiled.
+So this is only compiled in "make full_install".
+*/
+package elastic

weed/filer/elastic/v7/elastic_store.go

@@ -0,0 +1,356 @@
+//go:build elastic
+
+package elastic
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"strings"
+
+	jsoniter "github.com/json-iterator/go"
+	elastic "github.com/olivere/elastic/v7"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	weed_util "github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var (
+	indexType   = "_doc"
+	indexPrefix = ".seaweedfs_"
+	indexKV     = ".seaweedfs_kv_entries"
+	kvMappings  = ` {
+		     "mappings": {
+		   	"enabled": false,
+		       "properties": {
+		         "Value":{
+		           "type": "binary"
+		         }
+		       }
+		     }
+		   }`
+)
+
+type ESEntry struct {
+	ParentId string `json:"ParentId"`
+	Id       string `json:"Id,omitempty"`
+	Entry    *filer.Entry
+}
+
+type ESKVEntry struct {
+	Value []byte `json:"Value"`
+}
+
+func init() {
+	filer.Stores = append(filer.Stores, &ElasticStore{})
+	filer.Stores = append(filer.Stores, &Elastic8Store{})
+}
+
+type ElasticStore struct {
+	client      *elastic.Client
+	maxPageSize int
+	es8         bool
+}
+
+func (store *ElasticStore) GetName() string {
+	return "elastic7"
+}
+
+// Elastic8Store sorts listings on an indexed Id field since Elasticsearch 8 disallows _id fielddata.
+type Elastic8Store struct {
+	ElasticStore
+}
+
+func (store *Elastic8Store) GetName() string {
+	return "elastic8"
+}
+
+func (store *Elastic8Store) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
+	store.es8 = true
+	return store.ElasticStore.Initialize(configuration, prefix)
+}
+
+func (store *ElasticStore) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
+	options := []elastic.ClientOptionFunc{}
+	servers := configuration.GetStringSlice(prefix + "servers")
+	options = append(options, elastic.SetURL(servers...))
+	username := configuration.GetString(prefix + "username")
+	password := configuration.GetString(prefix + "password")
+	if username != "" && password != "" {
+		options = append(options, elastic.SetBasicAuth(username, password))
+	}
+	options = append(options, elastic.SetSniff(configuration.GetBool(prefix+"sniff_enabled")))
+	options = append(options, elastic.SetHealthcheck(configuration.GetBool(prefix+"healthcheck_enabled")))
+	store.maxPageSize = configuration.GetInt(prefix + "index.max_result_window")
+	if store.maxPageSize <= 0 {
+		store.maxPageSize = 10000
+	}
+	glog.Infof("filer store elastic endpoints: %v.", servers)
+	return store.initialize(options)
+}
+
+func (store *ElasticStore) initialize(options []elastic.ClientOptionFunc) (err error) {
+	ctx := context.Background()
+	store.client, err = elastic.NewClient(options...)
+	if err != nil {
+		return fmt.Errorf("init elastic %w", err)
+	}
+	if ok, err := store.client.IndexExists(indexKV).Do(ctx); err == nil && !ok {
+		_, err = store.client.CreateIndex(indexKV).Body(kvMappings).Do(ctx)
+		if err != nil {
+			return fmt.Errorf("create index(%s) %v", indexKV, err)
+		}
+	}
+	return nil
+}
+
+func (store *ElasticStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *ElasticStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *ElasticStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *ElasticStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed
+}
+
+func (store *ElasticStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	index := getIndex(entry.FullPath, false)
+	dir, _ := entry.FullPath.DirAndName()
+	id := weed_util.Md5String([]byte(entry.FullPath))
+	esEntry := &ESEntry{
+		ParentId: weed_util.Md5String([]byte(dir)),
+		Entry:    entry,
+	}
+	if store.es8 {
+		esEntry.Id = id
+	}
+	value, err := jsoniter.Marshal(esEntry)
+	if err != nil {
+		glog.ErrorfCtx(ctx, "insert entry(%s) %v.", string(entry.FullPath), err)
+		return fmt.Errorf("insert entry marshal %w", err)
+	}
+	_, err = store.client.Index().
+		Index(index).
+		Type(indexType).
+		Id(id).
+		BodyJson(string(value)).
+		Do(ctx)
+	if err != nil {
+		glog.ErrorfCtx(ctx, "insert entry(%s) %v.", string(entry.FullPath), err)
+		return fmt.Errorf("insert entry %w", err)
+	}
+	return nil
+}
+
+func (store *ElasticStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *ElasticStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) {
+	index := getIndex(fullpath, false)
+	id := weed_util.Md5String([]byte(fullpath))
+	searchResult, err := store.client.Get().
+		Index(index).
+		Type(indexType).
+		Id(id).
+		Do(ctx)
+	if elastic.IsNotFound(err) {
+		return nil, filer_pb.ErrNotFound
+	}
+	if searchResult != nil && searchResult.Found {
+		esEntry := &ESEntry{
+			ParentId: "",
+			Entry:    &filer.Entry{},
+		}
+		err := jsoniter.Unmarshal(searchResult.Source, esEntry)
+		return esEntry.Entry, err
+	}
+	glog.ErrorfCtx(ctx, "find entry(%s),%v.", string(fullpath), err)
+	return nil, filer_pb.ErrNotFound
+}
+
+func (store *ElasticStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
+	index := getIndex(fullpath, false)
+	id := weed_util.Md5String([]byte(fullpath))
+	strFullpath := string(fullpath)
+
+	// A top-level subdirectory refers to an Elasticsearch index.
+	// If we delete an entry at the top level, we should attempt to delete the corresponding Elasticsearch index.
+	if strings.Count(strFullpath, "/") == 1 {
+		entry, err2 := store.FindEntry(ctx, fullpath)
+		if err2 == nil && entry.IsDirectory() {
+			bucketIndex := indexPrefix + strFullpath[1:]
+			store.deleteIndex(ctx, bucketIndex)
+		}
+	}
+	return store.deleteEntry(ctx, index, id)
+}
+
+func (store *ElasticStore) deleteIndex(ctx context.Context, index string) (err error) {
+	deleteResult, err := store.client.DeleteIndex(index).Do(ctx)
+	if elastic.IsNotFound(err) || (err == nil && deleteResult.Acknowledged) {
+		return nil
+	}
+	glog.ErrorfCtx(ctx, "delete index(%s) %v.", index, err)
+	return err
+}
+
+func (store *ElasticStore) deleteEntry(ctx context.Context, index, id string) (err error) {
+	deleteResult, err := store.client.Delete().
+		Index(index).
+		Type(indexType).
+		Id(id).
+		Do(ctx)
+	if err == nil {
+		if deleteResult.Result == "deleted" || deleteResult.Result == "not_found" {
+			return nil
+		}
+	}
+	glog.ErrorfCtx(ctx, "delete entry(index:%s,_id:%s) %v.", index, id, err)
+	return fmt.Errorf("delete entry %w", err)
+}
+
+func (store *ElasticStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
+	_, err = store.ListDirectoryEntries(ctx, fullpath, "", false, math.MaxInt32, func(entry *filer.Entry) (bool, error) {
+		if err := store.DeleteEntry(ctx, entry.FullPath); err != nil {
+			glog.ErrorfCtx(ctx, "elastic delete %s: %v.", entry.FullPath, err)
+			return false, err
+		}
+		return true, nil
+	})
+	return
+}
+
+func (store *ElasticStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return store.listDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, eachEntryFunc)
+}
+
+func (store *ElasticStore) listDirectoryEntries(
+	ctx context.Context, fullpath weed_util.FullPath, startFileName string, inclusive bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	first := true
+	index := getIndex(fullpath, true)
+	nextStart := ""
+	parentId := weed_util.Md5String([]byte(fullpath))
+	if _, err = store.client.Refresh(index).Do(ctx); err != nil {
+		if elastic.IsNotFound(err) {
+			if _, err := store.client.CreateIndex(index).Do(ctx); err != nil {
+				return lastFileName, fmt.Errorf("create index(%s) %v", index, err)
+			}
+			return lastFileName, nil
+		}
+	}
+	for {
+		result := &elastic.SearchResult{}
+		if (startFileName == "" && first) || inclusive {
+			if result, err = store.search(ctx, index, parentId); err != nil {
+				glog.ErrorfCtx(ctx, "search (%s,%s,%t,%d) %v.", string(fullpath), startFileName, inclusive, limit, err)
+				return
+			}
+		} else {
+			fullPath := string(fullpath) + "/" + startFileName
+			if !first {
+				fullPath = nextStart
+			}
+			after := weed_util.Md5String([]byte(fullPath))
+			if result, err = store.searchAfter(ctx, index, parentId, after); err != nil {
+				glog.ErrorfCtx(ctx, "searchAfter (%s,%s,%t,%d) %v.", string(fullpath), startFileName, inclusive, limit, err)
+				return
+			}
+		}
+		first = false
+		for _, hit := range result.Hits.Hits {
+			esEntry := &ESEntry{
+				ParentId: "",
+				Entry:    &filer.Entry{},
+			}
+			if err := jsoniter.Unmarshal(hit.Source, esEntry); err == nil {
+				limit--
+				if limit < 0 {
+					return lastFileName, nil
+				}
+				nextStart = string(esEntry.Entry.FullPath)
+				fileName := esEntry.Entry.FullPath.Name()
+				if fileName == startFileName && !inclusive {
+					continue
+				}
+
+				resEachEntryFunc, resEachEntryFuncErr := eachEntryFunc(esEntry.Entry)
+				if resEachEntryFuncErr != nil {
+					glog.ErrorfCtx(ctx, "failed to process eachEntryFunc for entry %q: %v", fileName, resEachEntryFuncErr)
+					return lastFileName, fmt.Errorf("failed to process eachEntryFunc for entry %q: %w", fileName, resEachEntryFuncErr)
+				}
+
+				if !resEachEntryFunc {
+					return lastFileName, nil
+				}
+
+				lastFileName = fileName
+			}
+		}
+		if len(result.Hits.Hits) < store.maxPageSize {
+			break
+		}
+	}
+	return
+}
+
+func (store *ElasticStore) listSorter() elastic.Sorter {
+	field := "_id"
+	if store.es8 {
+		field = "Id.keyword"
+	}
+	// unmapped_type tolerates indexes with no documents yet
+	return elastic.NewFieldSort(field).Desc().UnmappedType("keyword")
+}
+
+func (store *ElasticStore) search(ctx context.Context, index, parentId string) (result *elastic.SearchResult, err error) {
+	if count, err := store.client.Count(index).Do(ctx); err == nil && count == 0 {
+		return &elastic.SearchResult{
+			Hits: &elastic.SearchHits{
+				Hits: make([]*elastic.SearchHit, 0)},
+		}, nil
+	}
+	queryResult, err := store.client.Search().
+		Index(index).
+		Query(elastic.NewMatchQuery("ParentId", parentId)).
+		Size(store.maxPageSize).
+		SortBy(store.listSorter()).
+		Do(ctx)
+	return queryResult, err
+}
+
+func (store *ElasticStore) searchAfter(ctx context.Context, index, parentId, after string) (result *elastic.SearchResult, err error) {
+	queryResult, err := store.client.Search().
+		Index(index).
+		Query(elastic.NewMatchQuery("ParentId", parentId)).
+		SearchAfter(after).
+		Size(store.maxPageSize).
+		SortBy(store.listSorter()).
+		Do(ctx)
+	return queryResult, err
+
+}
+
+func (store *ElasticStore) Shutdown() {
+	store.client.Stop()
+}
+
+func getIndex(fullpath weed_util.FullPath, isDirectory bool) string {
+	path := strings.Split(string(fullpath), "/")
+	if isDirectory && len(path) >= 2 {
+		return indexPrefix + strings.ToLower(path[1])
+	}
+	if len(path) > 2 {
+		return indexPrefix + strings.ToLower(path[1])
+	}
+	if len(path) == 2 {
+		return indexPrefix
+	}
+	return ""
+}

weed/filer/elastic/v7/elastic_store_kv.go

@@ -0,0 +1,67 @@
+//go:build elastic
+
+package elastic
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+
+	jsoniter "github.com/json-iterator/go"
+	elastic "github.com/olivere/elastic/v7"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+func (store *ElasticStore) KvDelete(ctx context.Context, key []byte) (err error) {
+	deleteResult, err := store.client.Delete().
+		Index(indexKV).
+		Type(indexType).
+		Id(string(key)).
+		Do(ctx)
+	if err == nil {
+		if deleteResult.Result == "deleted" || deleteResult.Result == "not_found" {
+			return nil
+		}
+	}
+	glog.ErrorfCtx(ctx, "delete key(id:%s) %v.", string(key), err)
+	return fmt.Errorf("delete key %w", err)
+}
+
+func (store *ElasticStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+	searchResult, err := store.client.Get().
+		Index(indexKV).
+		Type(indexType).
+		Id(string(key)).
+		Do(ctx)
+	if elastic.IsNotFound(err) {
+		return value, filer.ErrKvNotFound
+	}
+	if searchResult != nil && searchResult.Found {
+		esEntry := &ESKVEntry{}
+		if err := jsoniter.Unmarshal(searchResult.Source, esEntry); err == nil {
+			return esEntry.Value, nil
+		}
+	}
+	glog.ErrorfCtx(ctx, "find key(%s),%v.", string(key), err)
+	return value, filer.ErrKvNotFound
+}
+
+func (store *ElasticStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+	esEntry := &ESKVEntry{value}
+	val, err := jsoniter.Marshal(esEntry)
+	if err != nil {
+		glog.ErrorfCtx(ctx, "insert key(%s) %v.", string(key), err)
+		return fmt.Errorf("insert key %w", err)
+	}
+	_, err = store.client.Index().
+		Index(indexKV).
+		Type(indexType).
+		Id(string(key)).
+		BodyJson(string(val)).
+		Do(ctx)
+	if err != nil {
+		return fmt.Errorf("kv put: %w", err)
+	}
+	return nil
+}

weed/filer/empty_folder_cleanup/cleanup_queue.go

@@ -0,0 +1,228 @@
+package empty_folder_cleanup
+
+import (
+	"container/list"
+	"sync"
+	"time"
+)
+
+// CleanupQueue manages a deduplicated queue of folders pending cleanup.
+// It uses a doubly-linked list ordered by event time (oldest at front) and a map for O(1) deduplication.
+// Processing is triggered when:
+// - Queue size reaches maxSize, OR
+// - Oldest item exceeds maxAge
+type CleanupQueue struct {
+	mu       sync.Mutex
+	items    *list.List               // Linked list of *queueItem ordered by time (front = oldest)
+	itemsMap map[string]*list.Element // folder -> list element for O(1) lookup
+	maxSize  int                      // Max queue size before triggering cleanup
+	maxAge   time.Duration            // Max age before triggering cleanup
+}
+
+// queueItem represents an item in the cleanup queue
+type queueItem struct {
+	folder      string
+	triggeredBy string
+	queueTime   time.Time
+}
+
+// NewCleanupQueue creates a new CleanupQueue with the specified limits
+func NewCleanupQueue(maxSize int, maxAge time.Duration) *CleanupQueue {
+	return &CleanupQueue{
+		items:    list.New(),
+		itemsMap: make(map[string]*list.Element),
+		maxSize:  maxSize,
+		maxAge:   maxAge,
+	}
+}
+
+// Add adds a folder to the queue with the specified event time.
+// The item is inserted in time-sorted order (oldest at front) to handle out-of-order events.
+// If folder already exists with an older time, the time is updated and position adjusted.
+// Returns true if the folder was newly added, false if it was updated.
+func (q *CleanupQueue) Add(folder string, triggeredBy string, eventTime time.Time) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	// Check if folder already exists
+	if elem, exists := q.itemsMap[folder]; exists {
+		existingItem := elem.Value.(*queueItem)
+		// Only update if new event is later
+		if eventTime.After(existingItem.queueTime) {
+			// Remove from current position
+			q.items.Remove(elem)
+			// Re-insert with new time in sorted position
+			newElem := q.insertSorted(folder, triggeredBy, eventTime)
+			q.itemsMap[folder] = newElem
+		}
+		return false
+	}
+
+	// Insert new folder in sorted position
+	elem := q.insertSorted(folder, triggeredBy, eventTime)
+	q.itemsMap[folder] = elem
+	return true
+}
+
+// insertSorted inserts an item in the correct position to maintain time ordering (oldest at front)
+func (q *CleanupQueue) insertSorted(folder string, triggeredBy string, eventTime time.Time) *list.Element {
+	item := &queueItem{
+		folder:      folder,
+		triggeredBy: triggeredBy,
+		queueTime:   eventTime,
+	}
+
+	// Find the correct position (insert before the first item with a later time)
+	for elem := q.items.Back(); elem != nil; elem = elem.Prev() {
+		existingItem := elem.Value.(*queueItem)
+		if !eventTime.Before(existingItem.queueTime) {
+			// Insert after this element
+			return q.items.InsertAfter(item, elem)
+		}
+	}
+
+	// This item is the oldest, insert at front
+	return q.items.PushFront(item)
+}
+
+// Remove removes a specific folder from the queue (e.g., when a file is created).
+// Returns true if the folder was found and removed.
+func (q *CleanupQueue) Remove(folder string) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	elem, exists := q.itemsMap[folder]
+	if !exists {
+		return false
+	}
+
+	q.items.Remove(elem)
+	delete(q.itemsMap, folder)
+	return true
+}
+
+// ShouldProcess returns true if the queue should be processed.
+// This is true when:
+// - Queue size >= maxSize, OR
+// - Oldest item age > maxAge
+func (q *CleanupQueue) ShouldProcess() bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	return q.shouldProcessLocked()
+}
+
+// shouldProcessLocked checks if processing is needed (caller must hold lock)
+func (q *CleanupQueue) shouldProcessLocked() bool {
+	if q.items.Len() == 0 {
+		return false
+	}
+
+	// Check if queue is full
+	if q.items.Len() >= q.maxSize {
+		return true
+	}
+
+	// Check if oldest item exceeds max age
+	front := q.items.Front()
+	if front != nil {
+		item := front.Value.(*queueItem)
+		if time.Since(item.queueTime) > q.maxAge {
+			return true
+		}
+	}
+
+	return false
+}
+
+// Pop removes and returns the oldest folder from the queue.
+// Returns the folder and true if an item was available, or empty string and false if queue is empty.
+func (q *CleanupQueue) Pop() (string, string, bool) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	front := q.items.Front()
+	if front == nil {
+		return "", "", false
+	}
+
+	item := front.Value.(*queueItem)
+	q.items.Remove(front)
+	delete(q.itemsMap, item.folder)
+
+	return item.folder, item.triggeredBy, true
+}
+
+// PopOlderThan removes and returns the oldest folder only if it has been in the queue
+// for longer than the specified duration. Returns empty string and false if no item qualifies.
+func (q *CleanupQueue) PopOlderThan(minAge time.Duration) (string, string, bool) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	front := q.items.Front()
+	if front == nil {
+		return "", "", false
+	}
+
+	item := front.Value.(*queueItem)
+	if time.Since(item.queueTime) <= minAge {
+		return "", "", false
+	}
+
+	q.items.Remove(front)
+	delete(q.itemsMap, item.folder)
+	return item.folder, item.triggeredBy, true
+}
+
+// Peek returns the oldest folder without removing it.
+// Returns the folder and queue time if available, or empty values if queue is empty.
+func (q *CleanupQueue) Peek() (folder string, triggeredBy string, queueTime time.Time, ok bool) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	front := q.items.Front()
+	if front == nil {
+		return "", "", time.Time{}, false
+	}
+
+	item := front.Value.(*queueItem)
+	return item.folder, item.triggeredBy, item.queueTime, true
+}
+
+// Len returns the current queue size.
+func (q *CleanupQueue) Len() int {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return q.items.Len()
+}
+
+// Contains checks if a folder is in the queue.
+func (q *CleanupQueue) Contains(folder string) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	_, exists := q.itemsMap[folder]
+	return exists
+}
+
+// Clear removes all items from the queue.
+func (q *CleanupQueue) Clear() {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	q.items.Init()
+	q.itemsMap = make(map[string]*list.Element)
+}
+
+// OldestAge returns the age of the oldest item in the queue, or 0 if empty.
+func (q *CleanupQueue) OldestAge() time.Duration {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	front := q.items.Front()
+	if front == nil {
+		return 0
+	}
+
+	item := front.Value.(*queueItem)
+	return time.Since(item.queueTime)
+}

weed/filer/empty_folder_cleanup/cleanup_queue_test.go

@@ -0,0 +1,379 @@
+package empty_folder_cleanup
+
+import (
+	"testing"
+	"time"
+)
+
+func TestCleanupQueue_Add(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	now := time.Now()
+
+	// Add first item
+	if !q.Add("/buckets/b1/folder1", "item1", now) {
+		t.Error("expected Add to return true for new item")
+	}
+	if q.Len() != 1 {
+		t.Errorf("expected len 1, got %d", q.Len())
+	}
+
+	// Add second item with later time
+	if !q.Add("/buckets/b1/folder2", "item2", now.Add(1*time.Second)) {
+		t.Error("expected Add to return true for new item")
+	}
+	if q.Len() != 2 {
+		t.Errorf("expected len 2, got %d", q.Len())
+	}
+
+	// Add duplicate with newer time - should update and reposition
+	if q.Add("/buckets/b1/folder1", "item1-updated", now.Add(2*time.Second)) {
+		t.Error("expected Add to return false for existing item")
+	}
+	if q.Len() != 2 {
+		t.Errorf("expected len 2 after duplicate, got %d", q.Len())
+	}
+
+	// folder1 should now be at the back (newer time) - verify by popping
+	folder1, triggered1, _ := q.Pop()
+	folder2, triggered2, _ := q.Pop()
+	if folder1 != "/buckets/b1/folder2" || folder2 != "/buckets/b1/folder1" {
+		t.Errorf("expected folder1 to be moved to back, got %s, %s", folder1, folder2)
+	}
+	if triggered1 != "item2" || triggered2 != "item1-updated" {
+		t.Errorf("triggeredBy mismatch: got %s, %s", triggered1, triggered2)
+	}
+}
+
+func TestCleanupQueue_Add_OutOfOrder(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	baseTime := time.Now()
+
+	// Add items out of order
+	q.Add("/buckets/b1/folder3", "f3", baseTime.Add(3*time.Second))
+	q.Add("/buckets/b1/folder1", "f1", baseTime.Add(1*time.Second))
+	q.Add("/buckets/b1/folder2", "f2", baseTime.Add(2*time.Second))
+
+	// Items should be in time order (oldest first) - verify by popping
+	expected := []struct {
+		folder string
+		item   string
+	}{
+		{"/buckets/b1/folder1", "f1"},
+		{"/buckets/b1/folder2", "f2"},
+		{"/buckets/b1/folder3", "f3"},
+	}
+	for i, exp := range expected {
+		folder, item, ok := q.Pop()
+		if !ok || folder != exp.folder || item != exp.item {
+			t.Errorf("at index %d: expected %s by %s, got %s by %s", i, exp.folder, exp.item, folder, item)
+		}
+	}
+}
+
+func TestCleanupQueue_Add_DuplicateWithOlderTime(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	baseTime := time.Now()
+
+	// Add folder at t=5
+	q.Add("/buckets/b1/folder1", "t5", baseTime.Add(5*time.Second))
+
+	// Try to add same folder with older time - should NOT update
+	q.Add("/buckets/b1/folder1", "t2", baseTime.Add(2*time.Second))
+
+	// Time and triggeredBy should remain at t=5
+	_, triggered, queueTime, _ := q.Peek()
+	if queueTime != baseTime.Add(5*time.Second) || triggered != "t5" {
+		t.Errorf("expected time to remain unchanged, got %v and %s", queueTime, triggered)
+	}
+}
+
+func TestCleanupQueue_Remove(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	now := time.Now()
+
+	q.Add("/buckets/b1/folder1", "i1", now)
+	q.Add("/buckets/b1/folder2", "i2", now.Add(1*time.Second))
+	q.Add("/buckets/b1/folder3", "i3", now.Add(2*time.Second))
+
+	// Remove middle item
+	if !q.Remove("/buckets/b1/folder2") {
+		t.Error("expected Remove to return true for existing item")
+	}
+	if q.Len() != 2 {
+		t.Errorf("expected len 2, got %d", q.Len())
+	}
+	if q.Contains("/buckets/b1/folder2") {
+		t.Error("removed item should not be in queue")
+	}
+
+	// Remove non-existent item
+	if q.Remove("/buckets/b1/nonexistent") {
+		t.Error("expected Remove to return false for non-existent item")
+	}
+
+	// Verify order is preserved by popping
+	folder1, _, _ := q.Pop()
+	folder3, _, _ := q.Pop()
+	if folder1 != "/buckets/b1/folder1" || folder3 != "/buckets/b1/folder3" {
+		t.Errorf("unexpected order: %s, %s", folder1, folder3)
+	}
+}
+
+func TestCleanupQueue_Pop(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	now := time.Now()
+
+	// Pop from empty queue
+	folder, item, ok := q.Pop()
+	if ok {
+		t.Error("expected Pop to return false for empty queue")
+	}
+	if folder != "" || item != "" {
+		t.Errorf("expected empty results, got %s by %s", folder, item)
+	}
+
+	// Add items and pop in order
+	q.Add("/buckets/b1/folder1", "i1", now)
+	q.Add("/buckets/b1/folder2", "i2", now.Add(1*time.Second))
+	q.Add("/buckets/b1/folder3", "i3", now.Add(2*time.Second))
+
+	folder, _, ok = q.Pop()
+	if !ok || folder != "/buckets/b1/folder1" {
+		t.Errorf("expected folder1, got %s (ok=%v)", folder, ok)
+	}
+
+	folder, _, ok = q.Pop()
+	if !ok || folder != "/buckets/b1/folder2" {
+		t.Errorf("expected folder2, got %s (ok=%v)", folder, ok)
+	}
+
+	folder, _, ok = q.Pop()
+	if !ok || folder != "/buckets/b1/folder3" {
+		t.Errorf("expected folder3, got %s (ok=%v)", folder, ok)
+	}
+
+	// Queue should be empty now
+	if q.Len() != 0 {
+		t.Errorf("expected empty queue, got len %d", q.Len())
+	}
+}
+
+func TestCleanupQueue_Peek(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	now := time.Now()
+
+	// Peek empty queue
+	folder, item, _, ok := q.Peek()
+	if ok {
+		t.Error("expected Peek to return false for empty queue")
+	}
+
+	// Add item and peek
+	q.Add("/buckets/b1/folder1", "i1", now)
+	folder, item, queueTime, ok := q.Peek()
+	if !ok || folder != "/buckets/b1/folder1" || item != "i1" {
+		t.Errorf("expected folder1 by i1, got %s by %s (ok=%v)", folder, item, ok)
+	}
+	if queueTime != now {
+		t.Errorf("expected queue time %v, got %v", now, queueTime)
+	}
+
+	// Peek should not remove item
+	if q.Len() != 1 {
+		t.Errorf("Peek should not remove item, len=%d", q.Len())
+	}
+}
+
+func TestCleanupQueue_Contains(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	now := time.Now()
+
+	q.Add("/buckets/b1/folder1", "i1", now)
+
+	if !q.Contains("/buckets/b1/folder1") {
+		t.Error("expected Contains to return true")
+	}
+	if q.Contains("/buckets/b1/folder2") {
+		t.Error("expected Contains to return false for non-existent")
+	}
+}
+
+func TestCleanupQueue_ShouldProcess_MaxSize(t *testing.T) {
+	q := NewCleanupQueue(3, 10*time.Minute)
+	now := time.Now()
+
+	// Empty queue
+	if q.ShouldProcess() {
+		t.Error("empty queue should not need processing")
+	}
+
+	// Add items below max
+	q.Add("/buckets/b1/folder1", "i1", now)
+	q.Add("/buckets/b1/folder2", "i2", now.Add(1*time.Second))
+	if q.ShouldProcess() {
+		t.Error("queue below max should not need processing")
+	}
+
+	// Add item to reach max
+	q.Add("/buckets/b1/folder3", "i3", now.Add(2*time.Second))
+	if !q.ShouldProcess() {
+		t.Error("queue at max should need processing")
+	}
+}
+
+func TestCleanupQueue_ShouldProcess_MaxAge(t *testing.T) {
+	q := NewCleanupQueue(100, 100*time.Millisecond) // Short max age for testing
+
+	// Add item with old event time
+	oldTime := time.Now().Add(-1 * time.Second) // 1 second ago
+	q.Add("/buckets/b1/folder1", "i1", oldTime)
+
+	// Item is older than maxAge, should need processing
+	if !q.ShouldProcess() {
+		t.Error("old item should trigger processing")
+	}
+
+	// Clear and add fresh item
+	q.Clear()
+	q.Add("/buckets/b1/folder2", "i2", time.Now())
+
+	// Fresh item should not trigger processing
+	if q.ShouldProcess() {
+		t.Error("fresh item should not trigger processing")
+	}
+}
+
+func TestCleanupQueue_Clear(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	now := time.Now()
+
+	q.Add("/buckets/b1/folder1", "i1", now)
+	q.Add("/buckets/b1/folder2", "i2", now.Add(1*time.Second))
+	q.Add("/buckets/b1/folder3", "i3", now.Add(2*time.Second))
+
+	q.Clear()
+
+	if q.Len() != 0 {
+		t.Errorf("expected empty queue after Clear, got len %d", q.Len())
+	}
+	if q.Contains("/buckets/b1/folder1") {
+		t.Error("queue should not contain items after Clear")
+	}
+}
+
+func TestCleanupQueue_OldestAge(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+
+	// Empty queue
+	if q.OldestAge() != 0 {
+		t.Error("empty queue should have zero oldest age")
+	}
+
+	// Add item with time in the past
+	oldTime := time.Now().Add(-5 * time.Minute)
+	q.Add("/buckets/b1/folder1", "i1", oldTime)
+
+	// Age should be approximately 5 minutes
+	age := q.OldestAge()
+	if age < 4*time.Minute || age > 6*time.Minute {
+		t.Errorf("expected ~5m age, got %v", age)
+	}
+}
+
+func TestCleanupQueue_TimeOrder(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	baseTime := time.Now()
+
+	// Add items in order
+	items := []string{
+		"/buckets/b1/a",
+		"/buckets/b1/b",
+		"/buckets/b1/c",
+		"/buckets/b1/d",
+		"/buckets/b1/e",
+	}
+	for i, item := range items {
+		q.Add(item, "i", baseTime.Add(time.Duration(i)*time.Second))
+	}
+
+	// Pop should return in time order
+	for i, expected := range items {
+		got, _, ok := q.Pop()
+		if !ok {
+			t.Errorf("Pop %d: expected item, got empty", i)
+		}
+		if got != expected {
+			t.Errorf("Pop %d: expected %s, got %s", i, expected, got)
+		}
+	}
+}
+
+func TestCleanupQueue_DuplicateWithNewerTime(t *testing.T) {
+	q := NewCleanupQueue(100, 10*time.Minute)
+	baseTime := time.Now()
+
+	// Add items
+	q.Add("/buckets/b1/folder1", "i1", baseTime)
+	q.Add("/buckets/b1/folder2", "i2", baseTime.Add(1*time.Second))
+	q.Add("/buckets/b1/folder3", "i3", baseTime.Add(2*time.Second))
+
+	// Add duplicate with newer time - should update and reposition
+	q.Add("/buckets/b1/folder1", "i1-new", baseTime.Add(3*time.Second))
+
+	// folder1 should now be at the back (newest time) - verify by popping
+	expected := []string{"/buckets/b1/folder2", "/buckets/b1/folder3", "/buckets/b1/folder1"}
+	for i, exp := range expected {
+		folder, _, ok := q.Pop()
+		if !ok || folder != exp {
+			t.Errorf("at index %d: expected %s, got %s", i, exp, folder)
+		}
+	}
+}
+
+func TestCleanupQueue_Concurrent(t *testing.T) {
+	q := NewCleanupQueue(1000, 10*time.Minute)
+	done := make(chan bool)
+	now := time.Now()
+
+	// Concurrent adds
+	go func() {
+		for i := 0; i < 100; i++ {
+			q.Add("/buckets/b1/folder"+string(rune('A'+i%26)), "item", now.Add(time.Duration(i)*time.Millisecond))
+		}
+		done <- true
+	}()
+
+	// Concurrent removes
+	go func() {
+		for i := 0; i < 50; i++ {
+			q.Remove("/buckets/b1/folder" + string(rune('A'+i%26)))
+		}
+		done <- true
+	}()
+
+	// Concurrent pops
+	go func() {
+		for i := 0; i < 30; i++ {
+			q.Pop()
+		}
+		done <- true
+	}()
+
+	// Concurrent reads
+	go func() {
+		for i := 0; i < 100; i++ {
+			q.Len()
+			q.Contains("/buckets/b1/folderA")
+			q.ShouldProcess()
+		}
+		done <- true
+	}()
+
+	// Wait for all goroutines
+	for i := 0; i < 4; i++ {
+		<-done
+	}
+
+	// Just verify no panic occurred and queue is in consistent state
+	_ = q.Len()
+}

weed/filer/empty_folder_cleanup/empty_folder_cleaner.go

@@ -0,0 +1,810 @@
+package empty_folder_cleanup
+
+import (
+	"context"
+	"os"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const (
+	DefaultMaxCountCheck  = 1000
+	DefaultCacheExpiry    = 5 * time.Minute
+	DefaultQueueMaxSize   = 1000
+	DefaultQueueMaxAge    = 2 * time.Minute
+	DefaultProcessorSleep = 30 * time.Second // How often to check queue
+	DefaultMaxDeletedKept = 10000            // Deleted folders remembered for the restore check
+	// How long a deleted folder is kept so that a create event arriving for it can
+	// still put it back. It bounds how far behind the event stream may run, not how
+	// long the race window is.
+	DefaultObservationWindow = 2 * time.Minute
+)
+
+// DirectoryAttributes is what a restored directory needs to come back as it was.
+type DirectoryAttributes struct {
+	Mode       os.FileMode
+	Uid        uint32
+	Gid        uint32
+	UserName   string
+	GroupNames []string
+}
+
+// deletedFolder is a folder under observation for entries that landed while it was
+// being deleted. writtenTo is set by the create event for such an entry.
+type deletedFolder struct {
+	path      string
+	attrs     DirectoryAttributes
+	deletedAt time.Time
+	writtenTo bool
+}
+
+// FilerOperations defines the filer operations needed by EmptyFolderCleaner
+type FilerOperations interface {
+	CountDirectoryEntries(ctx context.Context, dirPath util.FullPath, limit int) (count int, err error)
+	DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32, ifNotModifiedAfter int64) error
+	GetEntryAttributes(ctx context.Context, p util.FullPath) (attributes map[string][]byte, err error)
+	IsDirectoryKeyObject(ctx context.Context, p util.FullPath) (bool, error)
+	DirectoryAttributes(ctx context.Context, p util.FullPath) (DirectoryAttributes, error)
+	EnsureDirectoryEntry(ctx context.Context, p util.FullPath, attrs DirectoryAttributes) error
+}
+
+// folderState tracks the state of a folder for empty folder cleanup
+type folderState struct {
+	roughCount  int       // Cached rough count (up to maxCountCheck)
+	lastAddTime time.Time // Last time an item was added
+	lastDelTime time.Time // Last time an item was deleted
+	lastCheck   time.Time // Last time we checked the actual count
+}
+
+type bucketCleanupPolicyState struct {
+	autoRemove bool
+	attrValue  string
+	lastCheck  time.Time
+}
+
+// EmptyFolderCleaner handles asynchronous cleanup of empty folders
+// Each filer owns specific folders via consistent hashing based on the peer filer list
+type EmptyFolderCleaner struct {
+	filer    FilerOperations
+	lockRing *lock_manager.LockRing
+	host     pb.ServerAddress
+
+	// Folder state tracking
+	mu                    sync.RWMutex
+	folderCounts          map[string]*folderState              // Rough count cache
+	bucketCleanupPolicies map[string]*bucketCleanupPolicyState // bucket path -> cleanup policy cache
+
+	// Folders deleted recently, kept so that a create event arriving for one of them
+	// can put it back
+	deleted        map[string]*deletedFolder
+	deletedDropped int
+
+	// Cleanup queue (thread-safe, has its own lock)
+	cleanupQueue *CleanupQueue
+
+	// Configuration
+	maxCountCheck  int           // Max items to count (1000)
+	cacheExpiry    time.Duration // How long to keep cache entries
+	processorSleep time.Duration // How often processor checks queue
+	bucketPath     string        // e.g., "/buckets"
+
+	// Control
+	enabled bool
+	stopCh  chan struct{}
+}
+
+// NewEmptyFolderCleaner creates a new EmptyFolderCleaner.
+// cleanupDelay controls how long an empty folder must remain in the queue before deletion.
+// If zero, DefaultQueueMaxAge is used.
+func NewEmptyFolderCleaner(filer FilerOperations, lockRing *lock_manager.LockRing, host pb.ServerAddress, bucketPath string, cleanupDelay time.Duration) *EmptyFolderCleaner {
+	if cleanupDelay <= 0 {
+		cleanupDelay = DefaultQueueMaxAge
+	}
+	efc := &EmptyFolderCleaner{
+		filer:                 filer,
+		lockRing:              lockRing,
+		host:                  host,
+		folderCounts:          make(map[string]*folderState),
+		bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
+		deleted:               make(map[string]*deletedFolder),
+		cleanupQueue:          NewCleanupQueue(DefaultQueueMaxSize, cleanupDelay),
+		maxCountCheck:         DefaultMaxCountCheck,
+		cacheExpiry:           DefaultCacheExpiry,
+		processorSleep:        DefaultProcessorSleep,
+		bucketPath:            bucketPath,
+		enabled:               true,
+		stopCh:                make(chan struct{}),
+	}
+	go efc.cacheEvictionLoop()
+	go efc.cleanupProcessor()
+	return efc
+}
+
+// SetEnabled enables or disables the cleaner
+func (efc *EmptyFolderCleaner) SetEnabled(enabled bool) {
+	efc.mu.Lock()
+	defer efc.mu.Unlock()
+	efc.enabled = enabled
+}
+
+// IsEnabled returns whether the cleaner is enabled
+func (efc *EmptyFolderCleaner) IsEnabled() bool {
+	efc.mu.RLock()
+	defer efc.mu.RUnlock()
+	return efc.enabled
+}
+
+// ownsFolder checks if this filer owns the folder via consistent hashing
+func (efc *EmptyFolderCleaner) ownsFolder(folder string) bool {
+	primary := efc.lockRing.GetPrimary(folder)
+	if primary == "" {
+		return true // Single filer case or no servers
+	}
+	return primary == efc.host
+}
+
+// OnDeleteEvent is called when a file or directory is deleted
+// Both file and directory deletions count towards making the parent folder empty
+// eventTime is the time when the delete event occurred (for proper ordering)
+func (efc *EmptyFolderCleaner) OnDeleteEvent(directory string, entryName string, isDirectory bool, eventTime time.Time) {
+	// Skip if not under bucket path (must be at least /buckets/<bucket>/...)
+	if efc.bucketPath != "" && !isUnderBucketPath(directory, efc.bucketPath) {
+		return
+	}
+
+	// Never queue the S3 multipart staging area; the upload lifecycle owns it.
+	if isMultipartUploadsPath(efc.bucketPath, directory) {
+		return
+	}
+
+	// Check if we own this folder
+	if !efc.ownsFolder(directory) {
+		glog.V(4).Infof("EmptyFolderCleaner: not owner of %s, skipping", directory)
+		return
+	}
+
+	efc.mu.Lock()
+	defer efc.mu.Unlock()
+
+	// Check enabled inside lock to avoid race with Stop()
+	if !efc.enabled {
+		return
+	}
+
+	glog.V(3).Infof("EmptyFolderCleaner: delete event in %s/%s (isDir=%v)", directory, entryName, isDirectory)
+
+	// Update cached count (create entry if needed)
+	state, exists := efc.folderCounts[directory]
+	if !exists {
+		state = &folderState{}
+		efc.folderCounts[directory] = state
+	}
+	if state.roughCount > 0 {
+		state.roughCount--
+	}
+	state.lastDelTime = eventTime
+
+	// Only add to cleanup queue if roughCount suggests folder might be empty
+	if state.roughCount > 0 {
+		glog.V(3).Infof("EmptyFolderCleaner: skipping queue for %s, roughCount=%d", directory, state.roughCount)
+		return
+	}
+
+	// Add to cleanup queue with event time (handles out-of-order events)
+	if efc.cleanupQueue.Add(directory, entryName, eventTime) {
+		glog.V(3).Infof("EmptyFolderCleaner: queued %s for cleanup (triggered by %s)", directory, entryName)
+	}
+}
+
+// OnCreateEvent is called when a file or directory is created
+// Both file and directory creations cancel pending cleanup for the parent folder
+func (efc *EmptyFolderCleaner) OnCreateEvent(directory string, entryName string, isDirectory bool) {
+	// Skip if not under bucket path (must be at least /buckets/<bucket>/...)
+	if efc.bucketPath != "" && !isUnderBucketPath(directory, efc.bucketPath) {
+		return
+	}
+
+	efc.mu.Lock()
+	defer efc.mu.Unlock()
+
+	// Check enabled inside lock to avoid race with Stop()
+	if !efc.enabled {
+		return
+	}
+
+	// Update cached count only if already tracked (no need to track new folders)
+	if state, exists := efc.folderCounts[directory]; exists {
+		state.roughCount++
+		state.lastAddTime = time.Now()
+	}
+
+	// An entry landing in a folder we just deleted is the race this cleaner cannot
+	// exclude: the folder was empty when checked and is gone now, so the entry has
+	// nothing holding it. The event says so outright, which beats going back to look.
+	if folder, found := efc.deleted[directory]; found {
+		folder.writtenTo = true
+		glog.V(2).Infof("EmptyFolderCleaner: %s was written to while being deleted, restoring it", directory)
+	}
+
+	// Remove from cleanup queue (cancel pending cleanup)
+	if efc.cleanupQueue.Remove(directory) {
+		glog.V(3).Infof("EmptyFolderCleaner: cancelled cleanup for %s due to new entry", directory)
+	}
+
+	// A directory that has just been created is a new incarnation, so a cleanup
+	// queued against the one it replaces would delete it rather than the folder
+	// that was found empty.
+	if isDirectory {
+		recreated := string(util.NewFullPath(directory, entryName))
+		if efc.cleanupQueue.Remove(recreated) {
+			glog.V(3).Infof("EmptyFolderCleaner: cancelled cleanup for %s, recreated", recreated)
+		}
+	}
+}
+
+// cleanupProcessor runs in background and processes the cleanup queue
+func (efc *EmptyFolderCleaner) cleanupProcessor() {
+	ticker := time.NewTicker(efc.processorSleep)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-efc.stopCh:
+			return
+		case <-ticker.C:
+			efc.processCleanupQueue()
+		}
+	}
+}
+
+// processCleanupQueue processes items from the cleanup queue
+func (efc *EmptyFolderCleaner) processCleanupQueue() {
+	efc.restoreFoldersWrittenDuringDelete()
+
+	if efc.cleanupQueue.Len() == 0 {
+		return
+	}
+
+	glog.V(3).Infof("EmptyFolderCleaner: processing cleanup queue (len=%d, oldest_age=%v)",
+		efc.cleanupQueue.Len(), efc.cleanupQueue.OldestAge())
+
+	// Only process items that have been queued longer than maxAge
+	for {
+		// Check if still enabled
+		if !efc.IsEnabled() {
+			return
+		}
+
+		// Only pop items old enough — newer items stay in the queue
+		folder, triggeredBy, ok := efc.cleanupQueue.PopOlderThan(efc.cleanupQueue.maxAge)
+		if !ok {
+			break
+		}
+
+		// Execute cleanup for this folder
+		efc.executeCleanup(folder, triggeredBy)
+	}
+}
+
+// restoreFoldersWrittenDuringDelete puts back folders that received an entry between
+// the emptiness check and the delete, which leaves that entry with no directory
+// holding it: reachable by its own path, but absent from any listing.
+//
+// A folder stays under observation for DefaultRestoreCheckWindow rather than being
+// checked once. A writer looks up the parent before inserting the child, so a check
+// can land in that gap and see nothing; ticks also coalesce when a pass runs long, so
+// "next pass" is not a delay at all. Re-checking for a bounded wall-clock window
+// covers both. This narrows the exposure rather than closing it - only making the
+// emptiness check and the delete atomic would do that.
+func (efc *EmptyFolderCleaner) restoreFoldersWrittenDuringDelete() {
+	efc.mu.Lock()
+	dropped := efc.deletedDropped
+	efc.deletedDropped = 0
+	var restore []*deletedFolder
+	for path, folder := range efc.deleted {
+		// The window applies whatever the folder's state is. Checking writtenTo first
+		// would keep a folder whose restore keeps failing forever, re-counting it on
+		// every pass.
+		if time.Since(folder.deletedAt) >= DefaultObservationWindow {
+			delete(efc.deleted, path)
+			continue
+		}
+		if folder.writtenTo {
+			restore = append(restore, folder)
+			delete(efc.deleted, path)
+		}
+	}
+	// A cascade takes ancestors along with the folder. Rebuild those from what they
+	// were too: leaving them to the descendant's restore would mint them from the
+	// descendant's own attributes, handing back access the ancestor did not grant.
+	for i := 0; i < len(restore); i++ {
+		ancestor, _ := util.FullPath(restore[i].path).DirAndName()
+		for ancestor != "" && ancestor != "/" {
+			if folder, found := efc.deleted[ancestor]; found {
+				restore = append(restore, folder)
+				delete(efc.deleted, ancestor)
+			}
+			ancestor, _ = util.FullPath(ancestor).DirAndName()
+		}
+	}
+	efc.mu.Unlock()
+
+	if dropped > 0 {
+		glog.V(1).Infof("EmptyFolderCleaner: %d deleted folders left unobserved, past the %d kept", dropped, DefaultMaxDeletedKept)
+	}
+	if len(restore) == 0 {
+		return
+	}
+
+	// Restore shallowest first, so a folder taken by the parent cascade is rebuilt
+	// with its own attributes before anything below it needs it as a parent.
+	sort.Slice(restore, func(i, j int) bool {
+		return strings.Count(restore[i].path, "/") < strings.Count(restore[j].path, "/")
+	})
+
+	ctx := context.Background()
+	var retry []*deletedFolder
+	for i, folder := range restore {
+		if !efc.IsEnabled() {
+			retry = append(retry, restore[i:]...)
+			break
+		}
+		// An event named this folder, but the entry may have been removed again since,
+		// in which case there is nothing to hold and it can stay gone. Ancestors pulled
+		// in above carry no event and are rebuilt regardless, since the folder below
+		// them needs them.
+		if folder.writtenTo {
+			count, err := efc.countItems(ctx, folder.path)
+			if err != nil {
+				glog.V(2).Infof("EmptyFolderCleaner: cannot count %s before restoring it: %v", folder.path, err)
+				retry = append(retry, folder)
+				continue
+			}
+			if count == 0 {
+				continue
+			}
+		}
+		glog.V(1).Infof("EmptyFolderCleaner: restoring %s, written to while it was being deleted", folder.path)
+		if err := efc.filer.EnsureDirectoryEntry(ctx, util.FullPath(folder.path), folder.attrs); err != nil {
+			glog.V(2).Infof("EmptyFolderCleaner: failed to restore %s: %v", folder.path, err)
+			retry = append(retry, folder)
+		}
+	}
+
+	if len(retry) == 0 {
+		return
+	}
+	efc.mu.Lock()
+	for _, folder := range retry {
+		if _, found := efc.deleted[folder.path]; !found {
+			efc.makeRoomForDeletedLocked()
+		}
+		efc.deleted[folder.path] = folder
+	}
+	efc.mu.Unlock()
+}
+
+// restoreIfWrittenTo puts folder back when an entry landed in it while it was being
+// deleted. True means the parent is no longer empty, so the caller must not cascade.
+func (efc *EmptyFolderCleaner) restoreIfWrittenTo(ctx context.Context, folder string, attrs DirectoryAttributes) bool {
+	count, err := efc.countItems(ctx, folder)
+	if err != nil || count > 0 {
+		// Ask for the retry here rather than leaving it to a create event that may
+		// already have gone by: this pass can be the only sight of the entry.
+		efc.mu.Lock()
+		if observed, found := efc.deleted[folder]; found {
+			observed.writtenTo = true
+		}
+		efc.mu.Unlock()
+	}
+	if err != nil {
+		glog.V(2).Infof("EmptyFolderCleaner: cannot re-check %s after deleting it: %v", folder, err)
+		return false
+	}
+	if count == 0 {
+		return false
+	}
+
+	glog.V(1).Infof("EmptyFolderCleaner: restoring %s, written to while it was being deleted", folder)
+	if err := efc.filer.EnsureDirectoryEntry(ctx, util.FullPath(folder), attrs); err != nil {
+		glog.V(2).Infof("EmptyFolderCleaner: failed to restore %s: %v", folder, err)
+		return false
+	}
+
+	efc.mu.Lock()
+	delete(efc.deleted, folder)
+	efc.mu.Unlock()
+	return true
+}
+
+// makeRoomForDeletedLocked drops the oldest of a small sample when the set is full.
+// The newest folders are the ones whose race is still live, so they must not be the
+// ones given up; sampling keeps this cheap under heavy deletion rates.
+func (efc *EmptyFolderCleaner) makeRoomForDeletedLocked() {
+	if len(efc.deleted) < DefaultMaxDeletedKept {
+		return
+	}
+	const sampleSize = 32
+	oldestPath, seen := "", 0
+	for path, folder := range efc.deleted {
+		if oldestPath == "" || folder.deletedAt.Before(efc.deleted[oldestPath].deletedAt) {
+			oldestPath = path
+		}
+		if seen++; seen >= sampleSize {
+			break
+		}
+	}
+	if oldestPath != "" {
+		delete(efc.deleted, oldestPath)
+		efc.deletedDropped++
+	}
+}
+
+// executeCleanup performs the actual cleanup of an empty folder
+func (efc *EmptyFolderCleaner) executeCleanup(folder string, triggeredBy string) {
+	// The bucket-shared .uploads staging tree holds in-progress multipart uploads.
+	// Deleting <bucket>/.uploads (reached here directly or via the parent cascade
+	// below) races a concurrent CreateMultipartUpload: the new upload's marker row
+	// is inserted between the emptiness check and the folder delete, then wiped,
+	// so the upload silently vanishes. Leave this tree to the upload lifecycle.
+	if isMultipartUploadsPath(efc.bucketPath, folder) {
+		return
+	}
+
+	efc.mu.Lock()
+
+	// Quick check: if we have cached count and it's > 0, skip
+	if state, exists := efc.folderCounts[folder]; exists {
+		if state.roughCount > 0 {
+			glog.V(3).Infof("EmptyFolderCleaner: skipping %s (triggered by %s), cached count=%d", folder, triggeredBy, state.roughCount)
+			efc.mu.Unlock()
+			return
+		}
+		// If there was an add after our delete, skip
+		if !state.lastAddTime.IsZero() && state.lastAddTime.After(state.lastDelTime) {
+			glog.V(3).Infof("EmptyFolderCleaner: skipping %s (triggered by %s), add happened after delete", folder, triggeredBy)
+			efc.mu.Unlock()
+			return
+		}
+	}
+	efc.mu.Unlock()
+
+	// Re-check ownership (topology might have changed)
+	if !efc.ownsFolder(folder) {
+		glog.V(3).Infof("EmptyFolderCleaner: no longer owner of %s (triggered by %s), skipping", folder, triggeredBy)
+		return
+	}
+
+	ctx := context.Background()
+	bucketPath, autoRemove, source, attrValue, err := efc.getBucketCleanupPolicy(ctx, folder)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			return
+		}
+		glog.V(2).Infof("EmptyFolderCleaner: failed to load bucket cleanup policy for folder %s (triggered by %s): %v", folder, triggeredBy, err)
+		return
+	}
+
+	if !autoRemove {
+		glog.V(3).Infof("EmptyFolderCleaner: skipping folder %s (triggered by %s), bucket %s auto-remove-empty-folders disabled (source=%s attr=%s)",
+			folder, triggeredBy, bucketPath, source, attrValue)
+		return
+	}
+
+	// Check if folder is actually empty (count up to maxCountCheck)
+	count, err := efc.countItems(ctx, folder)
+	if err != nil {
+		glog.V(2).Infof("EmptyFolderCleaner: error counting items in %s: %v", folder, err)
+		return
+	}
+
+	efc.mu.Lock()
+	// Update cache
+	if _, exists := efc.folderCounts[folder]; !exists {
+		efc.folderCounts[folder] = &folderState{}
+	}
+	efc.folderCounts[folder].roughCount = count
+	efc.folderCounts[folder].lastCheck = time.Now()
+	efc.mu.Unlock()
+
+	if count > 0 {
+		glog.V(4).Infof("EmptyFolderCleaner: folder %s (triggered by %s) has %d items, not empty", folder, triggeredBy, count)
+		return
+	}
+
+	// Skip explicitly created directory markers (e.g., PUT /bucket/folder/)
+	// These have a MIME type set and should be preserved even when empty
+	if isKeyObj, err := efc.filer.IsDirectoryKeyObject(ctx, util.FullPath(folder)); err != nil {
+		glog.V(2).Infof("EmptyFolderCleaner: error checking directory key object %s: %v", folder, err)
+		return
+	} else if isKeyObj {
+		glog.V(3).Infof("EmptyFolderCleaner: skipping %s (triggered by %s), explicit directory marker", folder, triggeredBy)
+		return
+	}
+
+	// Read what it would take to put this folder back before removing it; without
+	// that a restore would have to invent attributes for it.
+	attrs, err := efc.filer.DirectoryAttributes(ctx, util.FullPath(folder))
+	if err != nil {
+		glog.V(2).Infof("EmptyFolderCleaner: cannot read %s before deleting it: %v", folder, err)
+		return
+	}
+
+	// An S3 Tables catalog entry - a namespace, a table whose files a rename left
+	// behind at the old path, a view that never had any - is the directory itself,
+	// so its being empty says nothing about whether the catalog still names it.
+	if extended, err := efc.filer.GetEntryAttributes(ctx, util.FullPath(folder)); err != nil {
+		glog.V(2).Infof("EmptyFolderCleaner: error reading attributes of %s: %v", folder, err)
+		return
+	} else if isCatalogEntry(extended) {
+		glog.V(3).Infof("EmptyFolderCleaner: skipping %s (triggered by %s), s3tables catalog entry", folder, triggeredBy)
+		return
+	}
+
+	// Observe it before the delete rather than after. A delete can fail partway and
+	// still leave the folder gone - the redis stores remove the folder before their
+	// parent-list member - so a failure return is not proof that it is still there.
+	efc.mu.Lock()
+	if efc.deleted == nil {
+		efc.deleted = make(map[string]*deletedFolder)
+	}
+	efc.makeRoomForDeletedLocked()
+	efc.deleted[folder] = &deletedFolder{path: folder, attrs: attrs, deletedAt: time.Now()}
+	efc.mu.Unlock()
+
+	glog.Infof("EmptyFolderCleaner: deleting empty folder %s (triggered by %s)", folder, triggeredBy)
+	if err := efc.deleteFolder(ctx, folder); err != nil {
+		glog.V(2).Infof("EmptyFolderCleaner: failed to delete empty folder %s (triggered by %s): %v", folder, triggeredBy, err)
+		return
+	}
+
+	// Clean up cache entry
+	efc.mu.Lock()
+	delete(efc.folderCounts, folder)
+	efc.mu.Unlock()
+
+	// The delete's own emptiness check and the entry removal are not atomic either.
+	// Paired with parents being created after the insert, whoever acts second sees it.
+	if efc.restoreIfWrittenTo(ctx, folder, attrs) {
+		return
+	}
+
+	// After deleting this folder, immediately try to clean the parent.
+	// Relying solely on cascading metadata events would re-enter the full
+	// delay queue for each ancestor level, causing multi-minute cascading
+	// waits (e.g. 3 levels × 2m = 6m+).  Instead, walk up eagerly.
+	parentDir, _ := util.FullPath(folder).DirAndName()
+	if parentDir != "" && parentDir != folder &&
+		efc.bucketPath != "" && isUnderBucketPath(parentDir, efc.bucketPath) {
+		// Remove any pending queue entry for the parent so we don't
+		// double-process it later from a stale event.
+		efc.cleanupQueue.Remove(parentDir)
+		efc.executeCleanup(parentDir, triggeredBy)
+	}
+}
+
+// countItems counts items in a folder (up to maxCountCheck)
+func (efc *EmptyFolderCleaner) countItems(ctx context.Context, folder string) (int, error) {
+	return efc.filer.CountDirectoryEntries(ctx, util.FullPath(folder), efc.maxCountCheck)
+}
+
+// deleteFolder deletes an empty folder
+func (efc *EmptyFolderCleaner) deleteFolder(ctx context.Context, folder string) error {
+	return efc.filer.DeleteEntryMetaAndData(ctx, util.FullPath(folder), false, false, false, false, nil, 0)
+}
+
+func (efc *EmptyFolderCleaner) getBucketCleanupPolicy(ctx context.Context, folder string) (bucketPath string, autoRemove bool, source string, attrValue string, err error) {
+	bucketPath, ok := util.ExtractBucketPath(efc.bucketPath, folder, true)
+	if !ok {
+		return "", true, "default", "<not_bucket_path>", nil
+	}
+
+	now := time.Now()
+
+	efc.mu.RLock()
+	if state, found := efc.bucketCleanupPolicies[bucketPath]; found && now.Sub(state.lastCheck) <= efc.cacheExpiry {
+		efc.mu.RUnlock()
+		return bucketPath, state.autoRemove, "cache", state.attrValue, nil
+	}
+	efc.mu.RUnlock()
+
+	attrs, err := efc.filer.GetEntryAttributes(ctx, util.FullPath(bucketPath))
+	if err != nil {
+		return "", true, "", "", err
+	}
+
+	autoRemove, attrValue = autoRemoveEmptyFoldersEnabled(attrs)
+
+	efc.mu.Lock()
+	if efc.bucketCleanupPolicies == nil {
+		efc.bucketCleanupPolicies = make(map[string]*bucketCleanupPolicyState)
+	}
+	efc.bucketCleanupPolicies[bucketPath] = &bucketCleanupPolicyState{
+		autoRemove: autoRemove,
+		attrValue:  attrValue,
+		lastCheck:  now,
+	}
+	efc.mu.Unlock()
+
+	return bucketPath, autoRemove, "filer", attrValue, nil
+}
+
+// isCatalogEntry reports whether the directory is an s3tables catalog record.
+func isCatalogEntry(attrs map[string][]byte) bool {
+	for key := range attrs {
+		if strings.HasPrefix(key, s3_constants.ExtS3TablesPrefix) {
+			return true
+		}
+	}
+	return false
+}
+
+func autoRemoveEmptyFoldersEnabled(attrs map[string][]byte) (bool, string) {
+	if attrs == nil {
+		return true, "<no_attrs>"
+	}
+
+	value, found := attrs[s3_constants.ExtAllowEmptyFolders]
+	if !found {
+		return true, "<missing>"
+	}
+
+	text := strings.TrimSpace(string(value))
+	if text == "" {
+		return true, "<empty>"
+	}
+
+	return !strings.EqualFold(text, "true"), text
+}
+
+// isUnderPath checks if child is under parent path
+func isUnderPath(child, parent string) bool {
+	if parent == "" || parent == "/" {
+		return true
+	}
+	// Ensure parent ends without slash for proper prefix matching
+	if len(parent) > 0 && parent[len(parent)-1] == '/' {
+		parent = parent[:len(parent)-1]
+	}
+	// Child must start with parent and then have a / or be exactly parent
+	if len(child) < len(parent) {
+		return false
+	}
+	if child[:len(parent)] != parent {
+		return false
+	}
+	if len(child) == len(parent) {
+		return true
+	}
+	return child[len(parent)] == '/'
+}
+
+// isMultipartUploadsPath reports whether directory is the S3 multipart staging
+// root <bucket>/.uploads or anything beneath it.
+func isMultipartUploadsPath(bucketPath, directory string) bool {
+	if bucketPath == "" {
+		return false
+	}
+	dir, ok := util.ExtractBucketPath(bucketPath, directory, true)
+	if !ok {
+		return false
+	}
+	// requireChild guarantees directory starts with dir + "/", so slice past it.
+	first, _, _ := strings.Cut(directory[len(dir)+1:], "/")
+	return first == s3_constants.MultipartUploadsFolder
+}
+
+// isUnderBucketPath checks if directory is inside a bucket (under /buckets/<bucket>/...)
+// This ensures we only clean up folders inside buckets, not the buckets themselves
+func isUnderBucketPath(directory, bucketPath string) bool {
+	if bucketPath == "" {
+		return true
+	}
+	// Ensure bucketPath ends without slash
+	if len(bucketPath) > 0 && bucketPath[len(bucketPath)-1] == '/' {
+		bucketPath = bucketPath[:len(bucketPath)-1]
+	}
+	// Directory must be under bucketPath
+	if !isUnderPath(directory, bucketPath) {
+		return false
+	}
+	// Directory must be at least /buckets/<bucket>/<something>
+	// i.e., depth must be at least bucketPath depth + 2
+	// For /buckets (depth 1), we need at least /buckets/mybucket/folder (depth 3)
+	bucketPathDepth := strings.Count(bucketPath, "/")
+	directoryDepth := strings.Count(directory, "/")
+	return directoryDepth >= bucketPathDepth+2
+}
+
+// cacheEvictionLoop periodically removes stale entries from folderCounts
+func (efc *EmptyFolderCleaner) cacheEvictionLoop() {
+	ticker := time.NewTicker(efc.cacheExpiry)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-efc.stopCh:
+			return
+		case <-ticker.C:
+			efc.evictStaleCacheEntries()
+		}
+	}
+}
+
+// evictStaleCacheEntries removes cache entries that haven't been accessed recently
+func (efc *EmptyFolderCleaner) evictStaleCacheEntries() {
+	efc.mu.Lock()
+	defer efc.mu.Unlock()
+
+	now := time.Now()
+	expiredCount := 0
+	for folder, state := range efc.folderCounts {
+		// Skip if folder is in cleanup queue
+		if efc.cleanupQueue.Contains(folder) {
+			continue
+		}
+
+		// Find the most recent activity time for this folder
+		lastActivity := state.lastCheck
+		if state.lastAddTime.After(lastActivity) {
+			lastActivity = state.lastAddTime
+		}
+		if state.lastDelTime.After(lastActivity) {
+			lastActivity = state.lastDelTime
+		}
+
+		// Evict if no activity within cache expiry period
+		if now.Sub(lastActivity) > efc.cacheExpiry {
+			delete(efc.folderCounts, folder)
+			expiredCount++
+		}
+	}
+
+	for bucketPath, state := range efc.bucketCleanupPolicies {
+		if now.Sub(state.lastCheck) > efc.cacheExpiry {
+			delete(efc.bucketCleanupPolicies, bucketPath)
+		}
+	}
+
+	if expiredCount > 0 {
+		glog.V(3).Infof("EmptyFolderCleaner: evicted %d stale cache entries", expiredCount)
+	}
+}
+
+// Stop stops the cleaner and cancels all pending tasks
+func (efc *EmptyFolderCleaner) Stop() {
+	close(efc.stopCh)
+
+	efc.mu.Lock()
+	defer efc.mu.Unlock()
+
+	efc.enabled = false
+	efc.cleanupQueue.Clear()
+	efc.folderCounts = make(map[string]*folderState) // Clear cache on stop
+	efc.bucketCleanupPolicies = make(map[string]*bucketCleanupPolicyState)
+	efc.deleted, efc.deletedDropped = make(map[string]*deletedFolder), 0
+}
+
+// GetPendingCleanupCount returns the number of pending cleanup tasks (for testing)
+func (efc *EmptyFolderCleaner) GetPendingCleanupCount() int {
+	return efc.cleanupQueue.Len()
+}
+
+// GetCachedFolderCount returns the cached count for a folder (for testing)
+func (efc *EmptyFolderCleaner) GetCachedFolderCount(folder string) (int, bool) {
+	efc.mu.RLock()
+	defer efc.mu.RUnlock()
+	if state, exists := efc.folderCounts[folder]; exists {
+		return state.roughCount, true
+	}
+	return 0, false
+}

weed/filer/entry.go

@@ -0,0 +1,232 @@
+package filer
+
+import (
+	"os"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type Attr struct {
+	Mtime         time.Time   // time of last modification
+	Crtime        time.Time   // time of creation (OS X only)
+	Ctime         time.Time   // time of last inode change
+	Atime         time.Time   // time of last access
+	Mode          os.FileMode // file mode
+	Uid           uint32      // owner uid
+	Gid           uint32      // group gid
+	Mime          string      // mime type
+	TtlSec        int32       // ttl in seconds
+	UserName      string
+	GroupNames    []string
+	SymlinkTarget string
+	Md5           []byte
+	FileSize      uint64
+	Rdev          uint32
+	Inode         uint64
+}
+
+func (attr Attr) IsDirectory() bool {
+	return attr.Mode&os.ModeDir > 0
+}
+
+type Entry struct {
+	util.FullPath
+
+	Attr
+	Extended map[string][]byte
+
+	// the following is for files
+	Chunks []*filer_pb.FileChunk `json:"chunks,omitempty"`
+
+	HardLinkId         HardLinkId
+	HardLinkCounter    int32
+	Content            []byte
+	Remote             *filer_pb.RemoteEntry
+	Quota              int64
+	WORMEnforcedAtTsNs int64
+}
+
+func (entry *Entry) Size() uint64 {
+	return maxUint64(maxUint64(TotalSize(entry.GetChunks()), entry.FileSize), uint64(len(entry.Content)))
+}
+
+func (entry *Entry) Timestamp() time.Time {
+	if entry.IsDirectory() {
+		return entry.Crtime
+	} else {
+		return entry.Mtime
+	}
+}
+
+func (entry *Entry) ShallowClone() *Entry {
+	if entry == nil {
+		return nil
+	}
+	newEntry := &Entry{}
+	newEntry.FullPath = entry.FullPath
+	newEntry.Attr = entry.Attr
+	newEntry.Chunks = entry.Chunks
+	newEntry.Extended = entry.Extended
+	newEntry.HardLinkId = entry.HardLinkId
+	newEntry.HardLinkCounter = entry.HardLinkCounter
+	newEntry.Content = entry.Content
+	newEntry.Remote = entry.Remote
+	newEntry.Quota = entry.Quota
+
+	return newEntry
+}
+
+func (entry *Entry) ToProtoEntry() *filer_pb.Entry {
+	if entry == nil {
+		return nil
+	}
+	message := &filer_pb.Entry{}
+	message.Name = entry.FullPath.Name()
+	entry.ToExistingProtoEntry(message)
+	return message
+}
+
+func (entry *Entry) ToExistingProtoEntry(message *filer_pb.Entry) {
+	if entry == nil {
+		return
+	}
+	message.IsDirectory = entry.IsDirectory()
+	// Reuse pre-allocated attributes if available, otherwise allocate
+	if message.Attributes != nil {
+		EntryAttributeToExistingPb(entry, message.Attributes)
+	} else {
+		message.Attributes = EntryAttributeToPb(entry)
+	}
+	message.Chunks = entry.GetChunks()
+	message.Extended = entry.Extended
+	message.HardLinkId = entry.HardLinkId
+	message.HardLinkCounter = entry.HardLinkCounter
+	message.Content = entry.Content
+	message.RemoteEntry = entry.Remote
+	message.Quota = entry.Quota
+	message.WormEnforcedAtTsNs = entry.WORMEnforcedAtTsNs
+}
+
+func FromPbEntryToExistingEntry(message *filer_pb.Entry, fsEntry *Entry) {
+	fsEntry.Attr = PbToEntryAttribute(message.Attributes)
+	fsEntry.Chunks = message.Chunks
+	fsEntry.Extended = message.Extended
+	fsEntry.HardLinkId = HardLinkId(message.HardLinkId)
+	fsEntry.HardLinkCounter = message.HardLinkCounter
+	fsEntry.Content = message.Content
+	fsEntry.Remote = message.RemoteEntry
+	fsEntry.Quota = message.Quota
+	fsEntry.FileSize = FileSize(message)
+	fsEntry.WORMEnforcedAtTsNs = message.WormEnforcedAtTsNs
+}
+
+func (entry *Entry) ToProtoFullEntry() *filer_pb.FullEntry {
+	if entry == nil {
+		return nil
+	}
+	dir, _ := entry.FullPath.DirAndName()
+	return &filer_pb.FullEntry{
+		Dir:   dir,
+		Entry: entry.ToProtoEntry(),
+	}
+}
+
+func (entry *Entry) GetChunks() []*filer_pb.FileChunk {
+	return entry.Chunks
+}
+
+func FromPbEntry(dir string, entry *filer_pb.Entry) *Entry {
+	t := &Entry{}
+	t.FullPath = util.NewFullPath(dir, entry.Name)
+	FromPbEntryToExistingEntry(entry, t)
+	return t
+}
+
+func maxUint64(x, y uint64) uint64 {
+	if x > y {
+		return x
+	}
+	return y
+}
+
+func (entry *Entry) IsExpireS3Enabled() (exist bool) {
+	if entry.Extended != nil {
+		_, exist = entry.Extended[s3_constants.SeaweedFSExpiresS3]
+	}
+	return exist
+}
+
+func (entry *Entry) isS3Entry() bool {
+	if entry.Extended == nil {
+		return false
+	}
+
+	if _, found := entry.Extended[s3_constants.ExtAmzOwnerKey]; found {
+		return true
+	}
+
+	if _, found := entry.Extended[s3_constants.ExtETagKey]; found {
+		return true
+	}
+
+	for key := range entry.Extended {
+		if strings.HasPrefix(key, s3_constants.ExtAmzPrefix) {
+			return true
+		}
+	}
+	return false
+}
+
+// ApplyStorageTtl stamps the TTL the entry's data was placed with. Remote
+// entries never expire locally; the remote storage owns their lifecycle.
+func (entry *Entry) ApplyStorageTtl(ttlSec int32) {
+	if entry.Remote != nil {
+		ttlSec = 0
+	}
+	entry.TtlSec = ttlSec
+	entry.ApplyS3ExpiryMetadata()
+}
+
+func (entry *Entry) ApplyS3ExpiryMetadata() {
+	if entry.TtlSec == 0 {
+		return
+	}
+
+	if _, found := entry.Extended[s3_constants.SeaweedFSExpiresS3]; found {
+		return
+	}
+
+	// The s3 expiry path skips versioned entries, so stamping one would drop its
+	// expiry entirely instead of moving it onto mtime.
+	if entry.IsS3Versioning() {
+		return
+	}
+
+	if !entry.isS3Entry() {
+		return
+	}
+
+	entry.Extended[s3_constants.SeaweedFSExpiresS3] = []byte("true")
+
+	return
+}
+
+func (entry *Entry) IsS3Versioning() (exist bool) {
+	if entry.Extended != nil {
+		_, exist = entry.Extended[s3_constants.ExtVersionIdKey]
+	}
+	return exist
+}
+
+func (entry *Entry) GetS3ExpireTime() (expireTime time.Time) {
+	if entry.Mtime.IsZero() {
+		expireTime = entry.Crtime
+	} else {
+		expireTime = entry.Mtime
+	}
+	return expireTime.Add(time.Duration(entry.TtlSec) * time.Second)
+}

weed/filer/entry_codec.go

@@ -0,0 +1,240 @@
+package filer
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"sync"
+	"time"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// pbEntryPool reduces allocations in EncodeAttributesAndChunks and DecodeAttributesAndChunks
+// which are called on every filer store operation
+var pbEntryPool = sync.Pool{
+	New: func() interface{} {
+		return &filer_pb.Entry{
+			Attributes: &filer_pb.FuseAttributes{}, // Pre-allocate attributes
+		}
+	},
+}
+
+// resetPbEntry clears a protobuf Entry for reuse
+func resetPbEntry(e *filer_pb.Entry) {
+	// Use struct assignment to clear all fields including protobuf internal fields
+	// (unknownFields, sizeCache) that field-by-field reset would miss
+	attrs := e.Attributes
+	*e = filer_pb.Entry{}
+	if attrs == nil {
+		attrs = &filer_pb.FuseAttributes{}
+	} else {
+		resetFuseAttributes(attrs)
+	}
+	e.Attributes = attrs
+}
+
+// resetFuseAttributes clears FuseAttributes for reuse
+func resetFuseAttributes(a *filer_pb.FuseAttributes) {
+	// Use struct assignment to clear all fields including protobuf internal fields
+	*a = filer_pb.FuseAttributes{}
+}
+
+func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {
+	message := pbEntryPool.Get().(*filer_pb.Entry)
+	defer func() {
+		resetPbEntry(message)
+		pbEntryPool.Put(message)
+	}()
+
+	entry.ToExistingProtoEntry(message)
+
+	data, err := proto.Marshal(message)
+	if err != nil {
+		return nil, err
+	}
+
+	// Copy the data to a new slice since proto.Marshal may return a slice
+	// that shares memory with the message (not guaranteed to be a copy)
+	return append([]byte(nil), data...), nil
+}
+
+func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error {
+	message := pbEntryPool.Get().(*filer_pb.Entry)
+	defer func() {
+		resetPbEntry(message)
+		pbEntryPool.Put(message)
+	}()
+
+	if err := proto.Unmarshal(blob, message); err != nil {
+		return fmt.Errorf("decoding value blob for %s: %v", entry.FullPath, err)
+	}
+
+	FromPbEntryToExistingEntry(message, entry)
+
+	return nil
+}
+
+func EntryAttributeToPb(entry *Entry) *filer_pb.FuseAttributes {
+
+	return &filer_pb.FuseAttributes{
+		Crtime:        entry.Attr.Crtime.Unix(),
+		Mtime:         entry.Attr.Mtime.Unix(),
+		MtimeNs:       int32(entry.Attr.Mtime.Nanosecond()),
+		Ctime:         entry.Attr.Ctime.Unix(),
+		CtimeNs:       int32(entry.Attr.Ctime.Nanosecond()),
+		Atime:         atimeSecondsForPb(entry.Attr),
+		AtimeNs:       atimeNanosForPb(entry.Attr),
+		FileMode:      uint32(entry.Attr.Mode),
+		Uid:           entry.Uid,
+		Gid:           entry.Gid,
+		Mime:          entry.Mime,
+		TtlSec:        entry.Attr.TtlSec,
+		UserName:      entry.Attr.UserName,
+		GroupName:     entry.Attr.GroupNames,
+		SymlinkTarget: entry.Attr.SymlinkTarget,
+		Md5:           entry.Attr.Md5,
+		FileSize:      entry.Attr.FileSize,
+		Rdev:          entry.Attr.Rdev,
+		Inode:         entry.Attr.Inode,
+	}
+}
+
+func atimeSecondsForPb(attr Attr) int64 {
+	if attr.Atime.IsZero() {
+		return 0
+	}
+	return attr.Atime.Unix()
+}
+
+func atimeNanosForPb(attr Attr) int32 {
+	if attr.Atime.IsZero() {
+		return 0
+	}
+	return int32(attr.Atime.Nanosecond())
+}
+
+// EntryAttributeToExistingPb fills an existing FuseAttributes to avoid allocation.
+// Safe to call with nil attr (will return early without populating).
+func EntryAttributeToExistingPb(entry *Entry, attr *filer_pb.FuseAttributes) {
+	if attr == nil {
+		return
+	}
+	attr.Crtime = entry.Attr.Crtime.Unix()
+	attr.Mtime = entry.Attr.Mtime.Unix()
+	attr.MtimeNs = int32(entry.Attr.Mtime.Nanosecond())
+	attr.Ctime = entry.Attr.Ctime.Unix()
+	attr.CtimeNs = int32(entry.Attr.Ctime.Nanosecond())
+	attr.Atime = atimeSecondsForPb(entry.Attr)
+	attr.AtimeNs = atimeNanosForPb(entry.Attr)
+	attr.FileMode = uint32(entry.Attr.Mode)
+	attr.Uid = entry.Uid
+	attr.Gid = entry.Gid
+	attr.Mime = entry.Mime
+	attr.TtlSec = entry.Attr.TtlSec
+	attr.UserName = entry.Attr.UserName
+	attr.GroupName = entry.Attr.GroupNames
+	attr.SymlinkTarget = entry.Attr.SymlinkTarget
+	attr.Md5 = entry.Attr.Md5
+	attr.FileSize = entry.Attr.FileSize
+	attr.Rdev = entry.Attr.Rdev
+	attr.Inode = entry.Attr.Inode
+}
+
+func PbToEntryAttribute(attr *filer_pb.FuseAttributes) Attr {
+
+	t := Attr{}
+
+	if attr == nil {
+		return t
+	}
+
+	t.Crtime = time.Unix(attr.Crtime, 0)
+	t.Mtime = time.Unix(attr.Mtime, int64(attr.MtimeNs))
+	if attr.Ctime != 0 {
+		t.Ctime = time.Unix(attr.Ctime, int64(attr.CtimeNs))
+	} else {
+		t.Ctime = t.Mtime
+	}
+	if attr.Atime != 0 || attr.AtimeNs != 0 {
+		t.Atime = time.Unix(attr.Atime, int64(attr.AtimeNs))
+	} else {
+		t.Atime = t.Mtime
+	}
+	t.Mode = os.FileMode(attr.FileMode)
+	t.Uid = attr.Uid
+	t.Gid = attr.Gid
+	t.Mime = attr.Mime
+	t.TtlSec = attr.TtlSec
+	t.UserName = attr.UserName
+	t.GroupNames = attr.GroupName
+	t.SymlinkTarget = attr.SymlinkTarget
+	t.Md5 = attr.Md5
+	t.FileSize = attr.FileSize
+	t.Rdev = attr.Rdev
+	t.Inode = attr.Inode
+
+	return t
+}
+
+func EqualEntry(a, b *Entry) bool {
+	if a == b {
+		return true
+	}
+	if a == nil && b != nil || a != nil && b == nil {
+		return false
+	}
+	if !proto.Equal(EntryAttributeToPb(a), EntryAttributeToPb(b)) {
+		return false
+	}
+	if len(a.Chunks) != len(b.Chunks) {
+		return false
+	}
+
+	if !eq(a.Extended, b.Extended) {
+		return false
+	}
+
+	if !bytes.Equal(a.Md5, b.Md5) {
+		return false
+	}
+
+	for i := 0; i < len(a.Chunks); i++ {
+		if !proto.Equal(a.Chunks[i], b.Chunks[i]) {
+			return false
+		}
+	}
+
+	if !bytes.Equal(a.HardLinkId, b.HardLinkId) {
+		return false
+	}
+	if a.HardLinkCounter != b.HardLinkCounter {
+		return false
+	}
+	if !bytes.Equal(a.Content, b.Content) {
+		return false
+	}
+	if !proto.Equal(a.Remote, b.Remote) {
+		return false
+	}
+	if a.Quota != b.Quota {
+		return false
+	}
+	return true
+}
+
+func eq(a, b map[string][]byte) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for k, v := range a {
+		if w, ok := b[k]; !ok || !bytes.Equal(v, w) {
+			return false
+		}
+	}
+
+	return true
+}

weed/filer/entry_codec_atime_test.go

@@ -0,0 +1,65 @@
+package filer
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestEntryCodec_AtimeRoundTrip(t *testing.T) {
+	mtime := time.Unix(1_700_000_000, 123_456_789)
+	atime := time.Unix(1_700_001_000, 987_654_321)
+
+	source := &Entry{
+		FullPath: util.FullPath("/bucket/object"),
+		Attr: Attr{
+			Mtime: mtime,
+			Ctime: mtime,
+			Atime: atime,
+		},
+	}
+	pb := EntryAttributeToPb(source)
+	if pb.Atime != atime.Unix() {
+		t.Fatalf("expected proto atime %d, got %d", atime.Unix(), pb.Atime)
+	}
+	if pb.AtimeNs != int32(atime.Nanosecond()) {
+		t.Fatalf("expected proto atime_ns %d, got %d", atime.Nanosecond(), pb.AtimeNs)
+	}
+
+	decoded := PbToEntryAttribute(pb)
+	if !decoded.Atime.Equal(atime) {
+		t.Fatalf("expected decoded atime %v, got %v", atime, decoded.Atime)
+	}
+}
+
+func TestEntryCodec_AtimeZeroFallsBackToMtime(t *testing.T) {
+	mtime := time.Unix(1_700_000_000, 0)
+	pb := &filer_pb.FuseAttributes{
+		Mtime:   mtime.Unix(),
+		MtimeNs: int32(mtime.Nanosecond()),
+	}
+	decoded := PbToEntryAttribute(pb)
+	if !decoded.Atime.Equal(mtime) {
+		t.Fatalf("expected atime to fall back to mtime %v, got %v", mtime, decoded.Atime)
+	}
+}
+
+// Atime in the first second of the unix epoch encodes as Atime=0 with
+// AtimeNs>0; the decode path must treat that as a valid timestamp rather than
+// falling back to Mtime.
+func TestEntryCodec_AtimeSubSecondEpochPreserved(t *testing.T) {
+	mtime := time.Unix(1_700_000_000, 0)
+	pb := &filer_pb.FuseAttributes{
+		Mtime:   mtime.Unix(),
+		MtimeNs: int32(mtime.Nanosecond()),
+		Atime:   0,
+		AtimeNs: 500_000,
+	}
+	decoded := PbToEntryAttribute(pb)
+	want := time.Unix(0, 500_000)
+	if !decoded.Atime.Equal(want) {
+		t.Fatalf("expected sub-second-epoch atime %v, got %v", want, decoded.Atime)
+	}
+}

weed/filer/entry_codec_attributes.go

@@ -0,0 +1,213 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"unicode/utf8"
+
+	"google.golang.org/protobuf/encoding/protowire"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// Field numbers from filer.proto. Only the ones this decoder has to recognise
+// are named; every other Entry field is handed to the generated unmarshaller
+// as-is.
+const (
+	entryChunksField     = 3 // Entry.chunks
+	fileChunkOffsetField = 2 // FileChunk.offset
+	fileChunkSizeField   = 3 // FileChunk.size
+)
+
+// The chunk bytes are the one part of the blob the generated unmarshaller never
+// sees, so the checks it would have made are made here instead: a submessage
+// has to parse, and a proto3 string has to be valid UTF-8. Anything else in a
+// FileChunk is a scalar, which walking it already validates.
+// TestChunkValidationCoversEveryField fails if FileChunk gains a field of
+// either kind that is missing from these.
+var (
+	fileChunkMessageFields = map[protowire.Number]bool{
+		7: true, // fid, a FileId of scalars only, so walking it is a full check
+		8: true, // source_fid
+	}
+	fileChunkStringFields = map[protowire.Number]bool{
+		1: true, // file_id
+		5: true, // e_tag
+		6: true, // source_file_id
+	}
+)
+
+// attributesScratchPool holds the re-encoded entry, which is the blob minus its
+// chunks and so much smaller than what came in.
+var attributesScratchPool = sync.Pool{
+	New: func() any {
+		b := make([]byte, 0, 256)
+		return &b
+	},
+}
+
+// DecodeListedEntry decodes one listed entry, dropping the chunk list when the
+// listing asked for attributes only.
+func DecodeListedEntry(ctx context.Context, entry *Entry, blob []byte) error {
+	if filer_pb.ChunksOmitted(ctx) {
+		return entry.DecodeAttributesOnly(blob)
+	}
+	return entry.DecodeAttributesAndChunks(blob)
+}
+
+// DecodeAttributesOnly fills entry from blob without building its chunk list,
+// which is the bulk of the work for anything but a tiny file. Chunks are still
+// measured, because an entry whose stored FileSize is zero takes its size from
+// them, but no FileChunk is allocated.
+//
+// entry.Chunks is left nil. The one exception is a hard link, whose attributes
+// and chunks are replaced wholesale by a full decode of its own record in
+// FilerStoreWrapper.maybeReadHardLink straight after the store listing. Only a
+// caller that reads attributes and nothing else may use this.
+func (entry *Entry) DecodeAttributesOnly(blob []byte) error {
+	// The scratch buffer is only taken once a chunk is actually found, so an
+	// entry with none — every directory, for one — neither re-encodes nor
+	// touches the pool, and is unmarshalled where it lies.
+	var scratchPtr *[]byte
+	var scratch []byte
+	defer func() {
+		if scratchPtr != nil {
+			*scratchPtr = scratch
+			attributesScratchPool.Put(scratchPtr)
+		}
+	}()
+
+	var chunkExtent uint64
+	attributes := blob
+	for pos := 0; pos < len(blob); {
+		rest := blob[pos:]
+		num, typ, tagLen := protowire.ConsumeTag(rest)
+		if tagLen < 0 {
+			return fmt.Errorf("decoding value blob for %s: %w", entry.FullPath, protowire.ParseError(tagLen))
+		}
+		var valLen int
+		if num == entryChunksField && typ == protowire.BytesType {
+			chunk, n := protowire.ConsumeBytes(rest[tagLen:])
+			if n < 0 {
+				return fmt.Errorf("decoding value blob for %s: %w", entry.FullPath, protowire.ParseError(n))
+			}
+			valLen = n
+			end, err := chunkExtentEnd(chunk)
+			if err != nil {
+				return fmt.Errorf("decoding value blob for %s: %w", entry.FullPath, err)
+			}
+			if end > chunkExtent {
+				chunkExtent = end
+			}
+			if scratchPtr == nil {
+				scratchPtr = attributesScratchPool.Get().(*[]byte)
+				scratch = append((*scratchPtr)[:0], blob[:pos]...)
+			}
+		} else {
+			valLen = protowire.ConsumeFieldValue(num, typ, rest[tagLen:])
+			if valLen < 0 {
+				return fmt.Errorf("decoding value blob for %s: %w", entry.FullPath, protowire.ParseError(valLen))
+			}
+			if scratchPtr != nil {
+				scratch = append(scratch, rest[:tagLen+valLen]...)
+			}
+		}
+		pos += tagLen + valLen
+	}
+	if scratchPtr != nil {
+		attributes = scratch
+	}
+
+	message := pbEntryPool.Get().(*filer_pb.Entry)
+	defer func() {
+		resetPbEntry(message)
+		pbEntryPool.Put(message)
+	}()
+
+	if err := proto.Unmarshal(attributes, message); err != nil {
+		return fmt.Errorf("decoding value blob for %s: %v", entry.FullPath, err)
+	}
+
+	FromPbEntryToExistingEntry(message, entry)
+
+	// FromPbEntryToExistingEntry took the size over a chunk list that is not
+	// there, so fold in what the chunks actually reached. This is TotalSize.
+	if chunkExtent > entry.FileSize {
+		entry.FileSize = chunkExtent
+	}
+
+	return nil
+}
+
+// chunkExtentEnd reports where one encoded FileChunk ends, the offset plus size
+// that TotalSize maximises over, without building the chunk. A chunk this
+// rejects is one the full decoder rejects too, so a listing never reports a
+// size for an entry that cannot be opened.
+//
+// A manifest chunk needs no special handling: it carries the offset and size of
+// the whole range it stands for, and TotalSize does not resolve it either.
+func chunkExtentEnd(chunk []byte) (uint64, error) {
+	var offset int64
+	var size uint64
+	for len(chunk) > 0 {
+		num, typ, tagLen := protowire.ConsumeTag(chunk)
+		if tagLen < 0 {
+			return 0, protowire.ParseError(tagLen)
+		}
+		chunk = chunk[tagLen:]
+		if typ == protowire.VarintType && (num == fileChunkOffsetField || num == fileChunkSizeField) {
+			v, n := protowire.ConsumeVarint(chunk)
+			if n < 0 {
+				return 0, protowire.ParseError(n)
+			}
+			if num == fileChunkOffsetField {
+				offset = int64(v)
+			} else {
+				size = v
+			}
+			chunk = chunk[n:]
+			continue
+		}
+		if typ == protowire.BytesType {
+			v, n := protowire.ConsumeBytes(chunk)
+			if n < 0 {
+				return 0, protowire.ParseError(n)
+			}
+			if fileChunkMessageFields[num] {
+				if err := validateMessage(v); err != nil {
+					return 0, err
+				}
+			} else if fileChunkStringFields[num] && !utf8.Valid(v) {
+				return 0, errors.New("invalid UTF-8 in string field")
+			}
+			chunk = chunk[n:]
+			continue
+		}
+		n := protowire.ConsumeFieldValue(num, typ, chunk)
+		if n < 0 {
+			return 0, protowire.ParseError(n)
+		}
+		chunk = chunk[n:]
+	}
+	return uint64(offset + int64(size)), nil
+}
+
+// validateMessage walks an encoded message to check it parses, which is all the
+// generated unmarshaller would do for one whose fields are scalars.
+func validateMessage(b []byte) error {
+	for len(b) > 0 {
+		num, typ, tagLen := protowire.ConsumeTag(b)
+		if tagLen < 0 {
+			return protowire.ParseError(tagLen)
+		}
+		valLen := protowire.ConsumeFieldValue(num, typ, b[tagLen:])
+		if valLen < 0 {
+			return protowire.ParseError(valLen)
+		}
+		b = b[tagLen+valLen:]
+	}
+	return nil
+}

weed/filer/entry_codec_attributes_test.go

@@ -0,0 +1,444 @@
+package filer
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/encoding/protowire"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+// decodeBothWays round-trips entry and returns what each decoder made of it.
+func decodeBothWays(t *testing.T, entry *Entry) (full, attrsOnly Entry) {
+	t.Helper()
+	blob, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		t.Fatalf("encode: %v", err)
+	}
+	full.FullPath = entry.FullPath
+	if err := full.DecodeAttributesAndChunks(blob); err != nil {
+		t.Fatalf("full decode: %v", err)
+	}
+	attrsOnly.FullPath = entry.FullPath
+	if err := attrsOnly.DecodeAttributesOnly(blob); err != nil {
+		t.Fatalf("attributes-only decode: %v", err)
+	}
+	return full, attrsOnly
+}
+
+// assertSameButChunks checks the two decoders agree on everything a listing
+// reads. Chunks are the deliberate exception; size is not, because an entry can
+// carry a zero FileSize and take its size from the chunks.
+func assertSameButChunks(t *testing.T, full, attrsOnly Entry) {
+	t.Helper()
+	if attrsOnly.Chunks != nil {
+		t.Errorf("attributes-only decode built %d chunks, want none", len(attrsOnly.Chunks))
+	}
+	if full.FileSize != attrsOnly.FileSize {
+		t.Errorf("FileSize = %d, want %d", attrsOnly.FileSize, full.FileSize)
+	}
+	if full.Size() != attrsOnly.Size() {
+		t.Errorf("Size() = %d, want %d", attrsOnly.Size(), full.Size())
+	}
+	if !reflect.DeepEqual(full.Attr, attrsOnly.Attr) {
+		t.Errorf("Attr = %+v, want %+v", attrsOnly.Attr, full.Attr)
+	}
+	if string(full.HardLinkId) != string(attrsOnly.HardLinkId) {
+		t.Errorf("HardLinkId = %x, want %x", attrsOnly.HardLinkId, full.HardLinkId)
+	}
+	if full.HardLinkCounter != attrsOnly.HardLinkCounter {
+		t.Errorf("HardLinkCounter = %d, want %d", attrsOnly.HardLinkCounter, full.HardLinkCounter)
+	}
+	if string(full.Content) != string(attrsOnly.Content) {
+		t.Errorf("Content = %q, want %q", attrsOnly.Content, full.Content)
+	}
+	if full.Quota != attrsOnly.Quota {
+		t.Errorf("Quota = %d, want %d", attrsOnly.Quota, full.Quota)
+	}
+	if full.WORMEnforcedAtTsNs != attrsOnly.WORMEnforcedAtTsNs {
+		t.Errorf("WORMEnforcedAtTsNs = %d, want %d", attrsOnly.WORMEnforcedAtTsNs, full.WORMEnforcedAtTsNs)
+	}
+	if len(full.Extended) != len(attrsOnly.Extended) {
+		t.Errorf("Extended has %d keys, want %d", len(attrsOnly.Extended), len(full.Extended))
+	}
+	for k, v := range full.Extended {
+		if string(attrsOnly.Extended[k]) != string(v) {
+			t.Errorf("Extended[%q] = %q, want %q", k, attrsOnly.Extended[k], v)
+		}
+	}
+	if (full.Remote == nil) != (attrsOnly.Remote == nil) {
+		t.Errorf("Remote presence differs: %v vs %v", attrsOnly.Remote != nil, full.Remote != nil)
+	} else if full.Remote != nil && full.Remote.RemoteSize != attrsOnly.Remote.RemoteSize {
+		t.Errorf("Remote.RemoteSize = %d, want %d", attrsOnly.Remote.RemoteSize, full.Remote.RemoteSize)
+	}
+}
+
+func chunkAt(offset int64, size uint64, i int) *filer_pb.FileChunk {
+	return &filer_pb.FileChunk{
+		FileId:       fmt.Sprintf("3,01637037d6%04d", i),
+		Offset:       offset,
+		Size:         size,
+		ModifiedTsNs: int64(1700000000+i) * 1e9,
+		ETag:         "1a2b3c4d5e6f7890",
+		Fid:          &filer_pb.FileId{VolumeId: uint32(3 + i), FileKey: uint64(i), Cookie: 0x1637037d},
+		CipherKey:    []byte{1, 2, 3, 4},
+	}
+}
+
+func TestDecodeAttributesOnlyMatchesFullDecode(t *testing.T) {
+	now := time.Unix(1700000000, 123456789)
+
+	cases := []struct {
+		name  string
+		entry *Entry
+	}{
+		{"no chunks", &Entry{
+			FullPath: util.FullPath("/d/plain"),
+			Attr:     Attr{Mode: 0o644, Mtime: now, Crtime: now, Ctime: now, Uid: 99, Gid: 100, FileSize: 12},
+		}},
+		{"directory", &Entry{
+			FullPath: util.FullPath("/d/sub"),
+			Attr:     Attr{Mode: os.ModeDir | 0o755, Mtime: now, Crtime: now, Uid: 99, Gid: 100},
+		}},
+		{"one chunk", &Entry{
+			FullPath: util.FullPath("/d/one"),
+			Attr:     Attr{Mode: 0o644, Mtime: now, Crtime: now, Uid: 99, Gid: 100, FileSize: 4 << 20},
+			Chunks:   []*filer_pb.FileChunk{chunkAt(0, 4<<20, 0)},
+		}},
+		// The S3 copy and multipart paths deliberately store a zero FileSize and
+		// let the chunks define it, so this is the case that forces the extent
+		// walk rather than just skipping the field.
+		{"zero FileSize, size comes from chunks", &Entry{
+			FullPath: util.FullPath("/d/zerosize"),
+			Attr:     Attr{Mode: 0o644, Mtime: now, Crtime: now, Uid: 99, Gid: 100, FileSize: 0},
+			Chunks: []*filer_pb.FileChunk{
+				chunkAt(0, 4<<20, 0), chunkAt(4<<20, 4<<20, 1), chunkAt(8<<20, 1234, 2),
+			},
+		}},
+		{"chunks out of order", &Entry{
+			FullPath: util.FullPath("/d/unordered"),
+			Attr:     Attr{Mode: 0o644, Mtime: now, Crtime: now, Uid: 99, Gid: 100},
+			Chunks: []*filer_pb.FileChunk{
+				chunkAt(8<<20, 99, 0), chunkAt(0, 4<<20, 1), chunkAt(4<<20, 4<<20, 2),
+			},
+		}},
+		{"stored FileSize larger than chunks", &Entry{
+			FullPath: util.FullPath("/d/sparse"),
+			Attr:     Attr{Mode: 0o644, Mtime: now, Crtime: now, Uid: 99, Gid: 100, FileSize: 1 << 30},
+			Chunks:   []*filer_pb.FileChunk{chunkAt(0, 16, 0)},
+		}},
+		{"symlink", &Entry{
+			FullPath: util.FullPath("/d/link"),
+			Attr:     Attr{Mode: os.ModeSymlink | 0o777, Mtime: now, Crtime: now, SymlinkTarget: "../target"},
+		}},
+		{"hard link", &Entry{
+			FullPath:        util.FullPath("/d/hard"),
+			Attr:            Attr{Mode: 0o644, Mtime: now, Crtime: now},
+			HardLinkId:      HardLinkId([]byte{9, 8, 7, 6}),
+			HardLinkCounter: 3,
+		}},
+		{"inline content", &Entry{
+			FullPath: util.FullPath("/d/inline"),
+			Attr:     Attr{Mode: 0o644, Mtime: now, Crtime: now},
+			Content:  []byte("hello world"),
+		}},
+		{"extended attributes", &Entry{
+			FullPath: util.FullPath("/d/xattr"),
+			Attr:     Attr{Mode: 0o644, Mtime: now, Crtime: now},
+			Extended: map[string][]byte{"a": []byte("1"), "b": []byte("2"), "Seaweed-X": []byte("y")},
+		}},
+		{"remote entry", &Entry{
+			FullPath: util.FullPath("/d/remote"),
+			Attr:     Attr{Mode: 0o644, Mtime: now, Crtime: now, FileSize: 5},
+			Remote:   &filer_pb.RemoteEntry{RemoteSize: 4096, RemoteMtime: now.Unix() + 60, StorageName: "s3"},
+		}},
+		{"quota and worm", &Entry{
+			FullPath:           util.FullPath("/d/bucket"),
+			Attr:               Attr{Mode: os.ModeDir | 0o755, Mtime: now, Crtime: now},
+			Quota:              1 << 40,
+			WORMEnforcedAtTsNs: now.UnixNano(),
+		}},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			full, attrsOnly := decodeBothWays(t, tc.entry)
+			assertSameButChunks(t, full, attrsOnly)
+		})
+	}
+}
+
+// TestDecodeAttributesOnlyRandomEntries fuzzes the field combinations, since the
+// decoder walks the wire format by hand and has to stay in step with the
+// generated one as filer.proto grows.
+func TestDecodeAttributesOnlyRandomEntries(t *testing.T) {
+	rnd := rand.New(rand.NewSource(1))
+	for i := 0; i < 2000; i++ {
+		now := time.Unix(1600000000+rnd.Int63n(1e8), rnd.Int63n(1e9))
+		entry := &Entry{
+			FullPath: util.FullPath(fmt.Sprintf("/d/f%d", i)),
+			Attr: Attr{
+				Mode:  os.FileMode(rnd.Intn(0o777)),
+				Mtime: now, Crtime: now, Ctime: now,
+				Uid: uint32(rnd.Intn(70000)), Gid: uint32(rnd.Intn(70000)),
+				FileSize: uint64(rnd.Int63n(1 << 34)),
+				Inode:    rnd.Uint64(),
+				Rdev:     uint32(rnd.Intn(1 << 20)),
+				TtlSec:   int32(rnd.Intn(1000)),
+			},
+		}
+		if rnd.Intn(2) == 0 {
+			entry.Attr.FileSize = 0
+		}
+		if rnd.Intn(4) == 0 {
+			entry.Content = make([]byte, rnd.Intn(64))
+			rnd.Read(entry.Content)
+		}
+		if rnd.Intn(4) == 0 {
+			entry.Extended = map[string][]byte{}
+			for k := 0; k < rnd.Intn(4); k++ {
+				entry.Extended[fmt.Sprintf("k%d", k)] = []byte(fmt.Sprintf("v%d", rnd.Intn(1000)))
+			}
+		}
+		if rnd.Intn(8) == 0 {
+			entry.Remote = &filer_pb.RemoteEntry{RemoteSize: rnd.Int63n(1 << 30), RemoteMtime: now.Unix() + int64(rnd.Intn(120)) - 60}
+		}
+		for c := 0; c < rnd.Intn(20); c++ {
+			entry.Chunks = append(entry.Chunks, chunkAt(rnd.Int63n(1<<30), uint64(rnd.Int63n(1<<22)), c))
+		}
+		full, attrsOnly := decodeBothWays(t, entry)
+		assertSameButChunks(t, full, attrsOnly)
+	}
+}
+
+func TestDecodeAttributesOnlyRejectsGarbage(t *testing.T) {
+	var entry Entry
+	entry.FullPath = util.FullPath("/d/bad")
+	if err := entry.DecodeAttributesOnly([]byte{0xff, 0xff, 0xff, 0xff}); err == nil {
+		t.Fatal("expected an error for a malformed blob")
+	}
+}
+
+func BenchmarkDecode(b *testing.B) {
+	now := time.Unix(1700000000, 0)
+	for _, n := range []int{0, 1, 4, 16, 64} {
+		entry := &Entry{
+			FullPath: util.FullPath("/images/image-00000001.jpg"),
+			Attr:     Attr{Mode: 0o644, Mtime: now, Crtime: now, Ctime: now, Uid: 99, Gid: 100, FileSize: uint64(n) * 4 << 20},
+		}
+		for i := 0; i < n; i++ {
+			entry.Chunks = append(entry.Chunks, chunkAt(int64(i)*4<<20, 4<<20, i))
+		}
+		blob, err := entry.EncodeAttributesAndChunks()
+		if err != nil {
+			b.Fatal(err)
+		}
+		b.Run(fmt.Sprintf("chunks=%d/decoder=full", n), func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				var out Entry
+				if err := out.DecodeAttributesAndChunks(blob); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+		b.Run(fmt.Sprintf("chunks=%d/decoder=attrsonly", n), func(b *testing.B) {
+			b.ReportAllocs()
+			for i := 0; i < b.N; i++ {
+				var out Entry
+				if err := out.DecodeAttributesOnly(blob); err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}
+
+// TestProtoEntryWithoutChunksKeepsSize covers the wire contract the filer's
+// omit_chunks listing relies on: the size a client needs survives in the
+// attributes once the chunk list is dropped, including for the entries that
+// store a zero FileSize and take their size from the chunks.
+func TestProtoEntryWithoutChunksKeepsSize(t *testing.T) {
+	now := time.Unix(1700000000, 0)
+	for _, storedSize := range []uint64{0, 7, 1 << 30} {
+		stored := &Entry{
+			FullPath: util.FullPath("/d/obj"),
+			Attr:     Attr{Mode: 0o644, Mtime: now, Crtime: now, FileSize: storedSize},
+			Chunks:   []*filer_pb.FileChunk{chunkAt(0, 4<<20, 0), chunkAt(4<<20, 1234, 1)},
+		}
+		blob, err := stored.EncodeAttributesAndChunks()
+		if err != nil {
+			t.Fatalf("encode: %v", err)
+		}
+		// What the filer holds after reading the entry out of its store.
+		var loaded Entry
+		loaded.FullPath = stored.FullPath
+		if err := loaded.DecodeAttributesAndChunks(blob); err != nil {
+			t.Fatalf("decode: %v", err)
+		}
+		want := FileSize(loaded.ToProtoEntry())
+
+		pbEntry := loaded.ToProtoEntry()
+		pbEntry.Chunks = nil
+		if got := FileSize(pbEntry); got != want {
+			t.Errorf("stored FileSize %d: size over the wire = %d, want %d", storedSize, got, want)
+		}
+		if got := FromPbEntry("/d", pbEntry).Size(); got != want {
+			t.Errorf("stored FileSize %d: size at the client = %d, want %d", storedSize, got, want)
+		}
+	}
+}
+
+// TestChunkValidationCoversEveryField keeps the hand-rolled chunk walk honest as
+// filer.proto grows. The generated unmarshaller never sees the chunk bytes, so
+// every FileChunk field whose contents it would have checked -- a submessage, or
+// a proto3 string's UTF-8 -- has to be listed for the walk to check instead.
+func TestChunkValidationCoversEveryField(t *testing.T) {
+	fields := (&filer_pb.FileChunk{}).ProtoReflect().Descriptor().Fields()
+	for i := 0; i < fields.Len(); i++ {
+		f := fields.Get(i)
+		switch f.Kind() {
+		case protoreflect.MessageKind, protoreflect.GroupKind:
+			if !fileChunkMessageFields[protowire.Number(f.Number())] {
+				t.Errorf("FileChunk.%s (field %d) is a message but is not in fileChunkMessageFields, so a corrupt one would pass the listing decoder and fail the full one", f.Name(), f.Number())
+			}
+		case protoreflect.StringKind:
+			if !fileChunkStringFields[protowire.Number(f.Number())] {
+				t.Errorf("FileChunk.%s (field %d) is a string but is not in fileChunkStringFields, so invalid UTF-8 would pass the listing decoder and fail the full one", f.Name(), f.Number())
+			}
+		}
+	}
+}
+
+// corruptChunkBlobs builds entry blobs whose chunk bytes are damaged in ways the
+// generated unmarshaller rejects.
+func corruptChunkBlobs(t *testing.T) map[string][]byte {
+	t.Helper()
+	now := time.Unix(1700000000, 0)
+	base := func() *filer_pb.FileChunk {
+		return &filer_pb.FileChunk{Offset: 0, Size: 1024, Fid: &filer_pb.FileId{VolumeId: 3, FileKey: 7, Cookie: 9}}
+	}
+	blobFor := func(mangle func(raw []byte) []byte) []byte {
+		chunk := base()
+		chunkBytes, err := proto.Marshal(chunk)
+		if err != nil {
+			t.Fatalf("marshal chunk: %v", err)
+		}
+		chunkBytes = mangle(chunkBytes)
+		var blob []byte
+		blob = protowire.AppendTag(blob, entryChunksField, protowire.BytesType)
+		blob = protowire.AppendBytes(blob, chunkBytes)
+		attrs, err := proto.Marshal(&filer_pb.FuseAttributes{FileSize: 0, Mtime: now.Unix(), FileMode: 0o644})
+		if err != nil {
+			t.Fatalf("marshal attrs: %v", err)
+		}
+		blob = protowire.AppendTag(blob, 4, protowire.BytesType)
+		return protowire.AppendBytes(blob, attrs)
+	}
+
+	out := map[string][]byte{}
+	// The exact probe from review: a nested fid whose payload is not a message.
+	out["corrupt nested fid"] = blobFor(func(raw []byte) []byte {
+		var b []byte
+		b = protowire.AppendTag(b, 2, protowire.VarintType)
+		b = protowire.AppendVarint(b, 0)
+		b = protowire.AppendTag(b, 3, protowire.VarintType)
+		b = protowire.AppendVarint(b, 1024)
+		b = protowire.AppendTag(b, 7, protowire.BytesType)
+		return protowire.AppendBytes(b, []byte{0xff, 0xff, 0xff, 0xff})
+	})
+	out["invalid utf8 in file_id"] = blobFor(func(raw []byte) []byte {
+		var b []byte
+		b = protowire.AppendTag(b, 1, protowire.BytesType)
+		b = protowire.AppendBytes(b, []byte{0xff, 0xfe, 0xfd})
+		b = protowire.AppendTag(b, 3, protowire.VarintType)
+		return protowire.AppendVarint(b, 1024)
+	})
+	out["truncated chunk"] = blobFor(func(raw []byte) []byte { return raw[:len(raw)-1] })
+	return out
+}
+
+// TestDecodeAttributesOnlyRejectsWhatFullDecodeRejects is the invariant that
+// keeps a listing from showing a file that cannot then be opened: the fast path
+// must never accept a blob the full decoder turns away.
+func TestDecodeAttributesOnlyRejectsWhatFullDecodeRejects(t *testing.T) {
+	for name, blob := range corruptChunkBlobs(t) {
+		t.Run(name, func(t *testing.T) {
+			var full, attrsOnly Entry
+			full.FullPath = util.FullPath("/d/corrupt")
+			attrsOnly.FullPath = full.FullPath
+			fullErr := full.DecodeAttributesAndChunks(blob)
+			attrsErr := attrsOnly.DecodeAttributesOnly(blob)
+			if fullErr == nil {
+				t.Skip("full decoder accepts this blob, nothing to match")
+			}
+			if attrsErr == nil {
+				t.Errorf("full decode rejected the blob (%v) but attributes-only accepted it with FileSize=%d", fullErr, attrsOnly.FileSize)
+			}
+		})
+	}
+}
+
+// TestDecodeAttributesOnlyManifestChunk pins that a manifest chunk needs no
+// resolving: it carries the offset and size of the range it stands for, and
+// TotalSize does not resolve it either, so both decoders see one number.
+func TestDecodeAttributesOnlyManifestChunk(t *testing.T) {
+	now := time.Unix(1700000000, 0)
+	manifest := chunkAt(0, 64<<20, 0)
+	manifest.IsChunkManifest = true
+	entry := &Entry{
+		FullPath: util.FullPath("/d/big"),
+		// Zero stored size, so the manifest's extent is the only source.
+		Attr:   Attr{Mode: 0o644, Mtime: now, Crtime: now, FileSize: 0},
+		Chunks: []*filer_pb.FileChunk{manifest},
+	}
+	full, attrsOnly := decodeBothWays(t, entry)
+	assertSameButChunks(t, full, attrsOnly)
+	if attrsOnly.FileSize != 64<<20 {
+		t.Errorf("FileSize = %d, want %d from the manifest extent", attrsOnly.FileSize, 64<<20)
+	}
+}
+
+// TestDecodeAttributesOnlyFieldBeforeChunks exercises the prefix copy, which no
+// other case reaches: EncodeAttributesAndChunks emits chunks before every field
+// the other tests set, so `dropping` always turns on at pos 0 there.
+func TestDecodeAttributesOnlyFieldBeforeChunks(t *testing.T) {
+	now := time.Unix(1700000000, 0)
+	attrs, err := proto.Marshal(&filer_pb.FuseAttributes{FileSize: 0, Mtime: now.Unix(), FileMode: 0o755})
+	if err != nil {
+		t.Fatalf("marshal attrs: %v", err)
+	}
+	chunkBytes, err := proto.Marshal(chunkAt(0, 4<<20, 0))
+	if err != nil {
+		t.Fatalf("marshal chunk: %v", err)
+	}
+	// is_directory (2) ahead of chunks (3), so the walk has a prefix to copy.
+	var blob []byte
+	blob = protowire.AppendTag(blob, 2, protowire.VarintType)
+	blob = protowire.AppendVarint(blob, 1)
+	blob = protowire.AppendTag(blob, entryChunksField, protowire.BytesType)
+	blob = protowire.AppendBytes(blob, chunkBytes)
+	blob = protowire.AppendTag(blob, 4, protowire.BytesType)
+	blob = protowire.AppendBytes(blob, attrs)
+
+	var full, attrsOnly Entry
+	full.FullPath = util.FullPath("/d/dirwithchunks")
+	attrsOnly.FullPath = full.FullPath
+	if err := full.DecodeAttributesAndChunks(blob); err != nil {
+		t.Fatalf("full decode: %v", err)
+	}
+	if err := attrsOnly.DecodeAttributesOnly(blob); err != nil {
+		t.Fatalf("attributes-only decode: %v", err)
+	}
+	assertSameButChunks(t, full, attrsOnly)
+	if attrsOnly.FileSize != 4<<20 {
+		t.Errorf("FileSize = %d, want %d", attrsOnly.FileSize, 4<<20)
+	}
+}

weed/filer/entry_s3_expiry_test.go

@@ -0,0 +1,47 @@
+package filer
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+)
+
+func TestApplyS3ExpiryMetadata(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		ttlSec   int32
+		extended map[string][]byte
+		want     bool
+	}{
+		{
+			name:     "s3 entry with ttl",
+			ttlSec:   3600,
+			extended: map[string][]byte{s3_constants.ExtETagKey: []byte("abc123")},
+			want:     true,
+		},
+		{
+			name:     "no ttl",
+			extended: map[string][]byte{s3_constants.ExtETagKey: []byte("abc123")},
+		},
+		{
+			name:   "not an s3 entry",
+			ttlSec: 3600,
+		},
+		{
+			name:   "versioned entry keeps crtime expiry",
+			ttlSec: 3600,
+			extended: map[string][]byte{
+				s3_constants.ExtETagKey:      []byte("abc123"),
+				s3_constants.ExtVersionIdKey: []byte("v1"),
+			},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			entry := &Entry{Attr: Attr{TtlSec: tc.ttlSec}, Extended: tc.extended}
+			entry.ApplyS3ExpiryMetadata()
+			if got := entry.IsExpireS3Enabled(); got != tc.want {
+				t.Fatalf("IsExpireS3Enabled() = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}

weed/filer/etcd/etcd_store.go

@@ -0,0 +1,263 @@
+package etcd
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"strings"
+	"time"
+
+	"go.etcd.io/etcd/client/pkg/v3/transport"
+
+	clientv3 "go.etcd.io/etcd/client/v3"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	weed_util "github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const (
+	DIR_FILE_SEPARATOR = byte(0x00)
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &EtcdStore{})
+}
+
+type EtcdStore struct {
+	client        *clientv3.Client
+	etcdKeyPrefix string
+	timeout       time.Duration
+}
+
+func (store *EtcdStore) GetName() string {
+	return "etcd"
+}
+
+func (store *EtcdStore) Initialize(configuration weed_util.Configuration, prefix string) error {
+	configuration.SetDefault(prefix+"servers", "localhost:2379")
+	configuration.SetDefault(prefix+"timeout", "3s")
+
+	servers := configuration.GetString(prefix + "servers")
+
+	username := configuration.GetString(prefix + "username")
+	password := configuration.GetString(prefix + "password")
+	store.etcdKeyPrefix = configuration.GetString(prefix + "key_prefix")
+
+	timeoutStr := configuration.GetString(prefix + "timeout")
+	timeout, err := time.ParseDuration(timeoutStr)
+	if err != nil {
+		return fmt.Errorf("parse etcd store timeout: %w", err)
+	}
+	store.timeout = timeout
+
+	certFile := configuration.GetString(prefix + "tls_client_crt_file")
+	keyFile := configuration.GetString(prefix + "tls_client_key_file")
+	caFile := configuration.GetString(prefix + "tls_ca_file")
+
+	var tlsConfig *tls.Config
+	if caFile != "" {
+		tlsInfo := transport.TLSInfo{
+			CertFile:      certFile,
+			KeyFile:       keyFile,
+			TrustedCAFile: caFile,
+		}
+		var err error
+		tlsConfig, err = tlsInfo.ClientConfig()
+		if err != nil {
+			return fmt.Errorf("TLS client configuration error: %w", err)
+		}
+	}
+
+	return store.initialize(servers, username, password, store.timeout, tlsConfig)
+}
+
+func (store *EtcdStore) initialize(servers, username, password string, timeout time.Duration, tlsConfig *tls.Config) error {
+	glog.Infof("filer store etcd: %s", servers)
+
+	client, err := clientv3.New(clientv3.Config{
+		Endpoints:   strings.Split(servers, ","),
+		Username:    username,
+		Password:    password,
+		DialTimeout: timeout,
+		TLS:         tlsConfig,
+	})
+	if err != nil {
+		return fmt.Errorf("connect to etcd %s: %s", servers, err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), store.timeout)
+	defer cancel()
+
+	resp, err := client.Status(ctx, client.Endpoints()[0])
+	if err != nil {
+		client.Close()
+		return fmt.Errorf("error checking etcd connection: %s", err)
+	}
+
+	glog.V(0).InfofCtx(ctx, "сonnection to etcd has been successfully verified. etcd version: %s", resp.Version)
+	store.client = client
+
+	return nil
+}
+
+func (store *EtcdStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *EtcdStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *EtcdStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *EtcdStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	key := genKey(entry.DirAndName())
+
+	meta, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
+	}
+
+	if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+		meta = weed_util.MaybeGzipData(meta)
+	}
+
+	if _, err := store.client.Put(ctx, store.etcdKeyPrefix+string(key), string(meta)); err != nil {
+		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
+	}
+
+	return nil
+}
+
+func (store *EtcdStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *EtcdStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) {
+	key := genKey(fullpath.DirAndName())
+
+	resp, err := store.client.Get(ctx, store.etcdKeyPrefix+string(key))
+	if err != nil {
+		return nil, fmt.Errorf("get %s : %v", fullpath, err)
+	}
+
+	if len(resp.Kvs) == 0 {
+		return nil, filer_pb.ErrNotFound
+	}
+
+	entry = &filer.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(resp.Kvs[0].Value))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *EtcdStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
+	key := genKey(fullpath.DirAndName())
+
+	if _, err := store.client.Delete(ctx, store.etcdKeyPrefix+string(key)); err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
+	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
+
+	if _, err := store.client.Delete(ctx, store.etcdKeyPrefix+string(directoryPrefix), clientv3.WithPrefix()); err != nil {
+		return fmt.Errorf("deleteFolderChildren %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix)
+	lastFileStart := directoryPrefix
+	if startFileName != "" {
+		lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
+	}
+
+	resp, err := store.client.Get(ctx, store.etcdKeyPrefix+string(lastFileStart),
+		clientv3.WithRange(clientv3.GetPrefixRangeEnd(store.etcdKeyPrefix+string(directoryPrefix))),
+		clientv3.WithLimit(limit+1))
+	if err != nil {
+		return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
+	}
+
+	for _, kv := range resp.Kvs {
+		fileName := getNameFromKey(kv.Key)
+		if fileName == "" {
+			continue
+		}
+		if fileName == startFileName && !includeStartFile {
+			continue
+		}
+		limit--
+		if limit < 0 {
+			break
+		}
+		entry := &filer.Entry{
+			FullPath: weed_util.NewFullPath(string(dirPath), fileName),
+		}
+		if decodeErr := entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData(kv.Value)); decodeErr != nil {
+			err = decodeErr
+			glog.V(0).InfofCtx(ctx, "list %s : %v", entry.FullPath, err)
+			break
+		}
+
+		resEachEntryFunc, resEachEntryFuncErr := eachEntryFunc(entry)
+		if resEachEntryFuncErr != nil {
+			err = fmt.Errorf("failed to process eachEntryFunc: %w", resEachEntryFuncErr)
+			break
+		}
+
+		if !resEachEntryFunc {
+			break
+		}
+
+		lastFileName = fileName
+	}
+
+	return lastFileName, err
+}
+
+func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc)
+}
+
+func genKey(dirPath, fileName string) (key []byte) {
+	key = []byte(dirPath)
+	key = append(key, DIR_FILE_SEPARATOR)
+	key = append(key, []byte(fileName)...)
+	return key
+}
+
+func genDirectoryKeyPrefix(fullpath weed_util.FullPath, startFileName string) (keyPrefix []byte) {
+	keyPrefix = []byte(string(fullpath))
+	keyPrefix = append(keyPrefix, DIR_FILE_SEPARATOR)
+	if len(startFileName) > 0 {
+		keyPrefix = append(keyPrefix, []byte(startFileName)...)
+	}
+	return keyPrefix
+}
+
+func getNameFromKey(key []byte) string {
+	sepIndex := len(key) - 1
+	for sepIndex >= 0 && key[sepIndex] != DIR_FILE_SEPARATOR {
+		sepIndex--
+	}
+
+	return string(key[sepIndex+1:])
+}
+
+func (store *EtcdStore) Shutdown() {
+	store.client.Close()
+}

weed/filer/etcd/etcd_store_kv.go

@@ -0,0 +1,45 @@
+package etcd
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+)
+
+func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+
+	_, err = store.client.Put(ctx, store.etcdKeyPrefix+string(key), string(value))
+
+	if err != nil {
+		return fmt.Errorf("kv put: %w", err)
+	}
+
+	return nil
+}
+
+func (store *EtcdStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+
+	resp, err := store.client.Get(ctx, store.etcdKeyPrefix+string(key))
+
+	if err != nil {
+		return nil, fmt.Errorf("kv get: %w", err)
+	}
+
+	if len(resp.Kvs) == 0 {
+		return nil, filer.ErrKvNotFound
+	}
+
+	return resp.Kvs[0].Value, nil
+}
+
+func (store *EtcdStore) KvDelete(ctx context.Context, key []byte) (err error) {
+
+	_, err = store.client.Delete(ctx, store.etcdKeyPrefix+string(key))
+
+	if err != nil {
+		return fmt.Errorf("kv delete: %w", err)
+	}
+
+	return nil
+}

weed/filer/etcd/etcd_store_test.go

@@ -0,0 +1,17 @@
+package etcd
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer/store_test"
+)
+
+func TestStore(t *testing.T) {
+	// run "make test_etcd" under docker folder.
+	// to set up local env
+	if false {
+		store := &EtcdStore{}
+		store.initialize("localhost:2379", "", "", 3, nil)
+		store_test.TestFilerStore(t, store)
+	}
+}

weed/filer/filechunk_group.go

@@ -0,0 +1,331 @@
+package filer
+
+import (
+	"context"
+	"io"
+	"sync"
+
+	"golang.org/x/sync/errgroup"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+)
+
+type ChunkGroup struct {
+	lookupFn          wdclient.LookupFileIdFunctionType
+	sections          map[SectionIndex]*FileChunkSection
+	sectionsLock      sync.RWMutex
+	readerCache       *ReaderCache
+	concurrentReaders int
+	// cacheInvalidator lets manifest resolution drop stale volume locations, as ReaderCache does for chunk reads
+	cacheInvalidator CacheInvalidator
+	// manifestCache caches resolved chunk manifest bytes across repeated opens
+	// for the same mount. nil for non-mount callers (no caching).
+	manifestCache *ChunkManifestCache
+	// resolveErr records a manifest resolution failure, guarded by sectionsLock
+	resolveErr error
+}
+
+// NewChunkGroup creates a ChunkGroup with configurable concurrency.
+// concurrentReaders controls:
+// - Maximum parallel chunk fetches during read operations
+// - Read-ahead prefetch parallelism
+// - Number of concurrent section reads for large files
+// If concurrentReaders <= 0, defaults to 16.
+func NewChunkGroup(lookupFn wdclient.LookupFileIdFunctionType, chunkCache chunk_cache.ChunkCache, chunks []*filer_pb.FileChunk, concurrentReaders int, cacheInvalidator CacheInvalidator, manifestCache *ChunkManifestCache, budgets ...*ReaderCacheBudget) (*ChunkGroup, error) {
+	if concurrentReaders <= 0 {
+		concurrentReaders = 16
+	}
+	if concurrentReaders > 128 {
+		concurrentReaders = 128 // Cap to prevent excessive goroutine fan-out
+	}
+	// ReaderCache limit should be at least concurrentReaders to allow parallel prefetching
+	readerCacheLimit := concurrentReaders * 2
+	if readerCacheLimit < 32 {
+		readerCacheLimit = 32
+	}
+	group := &ChunkGroup{
+		lookupFn:          lookupFn,
+		sections:          make(map[SectionIndex]*FileChunkSection),
+		readerCache:       NewReaderCache(readerCacheLimit, chunkCache, lookupFn, cacheInvalidator, budgets...),
+		concurrentReaders: concurrentReaders,
+		cacheInvalidator:  cacheInvalidator,
+		manifestCache:     manifestCache,
+	}
+
+	err := group.SetChunks(chunks)
+	return group, err
+}
+
+// GetPrefetchCount returns the number of chunks to prefetch ahead during sequential reads.
+// This is derived from concurrentReaders to keep the network pipeline full.
+func (group *ChunkGroup) GetPrefetchCount() int {
+	// Prefetch at least 1, and scale with concurrency (roughly 1/4 of concurrent readers)
+	prefetch := group.concurrentReaders / 4
+	if prefetch < 1 {
+		prefetch = 1
+	}
+	if prefetch > 8 {
+		prefetch = 8 // Cap at 8 to avoid excessive memory usage
+	}
+	return prefetch
+}
+
+func (group *ChunkGroup) AddChunk(chunk *filer_pb.FileChunk) error {
+
+	group.sectionsLock.Lock()
+	defer group.sectionsLock.Unlock()
+
+	sectionIndexStart, sectionIndexStop := SectionIndex(chunk.Offset/SectionSize), SectionIndex((chunk.Offset+int64(chunk.Size))/SectionSize)
+	for si := sectionIndexStart; si < sectionIndexStop+1; si++ {
+		section, found := group.sections[si]
+		if !found {
+			section = NewFileChunkSection(si)
+			group.sections[si] = section
+		}
+		section.addChunk(chunk)
+	}
+	return nil
+}
+
+func (group *ChunkGroup) ReadDataAt(ctx context.Context, fileSize int64, buff []byte, offset int64) (n int, tsNs int64, err error) {
+	if offset >= fileSize {
+		return 0, 0, io.EOF
+	}
+
+	group.sectionsLock.RLock()
+	defer group.sectionsLock.RUnlock()
+
+	if group.resolveErr != nil {
+		return 0, 0, group.resolveErr
+	}
+
+	sectionIndexStart, sectionIndexStop := SectionIndex(offset/SectionSize), SectionIndex((offset+int64(len(buff)))/SectionSize)
+	numSections := int(sectionIndexStop - sectionIndexStart + 1)
+
+	// For single section or when concurrency is disabled, use sequential reading
+	if numSections <= 1 || group.concurrentReaders <= 1 {
+		return group.readDataAtSequential(ctx, fileSize, buff, offset, sectionIndexStart, sectionIndexStop)
+	}
+
+	// For multiple sections, use parallel reading
+	return group.readDataAtParallel(ctx, fileSize, buff, offset, sectionIndexStart, sectionIndexStop)
+}
+
+// readDataAtSequential reads sections sequentially (original behavior)
+func (group *ChunkGroup) readDataAtSequential(ctx context.Context, fileSize int64, buff []byte, offset int64, sectionIndexStart, sectionIndexStop SectionIndex) (n int, tsNs int64, err error) {
+	for si := sectionIndexStart; si < sectionIndexStop+1; si++ {
+		section, found := group.sections[si]
+		rangeStart, rangeStop := max(offset, int64(si*SectionSize)), min(offset+int64(len(buff)), int64((si+1)*SectionSize))
+		if rangeStart >= rangeStop {
+			continue
+		}
+		if !found {
+			rangeStop = min(rangeStop, fileSize)
+			for i := rangeStart; i < rangeStop; i++ {
+				buff[i-offset] = 0
+			}
+			n = int(int64(n) + rangeStop - rangeStart)
+			continue
+		}
+		xn, xTsNs, xErr := section.readDataAt(ctx, group, fileSize, buff[rangeStart-offset:rangeStop-offset], rangeStart)
+		if xErr != nil {
+			return n + xn, max(tsNs, xTsNs), xErr
+		}
+		n += xn
+		tsNs = max(tsNs, xTsNs)
+	}
+	return
+}
+
+// sectionReadResult holds the result of a section read operation
+type sectionReadResult struct {
+	sectionIndex SectionIndex
+	n            int
+	tsNs         int64
+	err          error
+}
+
+// readDataAtParallel reads multiple sections in parallel for better throughput
+func (group *ChunkGroup) readDataAtParallel(ctx context.Context, fileSize int64, buff []byte, offset int64, sectionIndexStart, sectionIndexStop SectionIndex) (n int, tsNs int64, err error) {
+	numSections := int(sectionIndexStop - sectionIndexStart + 1)
+
+	// Limit concurrency to the smaller of concurrentReaders and numSections
+	maxConcurrent := group.concurrentReaders
+	if numSections < maxConcurrent {
+		maxConcurrent = numSections
+	}
+
+	g, gCtx := errgroup.WithContext(ctx)
+	g.SetLimit(maxConcurrent)
+
+	results := make([]sectionReadResult, numSections)
+
+	for i := 0; i < numSections; i++ {
+		si := sectionIndexStart + SectionIndex(i)
+		idx := i
+
+		section, found := group.sections[si]
+		rangeStart, rangeStop := max(offset, int64(si*SectionSize)), min(offset+int64(len(buff)), int64((si+1)*SectionSize))
+		if rangeStart >= rangeStop {
+			continue
+		}
+
+		if !found {
+			// Zero-fill missing sections synchronously
+			rangeStop = min(rangeStop, fileSize)
+			for j := rangeStart; j < rangeStop; j++ {
+				buff[j-offset] = 0
+			}
+			results[idx] = sectionReadResult{
+				sectionIndex: si,
+				n:            int(rangeStop - rangeStart),
+				tsNs:         0,
+				err:          nil,
+			}
+			continue
+		}
+
+		// Capture variables for closure
+		sectionCopy := section
+		buffSlice := buff[rangeStart-offset : rangeStop-offset]
+		rangeStartCopy := rangeStart
+
+		g.Go(func() error {
+			xn, xTsNs, xErr := sectionCopy.readDataAt(gCtx, group, fileSize, buffSlice, rangeStartCopy)
+			results[idx] = sectionReadResult{
+				sectionIndex: si,
+				n:            xn,
+				tsNs:         xTsNs,
+				err:          xErr,
+			}
+			if xErr != nil && xErr != io.EOF {
+				return xErr
+			}
+			return nil
+		})
+	}
+
+	// Wait for all goroutines to complete
+	groupErr := g.Wait()
+
+	// Aggregate results
+	for _, result := range results {
+		n += result.n
+		tsNs = max(tsNs, result.tsNs)
+		// Collect first non-EOF error from results as fallback
+		if result.err != nil && result.err != io.EOF && err == nil {
+			err = result.err
+		}
+	}
+
+	// Prioritize errgroup error (first error that cancelled context)
+	if groupErr != nil {
+		err = groupErr
+	}
+
+	return
+}
+
+func (group *ChunkGroup) SetChunks(chunks []*filer_pb.FileChunk) error {
+	group.sectionsLock.Lock()
+	defer group.sectionsLock.Unlock()
+
+	var dataChunks []*filer_pb.FileChunk
+	for _, chunk := range chunks {
+
+		if !chunk.IsChunkManifest {
+			dataChunks = append(dataChunks, chunk)
+			continue
+		}
+
+		resolvedChunks, err := resolveOneChunkManifest(context.Background(), group.lookupFn, chunk, group.cacheInvalidator, group.manifestCache)
+		if err != nil {
+			group.resolveErr = err
+			return err
+		}
+
+		dataChunks = append(dataChunks, resolvedChunks...)
+	}
+
+	sections := make(map[SectionIndex]*FileChunkSection)
+
+	for _, chunk := range dataChunks {
+		sectionIndexStart, sectionIndexStop := SectionIndex(chunk.Offset/SectionSize), SectionIndex((chunk.Offset+int64(chunk.Size))/SectionSize)
+		for si := sectionIndexStart; si < sectionIndexStop+1; si++ {
+			section, found := sections[si]
+			if !found {
+				section = NewFileChunkSection(si)
+				sections[si] = section
+			}
+			section.chunks = append(section.chunks, chunk)
+		}
+	}
+
+	group.sections = sections
+	group.resolveErr = nil
+	return nil
+}
+
+const (
+	// see weedfs_file_lseek.go
+	SEEK_DATA uint32 = 3 // seek to next data after the offset
+	// SEEK_HOLE uint32 = 4 // seek to next hole after the offset
+)
+
+func (group *ChunkGroup) SearchChunks(ctx context.Context, offset, fileSize int64, whence uint32) (found bool, out int64, err error) {
+	group.sectionsLock.RLock()
+	defer group.sectionsLock.RUnlock()
+
+	// the section map is unreliable after a failed manifest resolution
+	if group.resolveErr != nil {
+		return false, 0, group.resolveErr
+	}
+
+	found, out = group.doSearchChunks(ctx, offset, fileSize, whence)
+	return found, out, nil
+}
+
+func (group *ChunkGroup) doSearchChunks(ctx context.Context, offset, fileSize int64, whence uint32) (found bool, out int64) {
+
+	sectionIndex, maxSectionIndex := SectionIndex(offset/SectionSize), SectionIndex(fileSize/SectionSize)
+	for si := sectionIndex; si <= maxSectionIndex; si++ {
+		sectionStart, sectionStop := sectionBounds(si, fileSize)
+		sectionStart = max(offset, sectionStart)
+		if sectionStart >= sectionStop {
+			continue
+		}
+
+		section, foundSection := group.sections[si]
+		if whence == SEEK_DATA {
+			if !foundSection {
+				continue
+			}
+			dataStart := section.DataStartOffset(ctx, group, sectionStart, fileSize)
+			if dataStart >= sectionStart && dataStart < sectionStop {
+				return true, dataStart
+			}
+			continue
+		}
+
+		// whence == SEEK_HOLE
+		if !foundSection {
+			return true, sectionStart
+		}
+		holeStart := section.NextStopOffset(ctx, group, sectionStart, fileSize)
+		if holeStart < sectionStop {
+			return true, holeStart
+		}
+	}
+
+	if whence == SEEK_DATA {
+		return false, 0
+	}
+	return true, fileSize
+}
+
+func (group *ChunkGroup) Close() error {
+	group.readerCache.destroy()
+	return nil
+}

weed/filer/filechunk_group_test.go

@@ -0,0 +1,467 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"io"
+	"math"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestChunkGroup_ReadDataAt_ErrorHandling(t *testing.T) {
+	// Test that ReadDataAt behaves correctly in various scenarios
+	// This indirectly verifies that our error handling fix works properly
+
+	// Create a ChunkGroup with no sections
+	group := &ChunkGroup{
+		sections: make(map[SectionIndex]*FileChunkSection),
+	}
+
+	t.Run("should return immediately on error", func(t *testing.T) {
+		// This test verifies that our fix is working by checking the behavior
+		// We'll create a simple scenario where the fix would make a difference
+
+		buff := make([]byte, 100)
+		fileSize := int64(1000)
+		offset := int64(0)
+
+		// With an empty ChunkGroup, we should get no error
+		n, tsNs, err := group.ReadDataAt(context.Background(), fileSize, buff, offset)
+
+		// Should return 100 (length of buffer) and no error since there are no sections
+		// and missing sections are filled with zeros
+		assert.Equal(t, 100, n)
+		assert.Equal(t, int64(0), tsNs)
+		assert.NoError(t, err)
+
+		// Verify buffer is filled with zeros
+		for i, b := range buff {
+			assert.Equal(t, byte(0), b, "buffer[%d] should be zero", i)
+		}
+	})
+
+	t.Run("should handle EOF correctly", func(t *testing.T) {
+		buff := make([]byte, 100)
+		fileSize := int64(50) // File smaller than buffer
+		offset := int64(0)
+
+		n, tsNs, err := group.ReadDataAt(context.Background(), fileSize, buff, offset)
+
+		// Should return 50 (file size) and no error
+		assert.Equal(t, 50, n)
+		assert.Equal(t, int64(0), tsNs)
+		assert.NoError(t, err)
+	})
+
+	t.Run("should return EOF when offset exceeds file size", func(t *testing.T) {
+		buff := make([]byte, 100)
+		fileSize := int64(50)
+		offset := int64(100) // Offset beyond file size
+
+		n, tsNs, err := group.ReadDataAt(context.Background(), fileSize, buff, offset)
+
+		assert.Equal(t, 0, n)
+		assert.Equal(t, int64(0), tsNs)
+		assert.Equal(t, io.EOF, err)
+	})
+
+	t.Run("should demonstrate the GitHub issue fix - errors should not be masked", func(t *testing.T) {
+		// This test demonstrates the exact scenario described in GitHub issue #6991
+		// where io.EOF could mask real errors if we continued processing sections
+
+		// The issue:
+		// - Before the fix: if section 1 returns a real error, but section 2 returns io.EOF,
+		//   the real error would be overwritten by io.EOF
+		// - After the fix: return immediately on any error, preserving the original error
+
+		// Our fix ensures that we return immediately on ANY error (including io.EOF)
+		// This test verifies that the fix pattern works correctly for the most critical cases
+
+		buff := make([]byte, 100)
+		fileSize := int64(1000)
+
+		// Test 1: Normal operation with no sections (filled with zeros)
+		n, tsNs, err := group.ReadDataAt(context.Background(), fileSize, buff, int64(0))
+		assert.Equal(t, 100, n, "should read full buffer")
+		assert.Equal(t, int64(0), tsNs, "timestamp should be zero for missing sections")
+		assert.NoError(t, err, "should not error for missing sections")
+
+		// Test 2: Reading beyond file size should return io.EOF immediately
+		n, tsNs, err = group.ReadDataAt(context.Background(), fileSize, buff, fileSize+1)
+		assert.Equal(t, 0, n, "should not read any bytes when beyond file size")
+		assert.Equal(t, int64(0), tsNs, "timestamp should be zero")
+		assert.Equal(t, io.EOF, err, "should return io.EOF when reading beyond file size")
+
+		// Test 3: Reading at exact file boundary
+		n, tsNs, err = group.ReadDataAt(context.Background(), fileSize, buff, fileSize)
+		assert.Equal(t, 0, n, "should not read any bytes at exact file size boundary")
+		assert.Equal(t, int64(0), tsNs, "timestamp should be zero")
+		assert.Equal(t, io.EOF, err, "should return io.EOF at file boundary")
+
+		// The key insight: Our fix ensures that ANY error from section.readDataAt()
+		// causes immediate return with proper context (bytes read + timestamp + error)
+		// This prevents later sections from masking earlier errors, especially
+		// preventing io.EOF from masking network errors or other real failures.
+	})
+
+	t.Run("Context Cancellation", func(t *testing.T) {
+		// Test 4: Context cancellation should be properly propagated through ReadDataAt
+
+		// This test verifies that the context parameter is properly threaded through
+		// the call chain and that cancellation checks are in place at the right points
+
+		// Test with a pre-cancelled context to ensure the cancellation is detected
+		ctx, cancel := context.WithCancel(context.Background())
+		cancel() // Cancel immediately
+
+		group := &ChunkGroup{
+			sections: make(map[SectionIndex]*FileChunkSection),
+		}
+
+		buff := make([]byte, 100)
+		fileSize := int64(1000)
+
+		// Call ReadDataAt with the already cancelled context
+		n, tsNs, err := group.ReadDataAt(ctx, fileSize, buff, int64(0))
+
+		// For an empty ChunkGroup (no sections), the operation will complete successfully
+		// since it just fills the buffer with zeros. However, the important thing is that
+		// the context is properly threaded through the call chain.
+		// The actual cancellation would be more evident with real chunk sections that
+		// perform network operations.
+
+		if err != nil {
+			// If an error is returned, it should be a context cancellation error
+			assert.True(t,
+				errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded),
+				"Expected context.Canceled or context.DeadlineExceeded, got: %v", err)
+		} else {
+			// If no error (operation completed before cancellation check),
+			// verify normal behavior for empty ChunkGroup
+			assert.Equal(t, 100, n, "should read full buffer size when no sections exist")
+			assert.Equal(t, int64(0), tsNs, "timestamp should be zero")
+			t.Log("Operation completed before context cancellation was checked - this is expected for empty ChunkGroup")
+		}
+	})
+
+	t.Run("Context Cancellation with Timeout", func(t *testing.T) {
+		// Test 5: Context with timeout should be respected
+
+		group := &ChunkGroup{
+			sections: make(map[SectionIndex]*FileChunkSection),
+		}
+
+		// Create a context with a very short timeout
+		ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond)
+		defer cancel()
+
+		buff := make([]byte, 100)
+		fileSize := int64(1000)
+
+		// This should fail due to timeout
+		n, tsNs, err := group.ReadDataAt(ctx, fileSize, buff, int64(0))
+
+		// For this simple case with no sections, it might complete before timeout
+		// But if it does timeout, we should handle it properly
+		if err != nil {
+			assert.True(t,
+				errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded),
+				"Expected context.Canceled or context.DeadlineExceeded when context times out, got: %v", err)
+		} else {
+			// If no error, verify normal behavior
+			assert.Equal(t, 100, n, "should read full buffer size when no sections exist")
+			assert.Equal(t, int64(0), tsNs, "timestamp should be zero")
+		}
+	})
+}
+
+func TestChunkGroup_SearchChunks_Cancellation(t *testing.T) {
+	t.Run("Context Cancellation in SearchChunks", func(t *testing.T) {
+		// Test that SearchChunks properly handles context cancellation
+
+		group := &ChunkGroup{
+			sections: make(map[SectionIndex]*FileChunkSection),
+		}
+
+		// Test with a pre-cancelled context
+		ctx, cancel := context.WithCancel(context.Background())
+		cancel() // Cancel immediately
+
+		fileSize := int64(1000)
+		offset := int64(0)
+		whence := uint32(3) // SEEK_DATA
+
+		// Call SearchChunks with cancelled context
+		found, resultOffset, _ := group.SearchChunks(ctx, offset, fileSize, whence)
+
+		// For an empty ChunkGroup, SearchChunks should complete quickly
+		// The main goal is to verify the context parameter is properly threaded through
+		// In real scenarios with actual chunk sections, context cancellation would be more meaningful
+
+		// Verify the function completes and returns reasonable values
+		assert.False(t, found, "should not find data in empty chunk group")
+		assert.Equal(t, int64(0), resultOffset, "should return 0 offset when no data found")
+
+		t.Log("SearchChunks completed with cancelled context - context threading verified")
+	})
+
+	t.Run("Context with Timeout in SearchChunks", func(t *testing.T) {
+		// Test SearchChunks with a timeout context
+
+		group := &ChunkGroup{
+			sections: make(map[SectionIndex]*FileChunkSection),
+		}
+
+		// Create a context with very short timeout
+		ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond)
+		defer cancel()
+
+		fileSize := int64(1000)
+		offset := int64(0)
+		whence := uint32(3) // SEEK_DATA
+
+		// Call SearchChunks - should complete quickly for empty group
+		found, resultOffset, _ := group.SearchChunks(ctx, offset, fileSize, whence)
+
+		// Verify reasonable behavior
+		assert.False(t, found, "should not find data in empty chunk group")
+		assert.Equal(t, int64(0), resultOffset, "should return 0 offset when no data found")
+	})
+}
+
+func TestChunkGroup_SearchChunks(t *testing.T) {
+	const seekHole uint32 = 4
+
+	type args struct {
+		offset   int64
+		fileSize int64
+		whence   uint32
+	}
+	tests := []struct {
+		name       string
+		chunks     []*filer_pb.FileChunk
+		args       args
+		wantFound  bool
+		wantOffset int64
+	}{
+		{
+			name: "SEEK_DATA starts at the first data range after a hole",
+			chunks: []*filer_pb.FileChunk{
+				{FileId: "data-1", Offset: 100, Size: 100},
+				{FileId: "data-2", Offset: 300, Size: 100},
+			},
+			args:       args{offset: 0, fileSize: 500, whence: SEEK_DATA},
+			wantFound:  true,
+			wantOffset: 100,
+		},
+		{
+			name: "SEEK_DATA preserves an offset inside a data range",
+			chunks: []*filer_pb.FileChunk{
+				{FileId: "data-1", Offset: 100, Size: 100},
+				{FileId: "data-2", Offset: 300, Size: 100},
+			},
+			args:       args{offset: 150, fileSize: 500, whence: SEEK_DATA},
+			wantFound:  true,
+			wantOffset: 150,
+		},
+		{
+			name: "SEEK_DATA crosses a hole between data ranges",
+			chunks: []*filer_pb.FileChunk{
+				{FileId: "data-1", Offset: 100, Size: 100},
+				{FileId: "data-2", Offset: 300, Size: 100},
+			},
+			args:       args{offset: 200, fileSize: 500, whence: SEEK_DATA},
+			wantFound:  true,
+			wantOffset: 300,
+		},
+		{
+			name: "SEEK_DATA returns no match after the final data range",
+			chunks: []*filer_pb.FileChunk{
+				{FileId: "data-1", Offset: 100, Size: 100},
+				{FileId: "data-2", Offset: 300, Size: 100},
+			},
+			args:       args{offset: 400, fileSize: 500, whence: SEEK_DATA},
+			wantFound:  false,
+			wantOffset: 0,
+		},
+		{
+			name: "SEEK_HOLE starts at the sparse prefix",
+			chunks: []*filer_pb.FileChunk{
+				{FileId: "data-1", Offset: 100, Size: 100},
+				{FileId: "data-2", Offset: 300, Size: 100},
+			},
+			args:       args{offset: 0, fileSize: 500, whence: seekHole},
+			wantFound:  true,
+			wantOffset: 0,
+		},
+		{
+			name: "SEEK_HOLE finds the transition after data",
+			chunks: []*filer_pb.FileChunk{
+				{FileId: "data-1", Offset: 100, Size: 100},
+				{FileId: "data-2", Offset: 300, Size: 100},
+			},
+			args:       args{offset: 150, fileSize: 500, whence: seekHole},
+			wantFound:  true,
+			wantOffset: 200,
+		},
+		{
+			name: "SEEK_HOLE preserves an offset inside a hole",
+			chunks: []*filer_pb.FileChunk{
+				{FileId: "data-1", Offset: 100, Size: 100},
+				{FileId: "data-2", Offset: 300, Size: 100},
+			},
+			args:       args{offset: 250, fileSize: 500, whence: seekHole},
+			wantFound:  true,
+			wantOffset: 250,
+		},
+		{
+			name: "SEEK_HOLE returns the implicit trailing hole",
+			chunks: []*filer_pb.FileChunk{
+				{FileId: "data-1", Offset: 100, Size: 100},
+				{FileId: "data-2", Offset: 300, Size: 100},
+			},
+			args:       args{offset: 400, fileSize: 500, whence: seekHole},
+			wantFound:  true,
+			wantOffset: 400,
+		},
+		{
+			name:       "SEEK_DATA at EOF has no match",
+			chunks:     []*filer_pb.FileChunk{{FileId: "data", Offset: 0, Size: 500}},
+			args:       args{offset: 500, fileSize: 500, whence: SEEK_DATA},
+			wantFound:  false,
+			wantOffset: 0,
+		},
+		{
+			name:       "SEEK_HOLE at EOF returns EOF",
+			chunks:     []*filer_pb.FileChunk{{FileId: "data", Offset: 0, Size: 500}},
+			args:       args{offset: 500, fileSize: 500, whence: seekHole},
+			wantFound:  true,
+			wantOffset: 500,
+		},
+		{
+			name:       "empty file has neither data nor a non-EOF hole",
+			chunks:     nil,
+			args:       args{offset: 0, fileSize: 0, whence: SEEK_DATA},
+			wantFound:  false,
+			wantOffset: 0,
+		},
+		{
+			name:       "empty file reports EOF for SEEK_HOLE",
+			chunks:     nil,
+			args:       args{offset: 0, fileSize: 0, whence: seekHole},
+			wantFound:  true,
+			wantOffset: 0,
+		},
+		{
+			name: "SEEK_DATA crosses a section boundary",
+			chunks: []*filer_pb.FileChunk{
+				{FileId: "data-0", Offset: 0, Size: 16},
+				{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
+				{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
+			},
+			args:       args{offset: 16, fileSize: 2*SectionSize + 32, whence: SEEK_DATA},
+			wantFound:  true,
+			wantOffset: SectionSize - 16,
+		},
+		{
+			name: "SEEK_DATA finds data after a section boundary hole",
+			chunks: []*filer_pb.FileChunk{
+				{FileId: "data-0", Offset: 0, Size: 16},
+				{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
+				{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
+			},
+			args:       args{offset: SectionSize, fileSize: 2*SectionSize + 32, whence: SEEK_DATA},
+			wantFound:  true,
+			wantOffset: SectionSize + 16,
+		},
+		{
+			name: "SEEK_HOLE finds a hole at a section boundary",
+			chunks: []*filer_pb.FileChunk{
+				{FileId: "data-0", Offset: 0, Size: 16},
+				{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
+				{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
+			},
+			args:       args{offset: SectionSize - 16, fileSize: 2*SectionSize + 32, whence: seekHole},
+			wantFound:  true,
+			wantOffset: SectionSize,
+		},
+		{
+			name: "SEEK_HOLE finds a missing section",
+			chunks: []*filer_pb.FileChunk{
+				{FileId: "data-0", Offset: 0, Size: 16},
+				{FileId: "data-0-tail", Offset: SectionSize - 16, Size: 16},
+				{FileId: "data-1", Offset: SectionSize + 16, Size: 16},
+			},
+			args:       args{offset: 2 * SectionSize, fileSize: 2*SectionSize + 32, whence: seekHole},
+			wantFound:  true,
+			wantOffset: 2 * SectionSize,
+		},
+		{
+			name: "SEEK_DATA finds data in the final section at MaxInt64 file size",
+			chunks: []*filer_pb.FileChunk{
+				{FileId: "final-data", Offset: math.MaxInt64 - 1, Size: 1},
+			},
+			args:       args{offset: math.MaxInt64 - 1, fileSize: math.MaxInt64, whence: SEEK_DATA},
+			wantFound:  true,
+			wantOffset: math.MaxInt64 - 1,
+		},
+		{
+			name: "SEEK_HOLE finds the final section hole at MaxInt64 file size",
+			chunks: []*filer_pb.FileChunk{
+				{FileId: "final-data", Offset: math.MaxInt64 - 2, Size: 1},
+			},
+			args:       args{offset: math.MaxInt64 - 2, fileSize: math.MaxInt64, whence: seekHole},
+			wantFound:  true,
+			wantOffset: math.MaxInt64 - 1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			group, err := NewChunkGroup(nil, nil, tt.chunks, 1, nil, nil)
+			if !assert.NoError(t, err) {
+				return
+			}
+
+			gotFound, gotOffset, err := group.SearchChunks(context.Background(), tt.args.offset, tt.args.fileSize, tt.args.whence)
+			assert.NoError(t, err)
+			assert.Equalf(t, tt.wantFound, gotFound, "SearchChunks(%v, %v, %v) found", tt.args.offset, tt.args.fileSize, tt.args.whence)
+			assert.Equalf(t, tt.wantOffset, gotOffset, "SearchChunks(%v, %v, %v) offset", tt.args.offset, tt.args.fileSize, tt.args.whence)
+		})
+	}
+}
+
+// ReadDataAt must return an error when chunk manifest resolution fails, instead of zero-filling.
+func TestChunkGroup_ReadDataAt_ManifestResolveFailure(t *testing.T) {
+	lookupErr := errors.New("lookup failed")
+	lookupFn := func(ctx context.Context, fileId string) ([]string, error) {
+		return nil, lookupErr
+	}
+
+	chunks := []*filer_pb.FileChunk{
+		{FileId: "1,1679011dc64abd40", IsChunkManifest: true, Offset: 0, Size: 1 << 20},
+	}
+
+	group, err := NewChunkGroup(lookupFn, nil, chunks, 1, nil, nil)
+	assert.Error(t, err, "manifest resolution should fail")
+
+	buff := make([]byte, 16)
+	n, _, readErr := group.ReadDataAt(context.Background(), 1<<20, buff, 0)
+	assert.ErrorIs(t, readErr, lookupErr)
+	assert.Equal(t, 0, n)
+
+	for _, whence := range []uint32{SEEK_DATA, 4 /* SEEK_HOLE */} {
+		found, _, seekErr := group.SearchChunks(context.Background(), 0, 1<<20, whence)
+		assert.ErrorIs(t, seekErr, lookupErr, "whence %d", whence)
+		assert.False(t, found, "whence %d", whence)
+	}
+
+	err = group.SetChunks([]*filer_pb.FileChunk{
+		{FileId: "2,data", Offset: 0, Size: 16},
+	})
+	assert.NoError(t, err)
+	assert.NoError(t, group.resolveErr)
+}

weed/filer/filechunk_manifest.go

@@ -0,0 +1,578 @@
+package filer
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+var bytesBufferPool = sync.Pool{
+	New: func() interface{} {
+		return new(bytes.Buffer)
+	},
+}
+
+// Keep Manifest reads bounded across all recursion levels of one resolution.
+const maxChunkManifestResolveWorkers = 4
+
+// Size of the job queue buffer. Large enough that submission does not block
+// under normal chunk counts, so a promptly-failing manifest is always queued
+// and can cancel stalled sibling reads once a worker picks it up.
+const chunkManifestResolveJobBufferSize = 128
+
+func HasChunkManifest(chunks []*filer_pb.FileChunk) bool {
+	for _, chunk := range chunks {
+		if chunk.IsChunkManifest {
+			return true
+		}
+	}
+	return false
+}
+
+func SeparateManifestChunks(chunks []*filer_pb.FileChunk) (manifestChunks, nonManifestChunks []*filer_pb.FileChunk) {
+	for _, c := range chunks {
+		if c.IsChunkManifest {
+			manifestChunks = append(manifestChunks, c)
+		} else {
+			nonManifestChunks = append(nonManifestChunks, c)
+		}
+	}
+	return
+}
+
+func ResolveChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset, stopOffset int64, invalidator CacheInvalidator) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) {
+	resolver := newChunkManifestResolver(ctx, lookupFileIdFn, invalidator)
+	defer resolver.close()
+	return resolver.resolve(chunks, startOffset, stopOffset)
+}
+
+type chunkManifestResolveJob struct {
+	chunk       *filer_pb.FileChunk
+	result      *chunkManifestResolveResult
+	done        *sync.WaitGroup
+	batchCtx    context.Context
+	batchCancel context.CancelFunc
+	batchOnce   *sync.Once
+}
+
+type chunkManifestResolveResult struct {
+	chunks         []*filer_pb.FileChunk
+	err            error
+	internalCancel bool
+}
+
+type chunkManifestResolver struct {
+	ctx            context.Context
+	parentCtx      context.Context
+	cancel         context.CancelFunc
+	lookupFileIdFn wdclient.LookupFileIdFunctionType
+	invalidator    CacheInvalidator
+	jobs           chan chunkManifestResolveJob
+	overflowSem    chan struct{}
+	workers        sync.WaitGroup
+	startOnce      sync.Once
+	started        bool
+}
+
+func newChunkManifestResolver(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, invalidator CacheInvalidator) *chunkManifestResolver {
+	workCtx, cancel := context.WithCancel(ctx)
+	resolver := &chunkManifestResolver{
+		ctx:            workCtx,
+		parentCtx:      ctx,
+		cancel:         cancel,
+		lookupFileIdFn: lookupFileIdFn,
+		invalidator:    invalidator,
+		jobs:           make(chan chunkManifestResolveJob, chunkManifestResolveJobBufferSize),
+		overflowSem:    make(chan struct{}, maxChunkManifestResolveWorkers),
+	}
+	return resolver
+}
+
+func (r *chunkManifestResolver) executeJob(job chunkManifestResolveJob) {
+	job.result.chunks, job.result.err = ResolveOneChunkManifest(job.batchCtx, r.lookupFileIdFn, job.chunk, r.invalidator)
+	if job.result.err != nil && r.parentCtx.Err() == nil {
+		if job.batchCtx.Err() != nil && errors.Is(job.result.err, context.Canceled) {
+			job.result.internalCancel = true
+		} else if job.batchCtx.Err() == nil {
+			job.batchOnce.Do(job.batchCancel)
+		}
+	}
+	job.done.Done()
+}
+
+func (r *chunkManifestResolver) executeOverflowJob(job chunkManifestResolveJob) {
+	select {
+	case r.overflowSem <- struct{}{}:
+		defer func() { <-r.overflowSem }()
+		r.executeJob(job)
+	case <-job.batchCtx.Done():
+		if job.result.err == nil {
+			job.result.err = job.batchCtx.Err()
+			job.result.internalCancel = r.parentCtx.Err() == nil && job.result.err != nil
+		}
+		job.done.Done()
+	case <-r.ctx.Done():
+		if job.result.err == nil {
+			job.result.err = r.ctx.Err()
+			job.result.internalCancel = r.parentCtx.Err() == nil && job.result.err != nil
+		}
+		job.done.Done()
+	}
+}
+
+func (r *chunkManifestResolver) worker() {
+	defer r.workers.Done()
+	for job := range r.jobs {
+		r.executeJob(job)
+	}
+}
+
+func (r *chunkManifestResolver) close() {
+	r.cancel()
+	close(r.jobs)
+	if r.started {
+		r.workers.Wait()
+	}
+}
+
+func (r *chunkManifestResolver) submit(job chunkManifestResolveJob) bool {
+	r.startOnce.Do(func() {
+		r.workers.Add(maxChunkManifestResolveWorkers)
+		for i := 0; i < maxChunkManifestResolveWorkers; i++ {
+			go r.worker()
+		}
+		r.started = true
+	})
+	select {
+	case r.jobs <- job:
+		return true
+	case <-r.ctx.Done():
+		return false
+	case <-job.batchCtx.Done():
+		// Batch was cancelled by a sibling failure; don't queue this job.
+		// Set the result so the caller doesn't overwrite it; the caller
+		// owns the WaitGroup decrement.
+		job.result.err = job.batchCtx.Err()
+		job.result.internalCancel = r.parentCtx.Err() == nil && job.result.err != nil
+		return false
+	default:
+		// Buffer is full (exceedingly rare: more than chunkManifestResolveJobBufferSize
+		// in-range manifests at one level). Run the job in a bounded overflow goroutine
+		// so a promptly-failing manifest can still cancel the batch without waiting for
+		// a worker slot, while keeping total concurrency bounded.
+		go r.executeOverflowJob(job)
+		return true
+	}
+}
+
+func (r *chunkManifestResolver) resolve(chunks []*filer_pb.FileChunk, startOffset, stopOffset int64) (dataChunks, manifestChunks []*filer_pb.FileChunk, manifestResolveErr error) {
+	type resolveSlot struct {
+		chunk  *filer_pb.FileChunk
+		result chunkManifestResolveResult
+	}
+
+	// Cancellation is scoped to this parallel read batch so a failure in a
+	// later manifest does not cancel recursive work for an earlier manifest
+	// that already completed. Recursion creates its own batch context derived
+	// from the still-alive resolver context.
+	batchCtx, batchCancel := context.WithCancel(r.ctx)
+	defer batchCancel()
+	var batchOnce sync.Once
+
+	slots := make([]resolveSlot, len(chunks))
+	var reads sync.WaitGroup
+	for i, chunk := range chunks {
+		if max(chunk.Offset, startOffset) >= min(chunk.Offset+int64(chunk.Size), stopOffset) {
+			continue
+		}
+
+		slots[i].chunk = chunk
+		if !chunk.IsChunkManifest {
+			continue
+		}
+
+		reads.Add(1)
+		if !r.submit(chunkManifestResolveJob{
+			chunk:       chunk,
+			result:      &slots[i].result,
+			done:        &reads,
+			batchCtx:    batchCtx,
+			batchCancel: batchCancel,
+			batchOnce:   &batchOnce,
+		}) {
+			// submit may have already set the result (batch cancellation).
+			// Only set it here for the resolver-cancellation case.
+			if slots[i].result.err == nil {
+				slots[i].result.err = r.ctx.Err()
+				slots[i].result.internalCancel = r.parentCtx.Err() == nil && slots[i].result.err != nil
+			}
+			reads.Done()
+		}
+	}
+	reads.Wait()
+	if err := r.parentCtx.Err(); err != nil {
+		return dataChunks, nil, err
+	}
+
+	// Recurse only after this level's reads release their worker slots. A
+	// worker must never wait for a child manifest while holding a slot.
+	//
+	// Pre-scan for the first real (non-internal-cancel) error before
+	// recursing. If a later manifest already failed, return its error
+	// promptly with data chunks that are already in hand, instead of
+	// blocking on recursive reads of earlier manifests' children.
+	for i, slot := range slots {
+		if slot.chunk == nil {
+			continue
+		}
+		if slot.result.err != nil {
+			if slot.result.internalCancel {
+				continue
+			}
+			for j := 0; j < i; j++ {
+				if slots[j].chunk == nil {
+					continue
+				}
+				if !slots[j].chunk.IsChunkManifest {
+					dataChunks = append(dataChunks, slots[j].chunk)
+					continue
+				}
+				for _, c := range slots[j].result.chunks {
+					if !c.IsChunkManifest && max(c.Offset, startOffset) < min(c.Offset+int64(c.Size), stopOffset) {
+						dataChunks = append(dataChunks, c)
+					}
+				}
+			}
+			return dataChunks, nil, slot.result.err
+		}
+	}
+
+	for _, slot := range slots {
+		if slot.chunk == nil {
+			continue
+		}
+		if slot.result.err != nil {
+			if slot.result.internalCancel {
+				continue
+			}
+			return dataChunks, nil, slot.result.err
+		}
+		if !slot.chunk.IsChunkManifest {
+			dataChunks = append(dataChunks, slot.chunk)
+			continue
+		}
+
+		manifestChunks = append(manifestChunks, slot.chunk)
+		subDataChunks, subManifestChunks, subErr := r.resolve(slot.result.chunks, startOffset, stopOffset)
+		if subErr != nil {
+			return dataChunks, nil, subErr
+		}
+		dataChunks = append(dataChunks, subDataChunks...)
+		manifestChunks = append(manifestChunks, subManifestChunks...)
+	}
+	return
+}
+
+// ResolveOneChunkManifest fetches and decodes a single manifest chunk. It is
+// the uncached, exported path used by every non-Mount caller; the Mount path
+// routes through resolveOneChunkManifest so it can share a per-mount cache.
+// Keeping this signature stable preserves the existing four-argument contract
+// for external callers.
+func ResolveOneChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunk *filer_pb.FileChunk, invalidator CacheInvalidator) (dataChunks []*filer_pb.FileChunk, manifestResolveErr error) {
+	return resolveOneChunkManifest(ctx, lookupFileIdFn, chunk, invalidator, nil)
+}
+
+// resolveOneChunkManifest is the cache-aware implementation. cache may be nil,
+// in which case the manifest is fetched and validated on every call, matching
+// the historical uncached behavior. A non-nil cache is owned by a single mount
+// (WFS) and coalesces concurrent cold misses via singleflight.
+func resolveOneChunkManifest(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunk *filer_pb.FileChunk, invalidator CacheInvalidator, cache *ChunkManifestCache) (dataChunks []*filer_pb.FileChunk, manifestResolveErr error) {
+	if !chunk.IsChunkManifest {
+		return
+	}
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+
+	key := chunkManifestCacheKey{
+		fileID:       chunk.GetFileIdString(),
+		cipherKey:    string(chunk.CipherKey),
+		isCompressed: chunk.IsCompressed,
+	}
+
+	fetch := func() ([]byte, error) {
+		bytesBuffer := bytesBufferPool.Get().(*bytes.Buffer)
+		bytesBuffer.Reset()
+		defer bytesBufferPool.Put(bytesBuffer)
+		if err := fetchWholeChunk(ctx, bytesBuffer, lookupFileIdFn, key.fileID, chunk.CipherKey, chunk.IsCompressed, invalidator); err != nil {
+			return nil, fmt.Errorf("fail to read manifest %s: %w", key.fileID, err)
+		}
+		// Copy before the buffer returns to the pool so concurrent callers
+		// cannot overwrite the slice before it is cached.
+		data := append([]byte(nil), bytesBuffer.Bytes()...)
+		// Validate before returning so fetchOrLoad only caches well-formed
+		// manifests. A malformed manifest must not poison the cache.
+		if err := proto.Unmarshal(data, &filer_pb.FileChunkManifest{}); err != nil {
+			return nil, fmt.Errorf("fail to unmarshal manifest %s: %w", key.fileID, err)
+		}
+		return data, nil
+	}
+
+	var manifestBytes []byte
+	if cache != nil {
+		data, err := cache.fetchOrLoad(ctx, key, fetch)
+		if err != nil {
+			return nil, err
+		}
+		manifestBytes = data
+	} else {
+		data, err := fetch()
+		if err != nil {
+			return nil, err
+		}
+		manifestBytes = data
+	}
+
+	m := &filer_pb.FileChunkManifest{}
+	if err := proto.Unmarshal(manifestBytes, m); err != nil {
+		return nil, fmt.Errorf("fail to unmarshal manifest %s: %w", key.fileID, err)
+	}
+
+	// recursive
+	filer_pb.AfterEntryDeserialization(m.Chunks)
+	return m.Chunks, nil
+}
+
+func fetchWholeChunk(ctx context.Context, bytesBuffer *bytes.Buffer, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool, invalidator CacheInvalidator) error {
+	urlStrings, err := lookupFileIdFn(ctx, fileId)
+	if err != nil {
+		glog.ErrorfCtx(ctx, "operation LookupFileId %s failed, err: %v", fileId, err)
+		return err
+	}
+	jwt := ChunkReadJwt(urlStrings, fileId)
+	if _, err = retriedStreamFetchChunkData(ctx, bytesBuffer, urlStrings, jwt, cipherKey, isGzipped, true, 0, 0, refreshUrls(ctx, invalidator, lookupFileIdFn, fileId)); err == nil {
+		return nil
+	}
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		// a cancelled read says nothing about where the volume lives, and the
+		// stream error it provoked is a symptom, not the cause
+		return ctxErr
+	}
+	return retryFetchWithFreshLocations(ctx, invalidator, lookupFileIdFn, fileId, urlStrings, err, func(newUrls []string) error {
+		// the failed attempt may have streamed a partial prefix into the buffer
+		bytesBuffer.Reset()
+		_, retryErr := retriedStreamFetchChunkData(ctx, bytesBuffer, newUrls, jwt, cipherKey, isGzipped, true, 0, 0, nil)
+		return retryErr
+	})
+}
+
+func fetchChunkRange(ctx context.Context, buffer []byte, lookupFileIdFn wdclient.LookupFileIdFunctionType, fileId string, cipherKey []byte, isGzipped bool, offset int64, refreshUrls util_http.RefreshUrlsFunc) (int, error) {
+	urlStrings, err := lookupFileIdFn(ctx, fileId)
+	if err != nil {
+		glog.ErrorfCtx(ctx, "operation LookupFileId %s failed, err: %v", fileId, err)
+		return 0, err
+	}
+	return util_http.RetriedFetchChunkData(ctx, buffer, urlStrings, cipherKey, isGzipped, false, offset, fileId, refreshUrls)
+}
+
+// retriedStreamFetchChunkData streams a chunk from the first location that
+// answers. refreshUrls may be nil; when a location failed and a later one
+// answered, it is called so the reads that follow start from a fresh list.
+func retriedStreamFetchChunkData(ctx context.Context, writer io.Writer, urlStrings []string, jwt string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, size int, refreshUrls util_http.RefreshUrlsFunc) (written int64, err error) {
+
+	var shouldRetry bool
+	var totalWritten int
+
+	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
+		// Check for context cancellation before starting retry loop
+		select {
+		case <-ctx.Done():
+			return int64(totalWritten), ctx.Err()
+		default:
+		}
+
+		retriedCnt := 0
+		var failed bool
+		for _, urlString := range util_http.ReachableFirst(urlStrings) {
+			// Check for context cancellation before each volume server request
+			select {
+			case <-ctx.Done():
+				return int64(totalWritten), ctx.Err()
+			default:
+			}
+
+			retriedCnt++
+			var localProcessed int
+			var writeErr error
+			shouldRetry, err = util_http.ReadUrlAsStream(ctx, util_http.AppendQueryParameter(urlString, "readDeleted", "true"), jwt, cipherKey, isGzipped, isFullChunk, offset, size, func(data []byte) {
+				// Check for context cancellation during data processing
+				select {
+				case <-ctx.Done():
+					writeErr = ctx.Err()
+					return
+				default:
+				}
+
+				if totalWritten > localProcessed {
+					toBeSkipped := totalWritten - localProcessed
+					if len(data) <= toBeSkipped {
+						localProcessed += len(data)
+						return // skip if already processed
+					}
+					data = data[toBeSkipped:]
+					localProcessed += toBeSkipped
+				}
+				var writtenCount int
+				writtenCount, writeErr = writer.Write(data)
+				localProcessed += writtenCount
+				totalWritten += writtenCount
+			})
+			if !shouldRetry {
+				break
+			}
+			if writeErr != nil {
+				err = writeErr
+				break
+			}
+			if err != nil {
+				failed = true
+				glog.V(0).InfofCtx(ctx, "read %s failed, err: %v", urlString, err)
+			} else {
+				break
+			}
+		}
+		if err == nil && failed && refreshUrls != nil {
+			refreshUrls()
+		}
+		// all nodes have tried it
+		if retriedCnt == len(urlStrings) {
+			break
+		}
+		if err != nil && shouldRetry {
+			glog.V(0).InfofCtx(ctx, "retry reading in %v", waitTime)
+			// Sleep with proper context cancellation and timer cleanup
+			timer := time.NewTimer(waitTime)
+			select {
+			case <-ctx.Done():
+				timer.Stop()
+				return int64(totalWritten), ctx.Err()
+			case <-timer.C:
+				// Continue with retry
+			}
+		} else {
+			break
+		}
+	}
+
+	return int64(totalWritten), err
+
+}
+
+// MaybeManifestize folds a flat chunk list into manifest chunks once it passes
+// ManifestBatch, so an entry's chunk list stays within what the metadata store
+// will hold. A fold that fails partway returns inputChunks unchanged -- never
+// the half-folded list, which drops the manifests the caller came in with --
+// and hands the blobs it had already saved to deleteChunks, since the flat list
+// it returns references none of them. deleteChunks may be nil where the caller
+// has no deleter to offer; then the blobs are only named in the log.
+func MaybeManifestize(saveFunc SaveDataAsChunkFunctionType, deleteChunks func([]*filer_pb.FileChunk), inputChunks []*filer_pb.FileChunk) (chunks []*filer_pb.FileChunk, err error) {
+	var saved []*filer_pb.FileChunk
+	record := func(reader io.Reader, name string, offset int64, tsNs int64, expectedDataSize uint64) (*filer_pb.FileChunk, error) {
+		chunk, saveErr := saveFunc(reader, name, offset, tsNs, expectedDataSize)
+		if saveErr == nil {
+			saved = append(saved, chunk)
+		}
+		return chunk, saveErr
+	}
+
+	chunks, err = doMaybeManifestize(record, inputChunks, ManifestBatch, mergeIntoManifest)
+	if err == nil {
+		return chunks, nil
+	}
+	if len(saved) > 0 {
+		if deleteChunks != nil {
+			deleteChunks(saved)
+		} else {
+			glog.V(0).Infof("manifestize failed, %d manifest blobs left unreferenced: %v", len(saved), err)
+		}
+	}
+	return inputChunks, err
+}
+
+func doMaybeManifestize(saveFunc SaveDataAsChunkFunctionType, inputChunks []*filer_pb.FileChunk, mergeFactor int, mergefn func(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer_pb.FileChunk) (manifestChunk *filer_pb.FileChunk, err error)) (chunks []*filer_pb.FileChunk, err error) {
+
+	var dataChunks []*filer_pb.FileChunk
+	for _, chunk := range inputChunks {
+		if !chunk.IsChunkManifest {
+			dataChunks = append(dataChunks, chunk)
+		} else {
+			chunks = append(chunks, chunk)
+		}
+	}
+
+	remaining := len(dataChunks)
+	for i := 0; i+mergeFactor <= len(dataChunks); i += mergeFactor {
+		chunk, err := mergefn(saveFunc, dataChunks[i:i+mergeFactor])
+		if err != nil {
+			// dataChunks is what is left after the manifests the caller
+			// already had were separated out; returning it would drop them
+			return inputChunks, err
+		}
+		chunks = append(chunks, chunk)
+		remaining -= mergeFactor
+	}
+	// remaining
+	for i := len(dataChunks) - remaining; i < len(dataChunks); i++ {
+		chunks = append(chunks, dataChunks[i])
+	}
+	return
+}
+
+func mergeIntoManifest(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer_pb.FileChunk) (manifestChunk *filer_pb.FileChunk, err error) {
+
+	filer_pb.BeforeEntrySerialization(dataChunks)
+
+	// create and serialize the manifest
+	data, serErr := proto.Marshal(&filer_pb.FileChunkManifest{
+		Chunks: dataChunks,
+	})
+	if serErr != nil {
+		return nil, fmt.Errorf("serializing manifest: %w", serErr)
+	}
+
+	minOffset, maxOffset := int64(math.MaxInt64), int64(math.MinInt64)
+	for _, chunk := range dataChunks {
+		if minOffset > int64(chunk.Offset) {
+			minOffset = chunk.Offset
+		}
+		if maxOffset < int64(chunk.Size)+chunk.Offset {
+			maxOffset = int64(chunk.Size) + chunk.Offset
+		}
+	}
+
+	manifestChunk, err = saveFunc(bytes.NewReader(data), "", 0, 0, uint64(len(data)))
+	if err != nil {
+		return nil, err
+	}
+	manifestChunk.IsChunkManifest = true
+	manifestChunk.Offset = minOffset
+	manifestChunk.Size = uint64(maxOffset - minOffset)
+
+	return
+}
+
+type SaveDataAsChunkFunctionType func(reader io.Reader, name string, offset int64, tsNs int64, expectedDataSize uint64) (chunk *filer_pb.FileChunk, err error)

weed/filer/filechunk_manifest_cache.go

@@ -0,0 +1,146 @@
+package filer
+
+import (
+	"container/list"
+	"context"
+	"fmt"
+	"sync"
+
+	"golang.org/x/sync/singleflight"
+)
+
+const (
+	MaxMountChunkManifestCacheEntries = 256
+	MaxMountChunkManifestCacheBytes   = 64 << 20
+)
+
+type chunkManifestCacheKey struct {
+	fileID       string
+	cipherKey    string
+	isCompressed bool
+}
+
+func (k chunkManifestCacheKey) flightKey() string {
+	return fmt.Sprintf("%s\x00%s\x00%t", k.fileID, k.cipherKey, k.isCompressed)
+}
+
+type chunkManifestCacheEntry struct {
+	key  chunkManifestCacheKey
+	data []byte
+}
+
+// ChunkManifestCache is a bounded, thread-safe LRU cache for chunk manifest
+// bytes. Each mount (WFS) owns its own instance so manifests fetched through
+// one filer backend are never served to another. Concurrent cold misses for
+// the same key are coalesced via singleflight so only one fetch runs.
+type ChunkManifestCache struct {
+	mu         sync.Mutex
+	maxEntries int
+	maxBytes   int64
+	bytes      int64
+	entries    map[chunkManifestCacheKey]*list.Element
+	lru        *list.List
+	flight     singleflight.Group
+}
+
+// NewChunkManifestCache creates a bounded LRU cache for chunk manifest bytes.
+func NewChunkManifestCache(maxEntries int, maxBytes int64) *ChunkManifestCache {
+	return &ChunkManifestCache{
+		maxEntries: maxEntries,
+		maxBytes:   maxBytes,
+		entries:    make(map[chunkManifestCacheKey]*list.Element),
+		lru:        list.New(),
+	}
+}
+
+func (c *ChunkManifestCache) get(key chunkManifestCacheKey) ([]byte, bool) {
+	c.mu.Lock()
+	element, found := c.entries[key]
+	if !found {
+		c.mu.Unlock()
+		return nil, false
+	}
+	c.lru.MoveToFront(element)
+	// entry.data is immutable after insertion; copy outside the lock so a
+	// large copy does not block concurrent hits, inserts, and evictions.
+	data := element.Value.(*chunkManifestCacheEntry).data
+	c.mu.Unlock()
+	return append([]byte(nil), data...), true
+}
+
+func (c *ChunkManifestCache) put(key chunkManifestCacheKey, data []byte) {
+	if c.maxEntries <= 0 || int64(len(data)) > c.maxBytes {
+		return
+	}
+
+	entry := &chunkManifestCacheEntry{
+		key:  key,
+		data: append([]byte(nil), data...),
+	}
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if old, found := c.entries[key]; found {
+		c.removeElement(old)
+	}
+
+	element := c.lru.PushFront(entry)
+	c.entries[key] = element
+	c.bytes += int64(len(entry.data))
+
+	for len(c.entries) > c.maxEntries || c.bytes > c.maxBytes {
+		c.removeElement(c.lru.Back())
+	}
+}
+
+// fetchOrLoad returns cached manifest bytes for key, or invokes fetch and
+// caches the result. Concurrent calls for the same key are coalesced via
+// singleflight so only one fetch runs during a cold burst. A caller whose
+// context is canceled while waiting for the in-flight fetch returns
+// ctx.Err() immediately rather than blocking for the leader's result.
+func (c *ChunkManifestCache) fetchOrLoad(ctx context.Context, key chunkManifestCacheKey, fetch func() ([]byte, error)) ([]byte, error) {
+	if data, ok := c.get(key); ok {
+		return data, nil
+	}
+	ch := c.flight.DoChan(key.flightKey(), func() (interface{}, error) {
+		// Re-check under the flight: another flight may have just populated
+		// the cache for this key.
+		if data, ok := c.get(key); ok {
+			return data, nil
+		}
+		data, err := fetch()
+		if err != nil {
+			return nil, err
+		}
+		c.put(key, data)
+		return data, nil
+	})
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case res := <-ch:
+		if res.Err != nil {
+			return nil, res.Err
+		}
+		return res.Val.([]byte), nil
+	}
+}
+
+func (c *ChunkManifestCache) clear() {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.entries = make(map[chunkManifestCacheKey]*list.Element)
+	c.lru.Init()
+	c.bytes = 0
+}
+
+func (c *ChunkManifestCache) removeElement(element *list.Element) {
+	if element == nil {
+		return
+	}
+	c.lru.Remove(element)
+	entry := element.Value.(*chunkManifestCacheEntry)
+	delete(c.entries, entry.key)
+	c.bytes -= int64(len(entry.data))
+}

weed/filer/filechunk_manifest_cache_benchmark_test.go

@@ -0,0 +1,72 @@
+package filer
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func BenchmarkManifestResolutionRepeatedOpen(b *testing.B) {
+	for _, delay := range []time.Duration{0, 5 * time.Millisecond, 20 * time.Millisecond} {
+		b.Run(fmt.Sprintf("latency=%s", delay), func(b *testing.B) {
+			fixture := newManifestReadFixture(b, map[string][]*filer_pb.FileChunk{
+				"benchmark-cached": {resolveTestData("benchmark-data", 0)},
+			}, map[string]time.Duration{"benchmark-cached": delay})
+			var lookups atomic.Int32
+			lookup := func(ctx context.Context, fileID string) ([]string, error) {
+				lookups.Add(1)
+				return fixture.lookup(ctx, fileID)
+			}
+			chunk := newManifestCacheTestChunk("benchmark-cached")
+			cache := NewChunkManifestCache(MaxMountChunkManifestCacheEntries, MaxMountChunkManifestCacheBytes)
+			_, err := resolveOneChunkManifest(context.Background(), lookup, chunk, nil, cache)
+			if err != nil {
+				b.Fatal(err)
+			}
+			fixture.loads.Store(0)
+			lookups.Store(0)
+
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				if _, err := resolveOneChunkManifest(context.Background(), lookup, chunk, nil, cache); err != nil {
+					b.Fatal(err)
+				}
+			}
+			b.StopTimer()
+			b.ReportMetric(float64(fixture.loads.Load())/float64(b.N), "fetches/op")
+			b.ReportMetric(float64(lookups.Load())/float64(b.N), "lookups/op")
+		})
+	}
+}
+
+func BenchmarkManifestResolutionWithoutCache(b *testing.B) {
+	for _, delay := range []time.Duration{0, 5 * time.Millisecond, 20 * time.Millisecond} {
+		b.Run(fmt.Sprintf("latency=%s", delay), func(b *testing.B) {
+			fixture := newManifestReadFixture(b, map[string][]*filer_pb.FileChunk{
+				"benchmark-uncached": {resolveTestData("benchmark-data", 0)},
+			}, map[string]time.Duration{"benchmark-uncached": delay})
+			var lookups atomic.Int32
+			lookup := func(ctx context.Context, fileID string) ([]string, error) {
+				lookups.Add(1)
+				return fixture.lookup(ctx, fileID)
+			}
+			chunk := newManifestCacheTestChunk("benchmark-uncached")
+
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				if _, err := ResolveOneChunkManifest(context.Background(), lookup, chunk, nil); err != nil {
+					b.Fatal(err)
+				}
+			}
+			b.StopTimer()
+			b.ReportMetric(float64(fixture.loads.Load())/float64(b.N), "fetches/op")
+			b.ReportMetric(float64(lookups.Load())/float64(b.N), "lookups/op")
+		})
+	}
+}

weed/filer/filechunk_manifest_cache_test.go

@@ -0,0 +1,305 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
+)
+
+func newManifestCacheTestChunk(fileID string) *filer_pb.FileChunk {
+	return resolveTestManifest(fileID, 0)
+}
+
+func newTestManifestCache(t testing.TB) *ChunkManifestCache {
+	t.Helper()
+	cache := NewChunkManifestCache(MaxMountChunkManifestCacheEntries, MaxMountChunkManifestCacheBytes)
+	t.Cleanup(cache.clear)
+	return cache
+}
+
+func TestChunkGroupManifestResolutionCachesRepeatedOpens(t *testing.T) {
+	cache := newTestManifestCache(t)
+	const manifestID = "cache-repeated-open"
+	fixture := newManifestReadFixture(t, map[string][]*filer_pb.FileChunk{
+		manifestID: {resolveTestData("cached-data", 0)},
+	}, nil)
+	var lookups atomic.Int32
+	lookup := func(ctx context.Context, fileID string) ([]string, error) {
+		lookups.Add(1)
+		return fixture.lookup(ctx, fileID)
+	}
+
+	for i := 0; i < 3; i++ {
+		_, err := NewChunkGroup(lookup, nil, []*filer_pb.FileChunk{
+			newManifestCacheTestChunk(manifestID),
+		}, 1, nil, cache)
+		require.NoError(t, err)
+	}
+
+	require.Equal(t, int32(1), lookups.Load(), "repeated Mount opens should reuse the Manifest lookup")
+	require.Equal(t, int32(1), fixture.loads.Load(), "repeated Mount opens should reuse the Manifest fetch")
+}
+
+func TestChunkGroupManifestResolutionDoesNotCacheFailedReads(t *testing.T) {
+	cache := newTestManifestCache(t)
+	const manifestID = "cache-failed-read"
+	fixture := newManifestReadFixture(t, nil, nil)
+	var lookups atomic.Int32
+	lookup := func(ctx context.Context, fileID string) ([]string, error) {
+		lookups.Add(1)
+		return fixture.lookup(ctx, fileID)
+	}
+	chunk := newManifestCacheTestChunk(manifestID)
+
+	_, err := NewChunkGroup(lookup, nil, []*filer_pb.FileChunk{chunk}, 1, nil, cache)
+	require.Error(t, err)
+
+	fixture.manifests[manifestID] = manifestBytes(t, resolveTestData("retried-data", 0))
+	_, err = NewChunkGroup(lookup, nil, []*filer_pb.FileChunk{chunk}, 1, nil, cache)
+	require.NoError(t, err)
+
+	require.Equal(t, int32(2), lookups.Load(), "a failed lookup must not poison later Mount opens")
+	require.Equal(t, int32(2), fixture.loads.Load(), "a failed fetch must not poison later Mount opens")
+}
+
+func TestChunkGroupManifestResolutionConcurrentWarmOpensReuseFetch(t *testing.T) {
+	cache := newTestManifestCache(t)
+	const manifestID = "cache-concurrent-warm-open"
+	fixture := newManifestReadFixture(t, map[string][]*filer_pb.FileChunk{
+		manifestID: {resolveTestData("concurrent-data", 0)},
+	}, nil)
+	var lookups atomic.Int32
+	lookup := func(ctx context.Context, fileID string) ([]string, error) {
+		lookups.Add(1)
+		return fixture.lookup(ctx, fileID)
+	}
+
+	_, err := NewChunkGroup(lookup, nil, []*filer_pb.FileChunk{
+		newManifestCacheTestChunk(manifestID),
+	}, 1, nil, cache)
+	require.NoError(t, err)
+
+	const concurrentOpens = 8
+	var wg sync.WaitGroup
+	errs := make(chan error, concurrentOpens)
+	for i := 0; i < concurrentOpens; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			_, openErr := NewChunkGroup(lookup, nil, []*filer_pb.FileChunk{
+				newManifestCacheTestChunk(manifestID),
+			}, 1, nil, cache)
+			errs <- openErr
+		}()
+	}
+	wg.Wait()
+	close(errs)
+	for openErr := range errs {
+		require.NoError(t, openErr)
+	}
+
+	require.Equal(t, int32(1), lookups.Load(), "warm concurrent Mount opens should not repeat the lookup")
+	require.Equal(t, int32(1), fixture.loads.Load(), "warm concurrent Mount opens should not repeat the fetch")
+}
+
+func TestChunkGroupManifestResolutionCoalescesColdMisses(t *testing.T) {
+	cache := newTestManifestCache(t)
+	const manifestID = "cache-cold-miss"
+	// Delay the manifest response so the leader's fetch is still in flight
+	// when the other concurrent opens arrive and join the singleflight.
+	fixture := newManifestReadFixture(t, map[string][]*filer_pb.FileChunk{
+		manifestID: {resolveTestData("cold-data", 0)},
+	}, map[string]time.Duration{manifestID: 50 * time.Millisecond})
+	var lookups atomic.Int32
+	lookup := func(ctx context.Context, fileID string) ([]string, error) {
+		lookups.Add(1)
+		return fixture.lookup(ctx, fileID)
+	}
+
+	const concurrentOpens = 8
+	var wg sync.WaitGroup
+	errs := make(chan error, concurrentOpens)
+	for i := 0; i < concurrentOpens; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			_, openErr := NewChunkGroup(lookup, nil, []*filer_pb.FileChunk{
+				newManifestCacheTestChunk(manifestID),
+			}, 1, nil, cache)
+			errs <- openErr
+		}()
+	}
+	wg.Wait()
+	close(errs)
+	for openErr := range errs {
+		require.NoError(t, openErr)
+	}
+
+	require.Equal(t, int32(1), lookups.Load(), "concurrent cold opens should coalesce into a single lookup")
+	require.Equal(t, int32(1), fixture.loads.Load(), "concurrent cold opens should coalesce into a single fetch")
+}
+
+func TestResolveOneChunkManifestDoesNotUseMountCache(t *testing.T) {
+	cache := newTestManifestCache(t)
+	const manifestID = "cache-public-resolver"
+	fixture := newManifestReadFixture(t, map[string][]*filer_pb.FileChunk{
+		manifestID: {resolveTestData("public-resolver-data", 0)},
+	}, nil)
+	var lookups atomic.Int32
+	lookup := func(ctx context.Context, fileID string) ([]string, error) {
+		lookups.Add(1)
+		return fixture.lookup(ctx, fileID)
+	}
+	chunk := newManifestCacheTestChunk(manifestID)
+
+	for i := 0; i < 2; i++ {
+		_, err := ResolveOneChunkManifest(context.Background(), lookup, chunk, nil)
+		require.NoError(t, err)
+	}
+
+	require.Equal(t, int32(2), lookups.Load(), "the general resolver must retain its uncached behavior")
+	require.Equal(t, int32(2), fixture.loads.Load(), "the general resolver must not use the Mount cache")
+
+	// The shared cache must remain empty: a nil-cache caller must not pollute
+	// a mount-owned cache, and vice versa.
+	_, found := cache.get(chunkManifestCacheKey{fileID: manifestID})
+	require.False(t, found, "a nil-cache resolve must not populate an unrelated cache")
+}
+
+func TestChunkGroupManifestResolutionDoesNotCacheMalformedManifest(t *testing.T) {
+	cache := newTestManifestCache(t)
+	const manifestID = "cache-malformed-manifest"
+	fixture := newManifestReadFixture(t, map[string][]*filer_pb.FileChunk{
+		manifestID: nil,
+	}, nil)
+	fixture.manifests[manifestID] = []byte("malformed manifest")
+	chunk := newManifestCacheTestChunk(manifestID)
+
+	_, err := NewChunkGroup(fixture.lookup, nil, []*filer_pb.FileChunk{chunk}, 1, nil, cache)
+	require.Error(t, err)
+
+	fixture.manifests[manifestID] = manifestBytes(t, resolveTestData("after-malformed-data", 0))
+	_, err = NewChunkGroup(fixture.lookup, nil, []*filer_pb.FileChunk{chunk}, 1, nil, cache)
+	require.NoError(t, err)
+	require.Equal(t, int32(2), fixture.loads.Load(), "a malformed manifest must not poison later Mount opens")
+}
+
+func TestChunkManifestCacheSeparatesReadParameters(t *testing.T) {
+	cache := NewChunkManifestCache(4, 1024)
+	base := chunkManifestCacheKey{fileID: "same-file"}
+	cache.put(base, []byte("manifest"))
+
+	_, found := cache.get(chunkManifestCacheKey{fileID: "same-file", cipherKey: "key"})
+	require.False(t, found, "cipher keys must be part of the cache key")
+	_, found = cache.get(chunkManifestCacheKey{fileID: "same-file", isCompressed: true})
+	require.False(t, found, "compression settings must be part of the cache key")
+	_, found = cache.get(chunkManifestCacheKey{fileID: "different-file"})
+	require.False(t, found, "FileIds must be part of the cache key")
+}
+
+func TestChunkManifestCacheCopiesData(t *testing.T) {
+	cache := NewChunkManifestCache(1, 1024)
+	key := chunkManifestCacheKey{fileID: "copy-data"}
+	original := []byte("manifest")
+	cache.put(key, original)
+	original[0] = 'X'
+
+	data, found := cache.get(key)
+	require.True(t, found)
+	require.Equal(t, []byte("manifest"), data)
+	data[0] = 'Y'
+
+	data, found = cache.get(key)
+	require.True(t, found)
+	require.Equal(t, []byte("manifest"), data)
+}
+
+func TestChunkManifestCacheEvictsLeastRecentlyUsedAndOversizedEntries(t *testing.T) {
+	cache := NewChunkManifestCache(2, 6)
+	first := chunkManifestCacheKey{fileID: "first"}
+	second := chunkManifestCacheKey{fileID: "second"}
+	third := chunkManifestCacheKey{fileID: "third"}
+	oversized := chunkManifestCacheKey{fileID: "oversized"}
+
+	cache.put(first, []byte("one"))
+	cache.put(second, []byte("two"))
+	_, found := cache.get(first)
+	require.True(t, found, "the first entry should be present before eviction")
+	cache.put(third, []byte("tri"))
+
+	_, found = cache.get(first)
+	require.True(t, found, "a recent entry should survive LRU eviction")
+	_, found = cache.get(second)
+	require.False(t, found, "the least recently used entry should be evicted")
+	_, found = cache.get(third)
+	require.True(t, found)
+
+	cache.put(oversized, []byte("too large"))
+	_, found = cache.get(oversized)
+	require.False(t, found, "an entry over the byte limit must not be cached")
+}
+
+func TestResolveOneChunkManifestHonorsCanceledContextOnCacheHit(t *testing.T) {
+	cache := NewChunkManifestCache(1, 1024)
+	chunk := newManifestCacheTestChunk("cache-canceled-hit")
+	cache.put(chunkManifestCacheKey{fileID: chunk.GetFileIdString()}, manifestBytes(t, resolveTestData("canceled-data", 0)))
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	lookupCalled := false
+	lookup := func(context.Context, string) ([]string, error) {
+		lookupCalled = true
+		return nil, errors.New("lookup should not be called")
+	}
+
+	_, err := resolveOneChunkManifest(ctx, lookup, chunk, nil, cache)
+	require.ErrorIs(t, err, context.Canceled)
+	require.False(t, lookupCalled, "a canceled cache hit must not issue a lookup")
+}
+
+func TestResolveOneChunkManifestCanceledWaiterReturnsDuringCoalescedMiss(t *testing.T) {
+	cache := newTestManifestCache(t)
+	const manifestID = "cache-canceled-waiter"
+	// Delay the manifest response so the leader's fetch is still in flight
+	// when the waiter arrives and cancels.
+	fixture := newManifestReadFixture(t, map[string][]*filer_pb.FileChunk{
+		manifestID: {resolveTestData("canceled-waiter-data", 0)},
+	}, map[string]time.Duration{manifestID: 100 * time.Millisecond})
+	chunk := newManifestCacheTestChunk(manifestID)
+
+	// Leader starts the fetch with a live context.
+	leaderCtx, leaderCancel := context.WithCancel(context.Background())
+	defer leaderCancel()
+	leaderDone := make(chan error, 1)
+	go func() {
+		_, err := resolveOneChunkManifest(leaderCtx, fixture.lookup, chunk, nil, cache)
+		leaderDone <- err
+	}()
+
+	// Waiter cancels its own context while the leader's fetch is still in
+	// flight; it must return context.Canceled promptly instead of blocking
+	// for the leader's result.
+	waiterCtx, waiterCancel := context.WithCancel(context.Background())
+	waiterCancel()
+	_, err := resolveOneChunkManifest(waiterCtx, fixture.lookup, chunk, nil, cache)
+	require.ErrorIs(t, err, context.Canceled)
+
+	// The leader must still complete successfully and populate the cache.
+	require.NoError(t, <-leaderDone)
+	data, found := cache.get(chunkManifestCacheKey{fileID: manifestID})
+	require.True(t, found, "the leader's successful fetch must populate the cache")
+	require.NotEmpty(t, data)
+}
+
+func manifestBytes(t testing.TB, chunks ...*filer_pb.FileChunk) []byte {
+	t.Helper()
+	data, err := proto.Marshal(&filer_pb.FileChunkManifest{Chunks: chunks})
+	require.NoError(t, err)
+	return data
+}

weed/filer/filechunk_manifest_resolve_test.go

@@ -0,0 +1,595 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
+)
+
+type manifestReadFixture struct {
+	server    *httptest.Server
+	mu        sync.Mutex
+	started   []string
+	completed []string
+	active    atomic.Int32
+	maxActive atomic.Int32
+	loads     atomic.Int32
+	manifests map[string][]byte
+	delays    map[string]time.Duration
+	stopReads chan struct{}
+}
+
+func newManifestReadFixture(t testing.TB, manifests map[string][]*filer_pb.FileChunk, delays map[string]time.Duration) *manifestReadFixture {
+	t.Helper()
+	encoded := make(map[string][]byte, len(manifests))
+	for id, chunks := range manifests {
+		data, err := proto.Marshal(&filer_pb.FileChunkManifest{Chunks: chunks})
+		require.NoError(t, err)
+		encoded[id] = data
+	}
+
+	fixture := &manifestReadFixture{manifests: encoded, delays: delays, stopReads: make(chan struct{})}
+	listener, err := (&net.ListenConfig{}).Listen(context.Background(), "tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	fixture.server = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		id := strings.TrimPrefix(r.URL.Path, "/")
+		fixture.loads.Add(1)
+		fixture.active.Add(1)
+		fixture.updateMaxActive()
+		fixture.mu.Lock()
+		fixture.started = append(fixture.started, id)
+		fixture.mu.Unlock()
+		finishRead := func() {
+			fixture.active.Add(-1)
+			fixture.mu.Lock()
+			fixture.completed = append(fixture.completed, id)
+			fixture.mu.Unlock()
+		}
+
+		if delay := fixture.delays[id]; delay > 0 {
+			timer := time.NewTimer(delay)
+			select {
+			case <-r.Context().Done():
+				timer.Stop()
+				finishRead()
+				return
+			case <-fixture.stopReads:
+				timer.Stop()
+				finishRead()
+				return
+			case <-timer.C:
+			}
+		}
+		data, ok := fixture.manifests[id]
+		if !ok {
+			http.NotFound(w, r)
+			finishRead()
+			return
+		}
+		w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data)))
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write(data)
+		finishRead()
+	}))
+	fixture.server.Listener = listener
+	fixture.server.Start()
+	t.Cleanup(fixture.server.Close)
+	return fixture
+}
+
+func (f *manifestReadFixture) updateMaxActive() {
+	for {
+		current := f.active.Load()
+		maximum := f.maxActive.Load()
+		if current <= maximum || f.maxActive.CompareAndSwap(maximum, current) {
+			return
+		}
+	}
+}
+
+func (f *manifestReadFixture) lookup(ctx context.Context, fileID string) ([]string, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	return []string{f.server.URL + "/" + fileID}, nil
+}
+
+func (f *manifestReadFixture) completionOrder() []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return append([]string(nil), f.completed...)
+}
+
+func resolveTestManifest(id string, offset int64) *filer_pb.FileChunk {
+	return &filer_pb.FileChunk{FileId: id, IsChunkManifest: true, Offset: offset, Size: 100}
+}
+
+func resolveTestData(id string, offset int64) *filer_pb.FileChunk {
+	return &filer_pb.FileChunk{FileId: id, Offset: offset, Size: 10}
+}
+
+func TestResolveChunkManifestParallelReadsAreBounded(t *testing.T) {
+	manifests := make(map[string][]*filer_pb.FileChunk)
+	inputs := make([]*filer_pb.FileChunk, 0, 8)
+	delays := make(map[string]time.Duration)
+	for i := 0; i < 8; i++ {
+		id := fmt.Sprintf("m%d", i)
+		manifests[id] = []*filer_pb.FileChunk{resolveTestData(fmt.Sprintf("d%d", i), int64(i*10))}
+		inputs = append(inputs, resolveTestManifest(id, int64(i*100)))
+		delays[id] = 50 * time.Millisecond
+	}
+	fixture := newManifestReadFixture(t, manifests, delays)
+
+	data, meta, err := ResolveChunkManifest(context.Background(), fixture.lookup, inputs, 0, 1000, nil)
+	require.NoError(t, err)
+	require.Len(t, data, len(inputs))
+	require.Len(t, meta, len(inputs))
+	require.GreaterOrEqual(t, fixture.maxActive.Load(), int32(2), "independent manifests should overlap")
+	require.LessOrEqual(t, fixture.maxActive.Load(), int32(4), "manifest reads must be bounded")
+}
+
+func TestResolveChunkManifestPreservesInputOrderWhenReadsCompleteOutOfOrder(t *testing.T) {
+	fixture := newManifestReadFixture(t,
+		map[string][]*filer_pb.FileChunk{
+			"slow": {resolveTestData("slow-data", 0)},
+			"fast": {resolveTestData("fast-data", 100)},
+		},
+		map[string]time.Duration{"slow": 80 * time.Millisecond, "fast": 5 * time.Millisecond},
+	)
+
+	data, meta, err := ResolveChunkManifest(context.Background(), fixture.lookup, []*filer_pb.FileChunk{
+		resolveTestManifest("slow", 0),
+		resolveTestManifest("fast", 100),
+	}, 0, 200, nil)
+	require.NoError(t, err)
+	require.Equal(t, []string{"slow-data", "fast-data"}, fileIDs(data))
+	require.Equal(t, []string{"slow", "fast"}, fileIDs(meta))
+	require.Equal(t, []string{"fast", "slow"}, fixture.completionOrder())
+}
+
+func TestResolveChunkManifestNestedManifestsKeepDepthFirstOrder(t *testing.T) {
+	fixture := newManifestReadFixture(t,
+		map[string][]*filer_pb.FileChunk{
+			"parent":  {resolveTestData("parent-data", 0), resolveTestManifest("nested", 10)},
+			"sibling": {resolveTestData("sibling-data", 20)},
+			"nested":  {resolveTestData("nested-data", 10)},
+		},
+		map[string]time.Duration{"parent": 60 * time.Millisecond, "sibling": 5 * time.Millisecond},
+	)
+
+	data, meta, err := ResolveChunkManifest(context.Background(), fixture.lookup, []*filer_pb.FileChunk{
+		resolveTestManifest("parent", 0),
+		resolveTestManifest("sibling", 20),
+	}, 0, 100, nil)
+	require.NoError(t, err)
+	require.Equal(t, []string{"parent-data", "nested-data", "sibling-data"}, fileIDs(data))
+	require.Equal(t, []string{"parent", "nested", "sibling"}, fileIDs(meta))
+	require.Equal(t, []string{"sibling", "parent", "nested"}, fixture.completionOrder())
+}
+
+func TestResolveChunkManifestFiltersBeforeReadingAndRecursively(t *testing.T) {
+	fixture := newManifestReadFixture(t,
+		map[string][]*filer_pb.FileChunk{
+			"outside": {resolveTestData("never-read", 0)},
+			"inside":  {resolveTestData("before", 0), resolveTestData("selected", 100), resolveTestData("after", 200)},
+		}, nil)
+
+	data, meta, err := ResolveChunkManifest(context.Background(), fixture.lookup, []*filer_pb.FileChunk{
+		resolveTestManifest("outside", 0),
+		resolveTestManifest("inside", 90),
+	}, 100, 200, nil)
+	require.NoError(t, err)
+	require.Equal(t, []string{"selected"}, fileIDs(data))
+	require.Equal(t, []string{"inside"}, fileIDs(meta))
+	require.Equal(t, int32(1), fixture.loads.Load(), "the non-overlapping parent manifest must not be read")
+}
+
+func TestResolveChunkManifestPropagatesReadAndFormatErrorsInInputOrder(t *testing.T) {
+	fixture := newManifestReadFixture(t,
+		map[string][]*filer_pb.FileChunk{
+			"valid":  {resolveTestData("valid-data", 0)},
+			"broken": nil,
+		}, map[string]time.Duration{"broken": 50 * time.Millisecond})
+	fixture.manifests["broken"] = []byte("not a protobuf manifest")
+
+	data, meta, err := ResolveChunkManifest(context.Background(), fixture.lookup, []*filer_pb.FileChunk{
+		resolveTestData("plain", 0),
+		resolveTestManifest("valid", 10),
+		resolveTestManifest("broken", 20),
+	}, 0, 100, nil)
+	require.Error(t, err)
+	require.Nil(t, meta)
+	require.Equal(t, []string{"plain", "valid-data"}, fileIDs(data))
+	require.Contains(t, err.Error(), "fail to unmarshal manifest broken")
+
+	lookupErr := errors.New("lookup failed")
+	lookup := func(context.Context, string) ([]string, error) { return nil, lookupErr }
+	_, _, err = ResolveChunkManifest(context.Background(), lookup, []*filer_pb.FileChunk{resolveTestManifest("lookup-error", 0)}, 0, 100, nil)
+	require.ErrorIs(t, err, lookupErr)
+}
+
+func TestResolveChunkManifestCancellationStopsAllReads(t *testing.T) {
+	fixture := newManifestReadFixture(t,
+		map[string][]*filer_pb.FileChunk{
+			"one": {resolveTestData("one-data", 0)},
+			"two": {resolveTestData("two-data", 10)},
+		},
+		map[string]time.Duration{"one": time.Second, "two": time.Second},
+	)
+	ctx, cancel := context.WithCancel(context.Background())
+	result := make(chan error, 1)
+	go func() {
+		_, _, err := ResolveChunkManifest(ctx, fixture.lookup, []*filer_pb.FileChunk{
+			resolveTestManifest("one", 0),
+			resolveTestManifest("two", 10),
+		}, 0, 100, nil)
+		result <- err
+	}()
+
+	deadline := time.Now().Add(time.Second)
+	for fixture.loads.Load() < 1 && time.Now().Before(deadline) {
+		time.Sleep(time.Millisecond)
+	}
+	close(fixture.stopReads)
+	cancel()
+	err := <-result
+	require.ErrorIs(t, err, context.Canceled)
+	require.Eventually(t, func() bool {
+		return fixture.active.Load() == 0
+	}, time.Second, time.Millisecond, "all manifest readers must exit after ResolveChunkManifest returns")
+}
+
+type manifestFailureReadFixture struct {
+	server       *httptest.Server
+	fastRelease  chan struct{}
+	slowStarted  chan struct{}
+	fastStarted  chan struct{}
+	slowCanceled chan struct{}
+}
+
+func newManifestFailureReadFixture(t testing.TB) *manifestFailureReadFixture {
+	t.Helper()
+	fixture := &manifestFailureReadFixture{
+		fastRelease:  make(chan struct{}),
+		slowStarted:  make(chan struct{}),
+		fastStarted:  make(chan struct{}),
+		slowCanceled: make(chan struct{}),
+	}
+	fixture.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch strings.TrimPrefix(r.URL.Path, "/") {
+		case "fast":
+			close(fixture.fastStarted)
+			<-fixture.fastRelease
+			w.Header().Set("Content-Length", "7")
+			w.WriteHeader(http.StatusOK)
+			_, _ = w.Write([]byte("invalid"))
+		case "slow":
+			close(fixture.slowStarted)
+			<-r.Context().Done()
+			close(fixture.slowCanceled)
+		}
+	}))
+	t.Cleanup(fixture.server.Close)
+	return fixture
+}
+
+func (f *manifestFailureReadFixture) lookup(ctx context.Context, fileID string) ([]string, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	return []string{f.server.URL + "/" + fileID}, nil
+}
+
+func waitForManifestFailureSignal(t *testing.T, signal <-chan struct{}) {
+	t.Helper()
+	select {
+	case <-signal:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for manifest request")
+	}
+}
+
+func TestResolveChunkManifestFastFailureCancelsSlowSibling(t *testing.T) {
+	testCases := []struct {
+		name       string
+		inputIDs   []string
+		expectedID string
+	}{
+		{name: "fast then slow returns quickly", inputIDs: []string{"fast", "slow"}, expectedID: "fast"},
+		{name: "slow then fast keeps real error", inputIDs: []string{"slow", "fast"}, expectedID: "fast"},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			fixture := newManifestFailureReadFixture(t)
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			chunks := make([]*filer_pb.FileChunk, 0, len(testCase.inputIDs))
+			for i, id := range testCase.inputIDs {
+				chunks = append(chunks, resolveTestManifest(id, int64(i*100)))
+			}
+			result := make(chan error, 1)
+			go func() {
+				_, _, err := ResolveChunkManifest(ctx, fixture.lookup, chunks, 0, 200, nil)
+				result <- err
+			}()
+
+			waitForManifestFailureSignal(t, fixture.fastStarted)
+			waitForManifestFailureSignal(t, fixture.slowStarted)
+			close(fixture.fastRelease)
+
+			var err error
+			select {
+			case err = <-result:
+			case <-time.After(time.Second):
+				cancel()
+				select {
+				case <-result:
+				case <-time.After(time.Second):
+					t.Fatal("ResolveChunkManifest did not finish after caller cancellation")
+				}
+				t.Fatal("fast manifest failure waited for the slow sibling")
+			}
+			require.Error(t, err)
+			require.Contains(t, err.Error(), "fail to unmarshal manifest "+testCase.expectedID)
+			require.NotErrorIs(t, err, context.Canceled)
+			waitForManifestFailureSignal(t, fixture.slowCanceled)
+		})
+	}
+}
+
+func TestResolveChunkManifestFastFailureCancelsEncryptedSibling(t *testing.T) {
+	fixture := newManifestFailureReadFixture(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	slowChunk := resolveTestManifest("slow", 0)
+	slowChunk.CipherKey = []byte("0123456789abcdef")
+	chunks := []*filer_pb.FileChunk{
+		slowChunk,
+		resolveTestManifest("fast", 100),
+	}
+	result := make(chan error, 1)
+	go func() {
+		_, _, err := ResolveChunkManifest(ctx, fixture.lookup, chunks, 0, 200, nil)
+		result <- err
+	}()
+
+	waitForManifestFailureSignal(t, fixture.fastStarted)
+	waitForManifestFailureSignal(t, fixture.slowStarted)
+	close(fixture.fastRelease)
+
+	var err error
+	select {
+	case err = <-result:
+	case <-time.After(time.Second):
+		cancel()
+		select {
+		case <-result:
+		case <-time.After(time.Second):
+			t.Fatal("ResolveChunkManifest did not finish after caller cancellation")
+		}
+		t.Fatal("fast manifest failure waited for the encrypted slow sibling")
+	}
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "fail to unmarshal manifest fast")
+	require.NotErrorIs(t, err, context.Canceled)
+	waitForManifestFailureSignal(t, fixture.slowCanceled)
+}
+
+func TestResolveChunkManifestKeepsRealErrorAfterInternalCancellation(t *testing.T) {
+	earlyStarted := make(chan struct{})
+	lateStarted := make(chan struct{})
+	earlyErr := errors.New("early lookup failed")
+	lateErr := errors.New("late lookup failed")
+	lookup := func(ctx context.Context, fileID string) ([]string, error) {
+		switch fileID {
+		case "early":
+			close(earlyStarted)
+			<-ctx.Done()
+			return nil, earlyErr
+		case "late":
+			close(lateStarted)
+			return nil, lateErr
+		default:
+			return nil, fmt.Errorf("unexpected manifest %s", fileID)
+		}
+	}
+
+	result := make(chan error, 1)
+	go func() {
+		_, _, err := ResolveChunkManifest(context.Background(), lookup, []*filer_pb.FileChunk{
+			resolveTestManifest("early", 0),
+			resolveTestManifest("late", 100),
+		}, 0, 200, nil)
+		result <- err
+	}()
+	waitForManifestFailureSignal(t, earlyStarted)
+	waitForManifestFailureSignal(t, lateStarted)
+
+	select {
+	case err := <-result:
+		require.ErrorIs(t, err, earlyErr)
+		require.NotErrorIs(t, err, lateErr)
+	case <-time.After(time.Second):
+		t.Fatal("ResolveChunkManifest did not return the input-order error")
+	}
+}
+
+func TestResolveChunkManifestReturnsLaterFailureWithoutRecursingEarlierChildren(t *testing.T) {
+	fixture := newManifestReadFixture(t,
+		map[string][]*filer_pb.FileChunk{
+			"a":       {resolveTestManifest("a-child", 0)},
+			"a-child": {resolveTestData("a-child-data", 0)},
+			"b":       nil,
+		},
+		map[string]time.Duration{"a": 5 * time.Millisecond, "b": 50 * time.Millisecond, "a-child": 2 * time.Second},
+	)
+	fixture.manifests["b"] = []byte("not a protobuf manifest")
+
+	start := time.Now()
+	data, meta, err := ResolveChunkManifest(context.Background(), fixture.lookup, []*filer_pb.FileChunk{
+		resolveTestManifest("a", 0),
+		resolveTestManifest("b", 100),
+	}, 0, 200, nil)
+	elapsed := time.Since(start)
+
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "fail to unmarshal manifest b")
+	require.Empty(t, data, "earlier manifest's children must not be recursed when a later manifest already failed")
+	require.Nil(t, meta)
+	require.Equal(t, int32(2), fixture.loads.Load(), "only the two top-level manifests must be read, not a-child")
+	require.Less(t, elapsed, time.Second, "must return promptly without waiting for a-child's slow read")
+}
+
+func TestResolveChunkManifestExcludesOutOfRangeChildrenOnLaterFailure(t *testing.T) {
+	fixture := newManifestReadFixture(t,
+		map[string][]*filer_pb.FileChunk{
+			"a": {
+				resolveTestData("a-in-range", 0),
+				resolveTestData("a-out-of-range", 500),
+			},
+			"b": nil,
+		},
+		map[string]time.Duration{"b": 50 * time.Millisecond},
+	)
+	fixture.manifests["b"] = []byte("not a protobuf manifest")
+
+	data, _, err := ResolveChunkManifest(context.Background(), fixture.lookup, []*filer_pb.FileChunk{
+		resolveTestManifest("a", 0),
+		resolveTestManifest("b", 100),
+	}, 0, 200, nil)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "fail to unmarshal manifest b")
+	require.Equal(t, []string{"a-in-range"}, fileIDs(data), "out-of-range child chunks must be excluded from partial results")
+}
+
+func TestResolveChunkManifestQueuesJobsBeyondWorkerCount(t *testing.T) {
+	// More manifests than workers. The first four stall; the fifth fails
+	// promptly. With a buffered job channel, the fifth job is queued without
+	// blocking submission. When a stalled read is released, a worker picks up
+	// the failing job, which cancels the batch and unblocks the rest.
+	stallRelease := make(chan struct{})
+	stallStarted := make(chan struct{}, 4)
+	lookup := func(ctx context.Context, fileID string) ([]string, error) {
+		switch fileID {
+		case "stall-0", "stall-1", "stall-2", "stall-3":
+			stallStarted <- struct{}{}
+			select {
+			case <-stallRelease:
+				return nil, nil
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			}
+		case "fail":
+			return nil, errors.New("prompt lookup failure")
+		default:
+			return nil, fmt.Errorf("unexpected manifest %s", fileID)
+		}
+	}
+
+	result := make(chan error, 1)
+	go func() {
+		_, _, err := ResolveChunkManifest(context.Background(), lookup, []*filer_pb.FileChunk{
+			resolveTestManifest("stall-0", 0),
+			resolveTestManifest("stall-1", 100),
+			resolveTestManifest("stall-2", 200),
+			resolveTestManifest("stall-3", 300),
+			resolveTestManifest("fail", 400),
+		}, 0, 500, nil)
+		result <- err
+	}()
+
+	// Wait until all 4 workers are stalled.
+	for i := 0; i < 4; i++ {
+		waitForManifestFailureSignal(t, stallStarted)
+	}
+
+	// Release the stalled reads so workers can pick up the queued failing job.
+	close(stallRelease)
+
+	select {
+	case err := <-result:
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "fail to read manifest fail")
+	case <-time.After(2 * time.Second):
+		t.Fatal("failing job was not picked up after releasing stalled workers")
+	}
+}
+
+func fileIDs(chunks []*filer_pb.FileChunk) []string {
+	ids := make([]string, 0, len(chunks))
+	for _, chunk := range chunks {
+		ids = append(ids, chunk.GetFileIdString())
+	}
+	return ids
+}
+
+func resolveChunkManifestSerialForBenchmark(ctx context.Context, lookupFileIdFn func(context.Context, string) ([]string, error), chunks []*filer_pb.FileChunk, startOffset, stopOffset int64) (dataChunks, manifestChunks []*filer_pb.FileChunk, err error) {
+	for _, chunk := range chunks {
+		if max(chunk.Offset, startOffset) >= min(chunk.Offset+int64(chunk.Size), stopOffset) {
+			continue
+		}
+		if !chunk.IsChunkManifest {
+			dataChunks = append(dataChunks, chunk)
+			continue
+		}
+
+		resolvedChunks, resolveErr := ResolveOneChunkManifest(ctx, lookupFileIdFn, chunk, nil)
+		if resolveErr != nil {
+			return dataChunks, nil, resolveErr
+		}
+		manifestChunks = append(manifestChunks, chunk)
+		subDataChunks, subManifestChunks, subErr := resolveChunkManifestSerialForBenchmark(ctx, lookupFileIdFn, resolvedChunks, startOffset, stopOffset)
+		if subErr != nil {
+			return dataChunks, nil, subErr
+		}
+		dataChunks = append(dataChunks, subDataChunks...)
+		manifestChunks = append(manifestChunks, subManifestChunks...)
+	}
+	return dataChunks, manifestChunks, nil
+}
+
+func BenchmarkResolveChunkManifestSerialVsParallel(b *testing.B) {
+	manifests := make(map[string][]*filer_pb.FileChunk)
+	inputs := make([]*filer_pb.FileChunk, 0, 8)
+	delays := make(map[string]time.Duration)
+	for i := 0; i < 8; i++ {
+		id := fmt.Sprintf("m%d", i)
+		manifests[id] = []*filer_pb.FileChunk{resolveTestData(fmt.Sprintf("d%d", i), int64(i*10))}
+		inputs = append(inputs, resolveTestManifest(id, int64(i*10)))
+		delays[id] = time.Millisecond
+	}
+	fixture := newManifestReadFixture(b, manifests, delays)
+	b.ReportAllocs()
+
+	b.Run("serial", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			_, _, err := resolveChunkManifestSerialForBenchmark(context.Background(), fixture.lookup, inputs, 0, 1000)
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+	b.Run("parallel", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			_, _, err := ResolveChunkManifest(context.Background(), fixture.lookup, inputs, 0, 1000, nil)
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+	})
+}

weed/filer/filechunk_manifest_rollback_test.go

@@ -0,0 +1,133 @@
+package filer
+
+import (
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func flatTestChunks(n int) []*filer_pb.FileChunk {
+	chunks := make([]*filer_pb.FileChunk, n)
+	for i := range chunks {
+		chunks[i] = &filer_pb.FileChunk{
+			FileId: fmt.Sprintf("1,%x", i+1),
+			Offset: int64(i) * 8,
+			Size:   8,
+		}
+	}
+	return chunks
+}
+
+type fakeManifestStore struct {
+	saves   int
+	failOn  int // 1-based save call to fail on; 0 = never
+	deleted []*filer_pb.FileChunk
+}
+
+func (s *fakeManifestStore) save(reader io.Reader, name string, offset int64, tsNs int64, expectedDataSize uint64) (*filer_pb.FileChunk, error) {
+	s.saves++
+	if s.saves == s.failOn {
+		return nil, fmt.Errorf("save %d failed", s.saves)
+	}
+	if _, err := io.Copy(io.Discard, reader); err != nil {
+		return nil, err
+	}
+	return &filer_pb.FileChunk{FileId: fmt.Sprintf("2,%x", s.saves), Offset: offset}, nil
+}
+
+func (s *fakeManifestStore) delete(chunks []*filer_pb.FileChunk) {
+	s.deleted = append(s.deleted, chunks...)
+}
+
+func TestMaybeManifestizeFolds(t *testing.T) {
+	store := &fakeManifestStore{}
+	chunks := flatTestChunks(ManifestBatch + 50)
+
+	result, err := MaybeManifestize(store.save, store.delete, chunks)
+	if err != nil {
+		t.Fatalf("MaybeManifestize: %v", err)
+	}
+
+	manifests, data := SeparateManifestChunks(result)
+	if len(manifests) != 1 || len(data) != 50 {
+		t.Fatalf("expected 1 manifest + 50 flat chunks, got %d + %d", len(manifests), len(data))
+	}
+	if store.saves != 1 || len(store.deleted) != 0 {
+		t.Errorf("expected 1 save and no deletes, got %d saves, %d deleted", store.saves, len(store.deleted))
+	}
+	if manifests[0].Offset != 0 || manifests[0].Size != uint64(ManifestBatch*8) {
+		t.Errorf("manifest span [%d,%d) wrong", manifests[0].Offset, manifests[0].Size)
+	}
+}
+
+func TestMaybeManifestizeBelowThreshold(t *testing.T) {
+	store := &fakeManifestStore{}
+	chunks := flatTestChunks(ManifestBatch - 1)
+
+	result, err := MaybeManifestize(store.save, store.delete, chunks)
+	if err != nil {
+		t.Fatalf("MaybeManifestize: %v", err)
+	}
+	if len(result) != len(chunks) || store.saves != 0 {
+		t.Fatalf("expected untouched flat list, got %d chunks, %d saves", len(result), store.saves)
+	}
+}
+
+// A fold that fails midway must hand back the caller's own list and delete the
+// manifest blobs its earlier batches already uploaded.
+func TestMaybeManifestizeRollsBackPartialFold(t *testing.T) {
+	store := &fakeManifestStore{failOn: 2}
+	chunks := flatTestChunks(2*ManifestBatch + 50)
+
+	result, err := MaybeManifestize(store.save, store.delete, chunks)
+	if err == nil {
+		t.Fatal("expected the failed save to be reported")
+	}
+	if len(result) != len(chunks) {
+		t.Fatalf("expected fallback to %d flat chunks, got %d", len(chunks), len(result))
+	}
+	if HasChunkManifest(result) {
+		t.Error("fallback list must not contain manifest chunks")
+	}
+	if len(store.deleted) != 1 || store.deleted[0].FileId != "2,1" {
+		t.Fatalf("expected the first saved blob deleted, got %+v", store.deleted)
+	}
+}
+
+// The manifests a caller came in with are not re-folded, and must survive a
+// fold that fails on the flat remainder: returning only the data chunks would
+// lose everything they cover.
+func TestMaybeManifestizeKeepsExistingManifestsOnFailure(t *testing.T) {
+	store := &fakeManifestStore{failOn: 2}
+	existing := &filer_pb.FileChunk{FileId: "m,1", IsChunkManifest: true, Offset: 0, Size: 24}
+	chunks := append([]*filer_pb.FileChunk{existing}, flatTestChunks(2*ManifestBatch+50)...)
+
+	result, _ := MaybeManifestize(store.save, store.delete, chunks)
+
+	if len(result) != len(chunks) {
+		t.Fatalf("expected original %d chunks, got %d", len(chunks), len(result))
+	}
+	if result[0].FileId != "m,1" || !result[0].IsChunkManifest {
+		t.Fatalf("existing manifest dropped: %+v", result[0])
+	}
+	if len(store.deleted) != 1 || store.deleted[0].FileId != "2,1" {
+		t.Fatalf("expected the first saved blob deleted, got %+v", store.deleted)
+	}
+}
+
+// Without a deleter the fold still falls back to the caller's list; the blobs
+// it saved are only reported.
+func TestMaybeManifestizeRollbackWithoutDeleter(t *testing.T) {
+	store := &fakeManifestStore{failOn: 2}
+	chunks := flatTestChunks(2*ManifestBatch + 50)
+
+	result, err := MaybeManifestize(store.save, nil, chunks)
+	if err == nil {
+		t.Fatal("expected the failed save to be reported")
+	}
+	if len(result) != len(chunks) || HasChunkManifest(result) {
+		t.Fatalf("expected fallback to %d flat chunks, got %d", len(chunks), len(result))
+	}
+}

weed/filer/filechunk_manifest_test.go

@@ -0,0 +1,732 @@
+package filer
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestDoMaybeManifestize(t *testing.T) {
+	var manifestTests = []struct {
+		inputs   []*filer_pb.FileChunk
+		expected []*filer_pb.FileChunk
+	}{
+		{
+			inputs: []*filer_pb.FileChunk{
+				{FileId: "1", IsChunkManifest: false},
+				{FileId: "2", IsChunkManifest: false},
+				{FileId: "3", IsChunkManifest: false},
+				{FileId: "4", IsChunkManifest: false},
+			},
+			expected: []*filer_pb.FileChunk{
+				{FileId: "12", IsChunkManifest: true},
+				{FileId: "34", IsChunkManifest: true},
+			},
+		},
+		{
+			inputs: []*filer_pb.FileChunk{
+				{FileId: "1", IsChunkManifest: true},
+				{FileId: "2", IsChunkManifest: false},
+				{FileId: "3", IsChunkManifest: false},
+				{FileId: "4", IsChunkManifest: false},
+			},
+			expected: []*filer_pb.FileChunk{
+				{FileId: "1", IsChunkManifest: true},
+				{FileId: "23", IsChunkManifest: true},
+				{FileId: "4", IsChunkManifest: false},
+			},
+		},
+		{
+			inputs: []*filer_pb.FileChunk{
+				{FileId: "1", IsChunkManifest: false},
+				{FileId: "2", IsChunkManifest: true},
+				{FileId: "3", IsChunkManifest: false},
+				{FileId: "4", IsChunkManifest: false},
+			},
+			expected: []*filer_pb.FileChunk{
+				{FileId: "2", IsChunkManifest: true},
+				{FileId: "13", IsChunkManifest: true},
+				{FileId: "4", IsChunkManifest: false},
+			},
+		},
+		{
+			inputs: []*filer_pb.FileChunk{
+				{FileId: "1", IsChunkManifest: true},
+				{FileId: "2", IsChunkManifest: true},
+				{FileId: "3", IsChunkManifest: false},
+				{FileId: "4", IsChunkManifest: false},
+			},
+			expected: []*filer_pb.FileChunk{
+				{FileId: "1", IsChunkManifest: true},
+				{FileId: "2", IsChunkManifest: true},
+				{FileId: "34", IsChunkManifest: true},
+			},
+		},
+	}
+
+	for i, mtest := range manifestTests {
+		println("test", i)
+		actual, _ := doMaybeManifestize(nil, mtest.inputs, 2, mockMerge)
+		assertEqualChunks(t, mtest.expected, actual)
+	}
+
+}
+
+func assertEqualChunks(t *testing.T, expected, actual []*filer_pb.FileChunk) {
+	assert.Equal(t, len(expected), len(actual))
+	for i := 0; i < len(actual); i++ {
+		assertEqualChunk(t, actual[i], expected[i])
+	}
+}
+func assertEqualChunk(t *testing.T, expected, actual *filer_pb.FileChunk) {
+	assert.Equal(t, expected.FileId, actual.FileId)
+	assert.Equal(t, expected.IsChunkManifest, actual.IsChunkManifest)
+}
+
+func mockMerge(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer_pb.FileChunk) (manifestChunk *filer_pb.FileChunk, err error) {
+
+	var buf bytes.Buffer
+	minOffset, maxOffset := int64(math.MaxInt64), int64(math.MinInt64)
+	for k := 0; k < len(dataChunks); k++ {
+		chunk := dataChunks[k]
+		buf.WriteString(chunk.FileId)
+		if minOffset > int64(chunk.Offset) {
+			minOffset = chunk.Offset
+		}
+		if maxOffset < int64(chunk.Size)+chunk.Offset {
+			maxOffset = int64(chunk.Size) + chunk.Offset
+		}
+	}
+
+	manifestChunk = &filer_pb.FileChunk{
+		FileId: buf.String(),
+	}
+	manifestChunk.IsChunkManifest = true
+	manifestChunk.Offset = minOffset
+	manifestChunk.Size = uint64(maxOffset - minOffset)
+
+	return
+}
+
+// ---------------------------------------------------------------------------
+// In-memory manifest store for round-trip tests
+// ---------------------------------------------------------------------------
+
+// testManifestStore provides in-memory save/resolve for manifest tests
+// without requiring volume servers.
+type testManifestStore struct {
+	stored map[string][]byte
+	nextID int
+}
+
+func newTestManifestStore() *testManifestStore {
+	return &testManifestStore{stored: make(map[string][]byte)}
+}
+
+func (s *testManifestStore) saveFunc() SaveDataAsChunkFunctionType {
+	return func(reader io.Reader, name string, offset int64, tsNs int64, _ uint64) (*filer_pb.FileChunk, error) {
+		data, err := io.ReadAll(reader)
+		if err != nil {
+			return nil, err
+		}
+		s.nextID++
+		chunk := &filer_pb.FileChunk{
+			Fid: &filer_pb.FileId{VolumeId: 999, FileKey: uint64(s.nextID), Cookie: 0},
+		}
+		s.stored[chunk.GetFileIdString()] = data
+		return chunk, nil
+	}
+}
+
+// resolve expands a single manifest chunk by deserializing the stored proto.
+func (s *testManifestStore) resolve(chunk *filer_pb.FileChunk) ([]*filer_pb.FileChunk, error) {
+	data, ok := s.stored[chunk.GetFileIdString()]
+	if !ok {
+		return nil, fmt.Errorf("manifest %q not in store", chunk.GetFileIdString())
+	}
+	m := &filer_pb.FileChunkManifest{}
+	if err := proto.Unmarshal(data, m); err != nil {
+		return nil, err
+	}
+	filer_pb.AfterEntryDeserialization(m.Chunks)
+	return m.Chunks, nil
+}
+
+// resolveAll expands all manifest chunks, returns (dataChunks, manifestChunks).
+func (s *testManifestStore) resolveAll(chunks []*filer_pb.FileChunk) (data, manifests []*filer_pb.FileChunk, err error) {
+	for _, c := range chunks {
+		if !c.IsChunkManifest {
+			data = append(data, c)
+			continue
+		}
+		manifests = append(manifests, c)
+		sub, resolveErr := s.resolve(c)
+		if resolveErr != nil {
+			return nil, nil, resolveErr
+		}
+		data = append(data, sub...)
+	}
+	return
+}
+
+func testChunk(volId uint32, key uint64, cookie uint32, offset int64, size uint64, tsNs int64) *filer_pb.FileChunk {
+	return &filer_pb.FileChunk{
+		Fid:          &filer_pb.FileId{VolumeId: volId, FileKey: key, Cookie: cookie},
+		Offset:       offset,
+		Size:         size,
+		ModifiedTsNs: tsNs,
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Manifest round-trip: create -> serialize -> deserialize -> verify
+// ---------------------------------------------------------------------------
+
+func TestManifestRoundTripPreservesChunks(t *testing.T) {
+	store := newTestManifestStore()
+
+	chunks := []*filer_pb.FileChunk{
+		testChunk(1, 1, 100, 0, 1000, 10),
+		testChunk(1, 2, 200, 1000, 1000, 20),
+		testChunk(2, 3, 300, 2000, 1000, 30),
+	}
+
+	origIds := make([]string, len(chunks))
+	for i, c := range chunks {
+		origIds[i] = c.GetFileIdString()
+	}
+
+	// Use real mergeIntoManifest with batch=3 so all go into one manifest
+	result, err := doMaybeManifestize(store.saveFunc(), chunks, 3, mergeIntoManifest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(result) != 1 || !result[0].IsChunkManifest {
+		t.Fatalf("expected 1 manifest chunk, got %d chunks", len(result))
+	}
+	if result[0].Offset != 0 || result[0].Size != 3000 {
+		t.Errorf("manifest coverage: offset=%d size=%d, want 0/3000", result[0].Offset, result[0].Size)
+	}
+
+	resolved, err := store.resolve(result[0])
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(resolved) != 3 {
+		t.Fatalf("expected 3 sub-chunks, got %d", len(resolved))
+	}
+	for i, got := range resolved {
+		if got.GetFileIdString() != origIds[i] {
+			t.Errorf("chunk %d: fileId %q != %q", i, got.GetFileIdString(), origIds[i])
+		}
+		if got.Offset != chunks[i].Offset {
+			t.Errorf("chunk %d: offset %d != %d", i, got.Offset, chunks[i].Offset)
+		}
+		if got.Size != chunks[i].Size {
+			t.Errorf("chunk %d: size %d != %d", i, got.Size, chunks[i].Size)
+		}
+		if got.ModifiedTsNs != chunks[i].ModifiedTsNs {
+			t.Errorf("chunk %d: ts %d != %d", i, got.ModifiedTsNs, chunks[i].ModifiedTsNs)
+		}
+	}
+}
+
+func TestMaybeManifestizePreservesSSEMetadata(t *testing.T) {
+	for _, sseType := range []filer_pb.SSEType{
+		filer_pb.SSEType_SSE_C,
+		filer_pb.SSEType_SSE_KMS,
+		filer_pb.SSEType_SSE_S3,
+	} {
+		t.Run(sseType.String(), func(t *testing.T) {
+			store := newTestManifestStore()
+			chunks := make([]*filer_pb.FileChunk, ManifestBatch+1)
+			for i := range chunks {
+				chunks[i] = testChunk(1, uint64(i+1), uint32(i+1), int64(i*8), 8, int64(i+1))
+				chunks[i].SseType = sseType
+				chunks[i].SseMetadata = []byte(fmt.Sprintf("metadata-%d", i))
+			}
+
+			packed, err := MaybeManifestize(store.saveFunc(), nil, chunks)
+			if err != nil {
+				t.Fatal(err)
+			}
+			manifests, remainder := SeparateManifestChunks(packed)
+			if len(manifests) != 1 || len(remainder) != 1 {
+				t.Fatalf("got %d manifests and %d remaining chunks", len(manifests), len(remainder))
+			}
+
+			resolved, err := store.resolve(manifests[0])
+			if err != nil {
+				t.Fatal(err)
+			}
+			for i, chunk := range resolved {
+				if chunk.GetSseType() != sseType {
+					t.Fatalf("chunk %d SSE type = %s, want %s", i, chunk.GetSseType(), sseType)
+				}
+				wantMetadata := fmt.Sprintf("metadata-%d", i)
+				if string(chunk.GetSseMetadata()) != wantMetadata {
+					t.Fatalf("chunk %d SSE metadata = %q, want %q", i, chunk.GetSseMetadata(), wantMetadata)
+				}
+			}
+		})
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Compact resolved overlapping manifests: older sub-chunks become garbage
+// ---------------------------------------------------------------------------
+
+func TestCompactResolvedOverlappingManifests(t *testing.T) {
+	store := newTestManifestStore()
+
+	// Batch 1: older writes covering [0, 3000)
+	batch1 := []*filer_pb.FileChunk{
+		testChunk(1, 1, 1, 0, 1000, 1),
+		testChunk(1, 2, 2, 1000, 1000, 2),
+		testChunk(1, 3, 3, 2000, 1000, 3),
+	}
+	origBatch1Ids := make(map[string]bool)
+	for _, c := range batch1 {
+		origBatch1Ids[c.GetFileIdString()] = true
+	}
+
+	// Batch 2: newer writes covering [0, 3000)
+	batch2 := []*filer_pb.FileChunk{
+		testChunk(2, 1, 1, 0, 1000, 10),
+		testChunk(2, 2, 2, 1000, 1000, 11),
+		testChunk(2, 3, 3, 2000, 1000, 12),
+	}
+	origBatch2Ids := make(map[string]bool)
+	for _, c := range batch2 {
+		origBatch2Ids[c.GetFileIdString()] = true
+	}
+
+	// Manifestize each batch separately (simulates successive flush cycles)
+	save := store.saveFunc()
+	manifest1, err := doMaybeManifestize(save, batch1, 3, mergeIntoManifest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	manifest2, err := doMaybeManifestize(save, batch2, 3, mergeIntoManifest)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Resolve all manifests into sub-chunks
+	allChunks := append(manifest1, manifest2...)
+	dataChunks, _, err := store.resolveAll(allChunks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(dataChunks) != 6 {
+		t.Fatalf("expected 6 resolved data chunks, got %d", len(dataChunks))
+	}
+
+	// CompactFileChunks on resolved sub-chunks (nil lookupFn is fine for non-manifest chunks)
+	compacted, garbage := CompactFileChunks(context.Background(), nil, dataChunks)
+
+	if len(compacted) != 3 {
+		t.Fatalf("expected 3 compacted chunks, got %d", len(compacted))
+	}
+	if len(garbage) != 3 {
+		t.Fatalf("expected 3 garbage chunks, got %d", len(garbage))
+	}
+
+	for _, c := range compacted {
+		if !origBatch2Ids[c.GetFileIdString()] {
+			t.Errorf("compacted chunk %q should be from newer batch", c.GetFileIdString())
+		}
+	}
+	for _, c := range garbage {
+		if !origBatch1Ids[c.GetFileIdString()] {
+			t.Errorf("garbage chunk %q should be from older batch", c.GetFileIdString())
+		}
+	}
+}
+
+// ---------------------------------------------------------------------------
+// DoMinusChunks: old manifest sub-chunks identified as garbage vs new chunks
+// ---------------------------------------------------------------------------
+
+func TestDoMinusChunksWithResolvedManifests(t *testing.T) {
+	store := newTestManifestStore()
+
+	// Old entry: data packed into a manifest
+	oldData := []*filer_pb.FileChunk{
+		testChunk(1, 1, 1, 0, 1000, 1),
+		testChunk(1, 2, 2, 1000, 1000, 2),
+		testChunk(1, 3, 3, 2000, 1000, 3),
+	}
+	oldManifest, err := doMaybeManifestize(store.saveFunc(), oldData, 3, mergeIntoManifest)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Resolve old manifests into sub-chunks (what MinusChunks does internally)
+	oldResolved, oldMeta, err := store.resolveAll(oldManifest)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// New entry: clean re-uploaded chunks (from merge) with different file IDs
+	newData := []*filer_pb.FileChunk{
+		testChunk(3, 10, 10, 0, 1500, 100),
+		testChunk(3, 11, 11, 1500, 1500, 101),
+	}
+
+	// DoMinusChunks: data sub-chunks in old but not in new
+	deltaData := DoMinusChunks(oldResolved, newData)
+	if len(deltaData) != 3 {
+		t.Fatalf("expected 3 old data chunks as garbage, got %d", len(deltaData))
+	}
+
+	// DoMinusChunks: manifest chunks in old but not in new
+	deltaMeta := DoMinusChunks(oldMeta, nil)
+	if len(deltaMeta) != 1 {
+		t.Fatalf("expected 1 old manifest chunk as garbage, got %d", len(deltaMeta))
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Small-batch manifestize: verify sub-chunks and remainder handling
+// ---------------------------------------------------------------------------
+
+func TestManifestizeSmallBatchWithRemainder(t *testing.T) {
+	store := newTestManifestStore()
+
+	// 7 chunks with batch=3: should produce 2 manifests + 1 remainder
+	chunks := make([]*filer_pb.FileChunk, 7)
+	for i := range chunks {
+		chunks[i] = testChunk(1, uint64(i+1), uint32(i+1), int64(i*100), 100, int64(i+1))
+	}
+
+	result, err := doMaybeManifestize(store.saveFunc(), chunks, 3, mergeIntoManifest)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	manifests := 0
+	regular := 0
+	for _, c := range result {
+		if c.IsChunkManifest {
+			manifests++
+		} else {
+			regular++
+		}
+	}
+	if manifests != 2 {
+		t.Errorf("expected 2 manifests, got %d", manifests)
+	}
+	if regular != 1 {
+		t.Errorf("expected 1 remainder chunk, got %d", regular)
+	}
+
+	// Resolve all and verify we get back 7 data chunks
+	data, _, err := store.resolveAll(result)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(data) != 7 {
+		t.Fatalf("expected 7 data chunks after resolve, got %d", len(data))
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Multiple overlapping manifests: compaction identifies all redundant sub-chunks
+// ---------------------------------------------------------------------------
+
+func TestCompactMultipleOverlappingManifestGenerations(t *testing.T) {
+	store := newTestManifestStore()
+	save := store.saveFunc()
+
+	const fileSize int64 = 3000
+	const chunkSize uint64 = 1000
+
+	// Simulate 5 generations of full-file writes, each manifestized separately.
+	var allManifests []*filer_pb.FileChunk
+	for gen := 0; gen < 5; gen++ {
+		tsBase := int64((gen + 1) * 100)
+		batch := []*filer_pb.FileChunk{
+			testChunk(uint32(gen+1), 1, uint32(gen), 0, chunkSize, tsBase),
+			testChunk(uint32(gen+1), 2, uint32(gen), int64(chunkSize), chunkSize, tsBase+1),
+			testChunk(uint32(gen+1), 3, uint32(gen), int64(chunkSize*2), chunkSize, tsBase+2),
+		}
+		manifest, err := doMaybeManifestize(save, batch, 3, mergeIntoManifest)
+		if err != nil {
+			t.Fatal(err)
+		}
+		allManifests = append(allManifests, manifest...)
+	}
+
+	if len(allManifests) != 5 {
+		t.Fatalf("expected 5 manifest chunks, got %d", len(allManifests))
+	}
+
+	// Resolve all 5 manifests: 15 data chunks total
+	dataChunks, _, err := store.resolveAll(allManifests)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(dataChunks) != 15 {
+		t.Fatalf("expected 15 resolved chunks, got %d", len(dataChunks))
+	}
+
+	// Compact: only generation 5 (the newest 3 chunks) should survive
+	compacted, garbage := CompactFileChunks(context.Background(), nil, dataChunks)
+	if len(compacted) != 3 {
+		t.Errorf("expected 3 compacted (gen 5), got %d", len(compacted))
+	}
+	if len(garbage) != 12 {
+		t.Errorf("expected 12 garbage (gens 1-4), got %d", len(garbage))
+	}
+
+	// Verify the surviving chunks are the newest (highest timestamps)
+	for _, c := range compacted {
+		if c.ModifiedTsNs < 500 {
+			t.Errorf("compacted chunk ts=%d should be from gen 5 (ts >= 500)", c.ModifiedTsNs)
+		}
+	}
+}
+
+// ---------------------------------------------------------------------------
+// Bloat detection: manifest sizes must count toward total
+// ---------------------------------------------------------------------------
+
+func TestManifestBloatDetection(t *testing.T) {
+	// Simulates what shouldMergeChunks sees after multiple flush cycles.
+	// With the fix, manifest sizes count toward totalChunkSize, so bloat
+	// from accumulated manifests is detected.
+
+	const fileSize uint64 = 10000
+
+	// 1 regular chunk covering the file (from latest compaction)
+	regular := []*filer_pb.FileChunk{
+		{Offset: 0, Size: fileSize, FileId: "latest", ModifiedTsNs: 100,
+			Fid: &filer_pb.FileId{VolumeId: 1, FileKey: 1}},
+	}
+
+	// N manifests each covering the full file (from prior flush cycles)
+	for n := 1; n <= 10; n++ {
+		var manifests []*filer_pb.FileChunk
+		for i := 0; i < n; i++ {
+			manifests = append(manifests, &filer_pb.FileChunk{
+				Offset: 0, Size: fileSize, IsChunkManifest: true,
+				Fid: &filer_pb.FileId{VolumeId: 900, FileKey: uint64(i + 1)},
+			})
+		}
+
+		allChunks := append(append([]*filer_pb.FileChunk{}, regular...), manifests...)
+		var totalStored uint64
+		for _, c := range allChunks {
+			totalStored += c.Size
+		}
+		totalFile := TotalSize(allChunks)
+
+		expectMerge := totalStored > 2*totalFile
+		t.Logf("n=%d manifests: totalStored=%d fileSize=%d ratio=%.1fx expectMerge=%v",
+			n, totalStored, totalFile, float64(totalStored)/float64(totalFile), expectMerge)
+
+		// With 1 regular (10000) + N manifests (N*10000):
+		// total = (N+1)*10000, fileSize=10000, ratio = N+1
+		// Merge when N+1 > 2, i.e., N >= 2
+		if n >= 2 && !expectMerge {
+			t.Errorf("n=%d: should trigger merge at ratio %.1f", n, float64(totalStored)/float64(totalFile))
+		}
+		if n < 2 && expectMerge {
+			t.Errorf("n=%d: should NOT trigger merge at ratio %.1f", n, float64(totalStored)/float64(totalFile))
+		}
+	}
+}
+
+// countingInvalidator stands in for the vidMap cache a mount or filer hands to
+// manifest resolution, recording how often the locations were dropped.
+type countingInvalidator struct {
+	invalidations atomic.Int32
+}
+
+func (inv *countingInvalidator) InvalidateCache(fileId string) {
+	inv.invalidations.Add(1)
+}
+
+// stagedLookup returns staleUrls once and freshUrls afterwards, the way a
+// vidMapClient re-reads the master once InvalidateCache has dropped the entry.
+type stagedLookup struct {
+	staleUrls []string
+	freshUrls []string
+	calls     atomic.Int32
+}
+
+func (l *stagedLookup) lookup(ctx context.Context, fileId string) ([]string, error) {
+	if l.calls.Add(1) == 1 {
+		return l.staleUrls, nil
+	}
+	return l.freshUrls, nil
+}
+
+func manifestServer(t *testing.T, manifestBytes []byte) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Length", strconv.Itoa(len(manifestBytes)))
+		w.Write(manifestBytes)
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func fetchManifestBuffer(t *testing.T) *bytes.Buffer {
+	t.Helper()
+	bytesBuffer := bytesBufferPool.Get().(*bytes.Buffer)
+	bytesBuffer.Reset()
+	t.Cleanup(func() { bytesBufferPool.Put(bytesBuffer) })
+	return bytesBuffer
+}
+
+// TestFetchWholeChunkRetriesFreshLocations covers the mount reading a multipart
+// file whose manifest volume has moved: the cached location is dead, so the
+// fetch has to drop it and come back with what the master knows now. The stale
+// server streams a prefix before dying, which the retry must not keep.
+func TestFetchWholeChunkRetriesFreshLocations(t *testing.T) {
+	manifestBytes, err := proto.Marshal(&filer_pb.FileChunkManifest{
+		Chunks: []*filer_pb.FileChunk{
+			{FileId: "100,abc", Offset: 0, Size: 8},
+			{FileId: "101,def", Offset: 8, Size: 8},
+		},
+	})
+	assert.NoError(t, err)
+
+	staleSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Length", strconv.Itoa(len(manifestBytes)+64))
+		w.Write([]byte("garbage-from-stale-server"))
+		w.(http.Flusher).Flush()
+		panic(http.ErrAbortHandler)
+	}))
+	defer staleSrv.Close()
+
+	lookup := &stagedLookup{
+		staleUrls: []string{staleSrv.URL + "/5,stale"},
+		freshUrls: []string{manifestServer(t, manifestBytes).URL + "/5,stale"},
+	}
+	inv := &countingInvalidator{}
+	bytesBuffer := fetchManifestBuffer(t)
+
+	assert.NoError(t, fetchWholeChunk(context.Background(), bytesBuffer, lookup.lookup, "5,stale", nil, false, inv))
+	assert.Equal(t, int32(1), inv.invalidations.Load())
+	assert.Equal(t, int32(2), lookup.calls.Load())
+
+	// the buffer must hold the fresh manifest alone, not the stale prefix ahead of it
+	decoded := &filer_pb.FileChunkManifest{}
+	assert.NoError(t, proto.Unmarshal(bytesBuffer.Bytes(), decoded))
+	assertEqualChunks(t, []*filer_pb.FileChunk{
+		{FileId: "100,abc", Offset: 0, Size: 8},
+		{FileId: "101,def", Offset: 8, Size: 8},
+	}, decoded.Chunks)
+}
+
+// TestFetchWholeChunkRefreshesLocationsAfterPartialFailure is the manifest read
+// whose cached list still names a dead replica ahead of a live one: the read
+// succeeds, and the cached entry must still be dropped and looked up again.
+func TestFetchWholeChunkRefreshesLocationsAfterPartialFailure(t *testing.T) {
+	manifestBytes, err := proto.Marshal(&filer_pb.FileChunkManifest{
+		Chunks: []*filer_pb.FileChunk{{FileId: "100,abc", Offset: 0, Size: 8}},
+	})
+	assert.NoError(t, err)
+
+	liveURL := manifestServer(t, manifestBytes).URL + "/5,abc"
+	dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	deadURL := dead.URL
+	dead.Close()
+
+	lookup := &stagedLookup{
+		staleUrls: []string{deadURL + "/5,abc", liveURL},
+		freshUrls: []string{liveURL},
+	}
+	inv := &countingInvalidator{}
+	bytesBuffer := fetchManifestBuffer(t)
+
+	assert.NoError(t, fetchWholeChunk(context.Background(), bytesBuffer, lookup.lookup, "5,abc", nil, false, inv))
+	assert.Equal(t, int32(1), inv.invalidations.Load())
+	assert.Equal(t, int32(2), lookup.calls.Load())
+	decoded := &filer_pb.FileChunkManifest{}
+	assert.NoError(t, proto.Unmarshal(bytesBuffer.Bytes(), decoded))
+	assertEqualChunks(t, []*filer_pb.FileChunk{{FileId: "100,abc", Offset: 0, Size: 8}}, decoded.Chunks)
+}
+
+// TestFetchWholeChunkWithoutInvalidator keeps the old behavior for callers whose
+// lookup function has no cache to drop: the fetch error surfaces as it is.
+func TestFetchWholeChunkWithoutInvalidator(t *testing.T) {
+	failSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer failSrv.Close()
+
+	lookup := &stagedLookup{
+		staleUrls: []string{failSrv.URL + "/5,abc"},
+		freshUrls: []string{"http://unused:8080/5,abc"},
+	}
+
+	assert.Error(t, fetchWholeChunk(context.Background(), fetchManifestBuffer(t), lookup.lookup, "5,abc", nil, false, nil))
+	assert.Equal(t, int32(1), lookup.calls.Load())
+}
+
+// TestFetchWholeChunkUnchangedLocations guards against looping: when the master
+// still reports the servers that just failed, there is nothing new to try.
+func TestFetchWholeChunkUnchangedLocations(t *testing.T) {
+	failSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+	}))
+	defer failSrv.Close()
+
+	lookup := &stagedLookup{
+		staleUrls: []string{failSrv.URL + "/5,abc"},
+		freshUrls: []string{failSrv.URL + "/5,abc"},
+	}
+	inv := &countingInvalidator{}
+
+	assert.Error(t, fetchWholeChunk(context.Background(), fetchManifestBuffer(t), lookup.lookup, "5,abc", nil, false, inv))
+	assert.Equal(t, int32(2), lookup.calls.Load())
+	assert.Equal(t, int32(1), inv.invalidations.Load())
+}
+
+// TestFetchWholeChunkCancelledKeepsLocations checks that a caller walking away
+// mid-read does not cost every other reader a master round trip.
+func TestFetchWholeChunkCancelledKeepsLocations(t *testing.T) {
+	lookup := &stagedLookup{
+		staleUrls: []string{"http://unused:8080/5,abc"},
+		freshUrls: []string{"http://elsewhere:8080/5,abc"},
+	}
+	inv := &countingInvalidator{}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	err := fetchWholeChunk(ctx, fetchManifestBuffer(t), lookup.lookup, "5,abc", nil, false, inv)
+	assert.ErrorIs(t, err, context.Canceled)
+	assert.Equal(t, int32(0), inv.invalidations.Load())
+	assert.Equal(t, int32(1), lookup.calls.Load())
+
+	// the cancellation must survive the wrapping ResolveOneChunkManifest does,
+	// or callers cannot tell a dead volume server from their own abort
+	manifestChunk := &filer_pb.FileChunk{FileId: "5,abc", IsChunkManifest: true}
+	_, resolveErr := ResolveOneChunkManifest(ctx, lookup.lookup, manifestChunk, inv)
+	assert.ErrorIs(t, resolveErr, context.Canceled)
+
+	// volume.fsck resolves manifests with no invalidator and still has to tell
+	// its own abort from a corrupt manifest, so the nil path keeps the identity
+	noInvalidator := retryFetchWithFreshLocations(ctx, nil, lookup.lookup, "5,abc", nil, fmt.Errorf("stale server said no"), func([]string) error {
+		t.Fatal("refetch must not run on a cancelled read")
+		return nil
+	})
+	assert.ErrorIs(t, noInvalidator, context.Canceled)
+}

weed/filer/filechunk_section.go

@@ -0,0 +1,158 @@
+package filer
+
+import (
+	"context"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+const SectionSize = 2 * 1024 * 1024 * 32 // 64MiB
+type SectionIndex int64
+type FileChunkSection struct {
+	sectionIndex     SectionIndex
+	chunks           []*filer_pb.FileChunk
+	visibleIntervals *IntervalList[*VisibleInterval]
+	chunkViews       *IntervalList[*ChunkView]
+	reader           *ChunkReadAt
+	lock             sync.RWMutex
+	isPrepared       bool
+}
+
+func sectionBounds(sectionIndex SectionIndex, fileSize int64) (start, stop int64) {
+	start = int64(sectionIndex) * SectionSize
+	stop = fileSize
+	if sectionIndex < SectionIndex(fileSize/SectionSize) {
+		stop = start + SectionSize
+	}
+	return
+}
+
+func NewFileChunkSection(si SectionIndex) *FileChunkSection {
+	return &FileChunkSection{
+		sectionIndex: si,
+	}
+}
+
+func (section *FileChunkSection) addChunk(chunk *filer_pb.FileChunk) error {
+	section.lock.Lock()
+	defer section.lock.Unlock()
+
+	start, stop := max(int64(section.sectionIndex)*SectionSize, chunk.Offset), min(((int64(section.sectionIndex)+1)*SectionSize), chunk.Offset+int64(chunk.Size))
+
+	section.chunks = append(section.chunks, chunk)
+
+	if section.visibleIntervals == nil {
+		section.visibleIntervals = readResolvedChunks(section.chunks, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
+	} else {
+		MergeIntoVisibles(section.visibleIntervals, start, stop, chunk)
+		garbageFileIds := FindGarbageChunks(section.visibleIntervals, start, stop)
+		removeGarbageChunks(section, garbageFileIds)
+	}
+
+	if section.chunkViews != nil {
+		MergeIntoChunkViews(section.chunkViews, start, stop, chunk)
+	}
+
+	return nil
+}
+
+func removeGarbageChunks(section *FileChunkSection, garbageFileIds map[string]struct{}) {
+	for i := 0; i < len(section.chunks); {
+		t := section.chunks[i]
+		length := len(section.chunks)
+		if _, found := garbageFileIds[t.FileId]; found {
+			if i < length-1 {
+				section.chunks[i] = section.chunks[length-1]
+			}
+			section.chunks = section.chunks[:length-1]
+		} else {
+			i++
+		}
+	}
+}
+
+func (section *FileChunkSection) setupForRead(ctx context.Context, group *ChunkGroup, fileSize int64) {
+	section.lock.Lock()
+	defer section.lock.Unlock()
+
+	sectionStart, sectionStop := sectionBounds(section.sectionIndex, fileSize)
+
+	if section.isPrepared {
+		section.reader.fileSize = fileSize
+		return
+	}
+
+	if section.visibleIntervals == nil {
+		section.visibleIntervals = readResolvedChunks(section.chunks, sectionStart, sectionStop)
+		section.chunks, _ = SeparateGarbageChunks(section.visibleIntervals, section.chunks)
+		if section.reader != nil {
+			_ = section.reader.Close()
+			section.reader = nil
+		}
+	}
+	if section.chunkViews == nil {
+		section.chunkViews = ViewFromVisibleIntervals(section.visibleIntervals, sectionStart, sectionStop)
+	}
+
+	if section.reader == nil {
+		section.reader = NewChunkReaderAtFromClient(ctx, group.readerCache, section.chunkViews, sectionStop, group.GetPrefetchCount())
+	}
+
+	section.isPrepared = true
+	section.reader.fileSize = fileSize
+}
+
+func (section *FileChunkSection) readDataAt(ctx context.Context, group *ChunkGroup, fileSize int64, buff []byte, offset int64) (n int, tsNs int64, err error) {
+
+	section.setupForRead(ctx, group, fileSize)
+	section.lock.RLock()
+	defer section.lock.RUnlock()
+
+	return section.reader.ReadAtWithTime(ctx, buff, offset)
+}
+
+func (section *FileChunkSection) DataStartOffset(ctx context.Context, group *ChunkGroup, offset int64, fileSize int64) int64 {
+
+	section.setupForRead(ctx, group, fileSize)
+	section.lock.RLock()
+	defer section.lock.RUnlock()
+
+	for x := section.visibleIntervals.Front(); x != nil; x = x.Next {
+		visible := x.Value
+		if visible.stop <= offset {
+			continue
+		}
+		if offset < visible.start {
+			return visible.start
+		}
+		return offset
+	}
+	return -1
+}
+
+func (section *FileChunkSection) NextStopOffset(ctx context.Context, group *ChunkGroup, offset int64, fileSize int64) int64 {
+
+	section.setupForRead(ctx, group, fileSize)
+	section.lock.RLock()
+	defer section.lock.RUnlock()
+
+	isAfterOffset := false
+	for x := section.visibleIntervals.Front(); x != nil; x = x.Next {
+		visible := x.Value
+		if !isAfterOffset {
+			if visible.stop <= offset {
+				continue
+			}
+			isAfterOffset = true
+		}
+		if offset < visible.start {
+			return offset
+		}
+		// now visible.start <= offset
+		if offset < visible.stop {
+			offset = visible.stop
+		}
+	}
+	return offset
+}

weed/filer/filechunk_section_test.go

@@ -0,0 +1,49 @@
+package filer
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func Test_removeGarbageChunks(t *testing.T) {
+	section := NewFileChunkSection(0)
+	section.chunks = append(section.chunks, &filer_pb.FileChunk{
+		FileId:       "0",
+		Offset:       0,
+		Size:         1,
+		ModifiedTsNs: 0,
+	})
+	section.chunks = append(section.chunks, &filer_pb.FileChunk{
+		FileId:       "1",
+		Offset:       1,
+		Size:         1,
+		ModifiedTsNs: 1,
+	})
+	section.chunks = append(section.chunks, &filer_pb.FileChunk{
+		FileId:       "2",
+		Offset:       2,
+		Size:         1,
+		ModifiedTsNs: 2,
+	})
+	section.chunks = append(section.chunks, &filer_pb.FileChunk{
+		FileId:       "3",
+		Offset:       3,
+		Size:         1,
+		ModifiedTsNs: 3,
+	})
+	section.chunks = append(section.chunks, &filer_pb.FileChunk{
+		FileId:       "4",
+		Offset:       4,
+		Size:         1,
+		ModifiedTsNs: 4,
+	})
+	garbageFileIds := make(map[string]struct{})
+	garbageFileIds["0"] = struct{}{}
+	garbageFileIds["2"] = struct{}{}
+	garbageFileIds["4"] = struct{}{}
+	removeGarbageChunks(section, garbageFileIds)
+	if len(section.chunks) != 2 {
+		t.Errorf("remove chunk 2 failed")
+	}
+}

weed/filer/filechunks.go

@@ -0,0 +1,326 @@
+package filer
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"math"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+)
+
+func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
+	for _, c := range chunks {
+		t := uint64(c.Offset + int64(c.Size))
+		if size < t {
+			size = t
+		}
+	}
+	return
+}
+
+func FileSize(entry *filer_pb.Entry) (size uint64) {
+	if entry == nil || entry.Attributes == nil {
+		return 0
+	}
+	fileSize := entry.Attributes.FileSize
+	if entry.RemoteEntry != nil {
+		if entry.RemoteEntry.RemoteMtime > entry.Attributes.Mtime {
+			fileSize = maxUint64(fileSize, uint64(entry.RemoteEntry.RemoteSize))
+		}
+	}
+	return maxUint64(TotalSize(entry.GetChunks()), fileSize)
+}
+
+func ETag(entry *filer_pb.Entry) (etag string) {
+	if entry.Attributes == nil || entry.Attributes.Md5 == nil {
+		return ETagChunks(entry.GetChunks())
+	}
+	return fmt.Sprintf("%x", entry.Attributes.Md5)
+}
+
+func ETagEntry(entry *Entry) (etag string) {
+	if entry.IsInRemoteOnly() {
+		return entry.Remote.RemoteETag
+	}
+	if entry.Attr.Md5 == nil {
+		return ETagChunks(entry.GetChunks())
+	}
+	return fmt.Sprintf("%x", entry.Attr.Md5)
+}
+
+func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) {
+	if len(chunks) == 1 {
+		return fmt.Sprintf("%x", util.Base64Md5ToBytes(chunks[0].ETag))
+	}
+	var md5Digests [][]byte
+	for _, c := range chunks {
+		md5Digests = append(md5Digests, util.Base64Md5ToBytes(c.ETag))
+	}
+	finalETag := fmt.Sprintf("%x-%d", util.Md5(bytes.Join(md5Digests, nil)), len(chunks))
+	return finalETag
+}
+
+func CompactFileChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {
+
+	visibles, _ := NonOverlappingVisibleIntervals(ctx, lookupFileIdFn, chunks, 0, math.MaxInt64)
+
+	compacted, garbage = SeparateGarbageChunks(visibles, chunks)
+
+	return
+}
+
+func SeparateGarbageChunks(visibles *IntervalList[*VisibleInterval], chunks []*filer_pb.FileChunk) (compacted []*filer_pb.FileChunk, garbage []*filer_pb.FileChunk) {
+	fileIds := make(map[string]bool)
+	for x := visibles.Front(); x != nil; x = x.Next {
+		interval := x.Value
+		fileIds[interval.fileId] = true
+	}
+	for _, chunk := range chunks {
+		if _, found := fileIds[chunk.GetFileIdString()]; found {
+			compacted = append(compacted, chunk)
+		} else {
+			garbage = append(garbage, chunk)
+		}
+	}
+	return compacted, garbage
+}
+
+func FindGarbageChunks(visibles *IntervalList[*VisibleInterval], start int64, stop int64) (garbageFileIds map[string]struct{}) {
+	garbageFileIds = make(map[string]struct{})
+	for x := visibles.Front(); x != nil; x = x.Next {
+		interval := x.Value
+		offset := interval.start - interval.offsetInChunk
+		if start <= offset && offset+int64(interval.chunkSize) <= stop {
+			garbageFileIds[interval.fileId] = struct{}{}
+		}
+	}
+	return
+}
+
+func MinusChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, as, bs []*filer_pb.FileChunk, invalidator CacheInvalidator) (delta []*filer_pb.FileChunk, err error) {
+
+	aData, aMeta, aErr := ResolveChunkManifest(ctx, lookupFileIdFn, as, 0, math.MaxInt64, invalidator)
+	if aErr != nil {
+		return nil, aErr
+	}
+	bData, bMeta, bErr := ResolveChunkManifest(ctx, lookupFileIdFn, bs, 0, math.MaxInt64, invalidator)
+	if bErr != nil {
+		return nil, bErr
+	}
+
+	delta = append(delta, DoMinusChunks(aData, bData)...)
+	delta = append(delta, DoMinusChunks(aMeta, bMeta)...)
+	return
+}
+
+func DoMinusChunks(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) {
+
+	fileIds := make(map[string]bool)
+	for _, interval := range bs {
+		fileIds[interval.GetFileIdString()] = true
+	}
+	for _, chunk := range as {
+		if _, found := fileIds[chunk.GetFileIdString()]; !found {
+			delta = append(delta, chunk)
+		}
+	}
+
+	return
+}
+
+func DoMinusChunksBySourceFileId(as, bs []*filer_pb.FileChunk) (delta []*filer_pb.FileChunk) {
+
+	fileIds := make(map[string]bool)
+	for _, interval := range bs {
+		fileIds[interval.GetFileIdString()] = true
+		fileIds[interval.GetSourceFileId()] = true
+	}
+	for _, chunk := range as {
+		_, sourceFileIdFound := fileIds[chunk.GetSourceFileId()]
+		_, fileIdFound := fileIds[chunk.GetFileId()]
+		if !sourceFileIdFound && !fileIdFound {
+			delta = append(delta, chunk)
+		}
+	}
+
+	return
+}
+
+type ChunkView struct {
+	FileId        string
+	OffsetInChunk int64 // offset within the chunk
+	ViewSize      uint64
+	ViewOffset    int64 // actual offset in the file, for the data specified via [offset, offset+size) in current chunk
+	ChunkSize     uint64
+	CipherKey     []byte
+	IsGzipped     bool
+	ModifiedTsNs  int64
+}
+
+func (cv *ChunkView) SetStartStop(start, stop int64) {
+	cv.OffsetInChunk += start - cv.ViewOffset
+	cv.ViewOffset = start
+	cv.ViewSize = uint64(stop - start)
+}
+func (cv *ChunkView) Clone() IntervalValue {
+	return &ChunkView{
+		FileId:        cv.FileId,
+		OffsetInChunk: cv.OffsetInChunk,
+		ViewSize:      cv.ViewSize,
+		ViewOffset:    cv.ViewOffset,
+		ChunkSize:     cv.ChunkSize,
+		CipherKey:     cv.CipherKey,
+		IsGzipped:     cv.IsGzipped,
+		ModifiedTsNs:  cv.ModifiedTsNs,
+	}
+}
+
+func (cv *ChunkView) IsFullChunk() bool {
+	// IsFullChunk returns true if the view covers the entire chunk from the beginning.
+	// This prevents bandwidth amplification when range requests happen to align
+	// with chunk boundaries but don't actually want the full chunk.
+	return cv.OffsetInChunk == 0 && cv.ViewSize == cv.ChunkSize
+}
+
+func ViewFromChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, offset int64, size int64) (chunkViews *IntervalList[*ChunkView]) {
+
+	visibles, _ := NonOverlappingVisibleIntervals(ctx, lookupFileIdFn, chunks, offset, offset+size)
+
+	return ViewFromVisibleIntervals(visibles, offset, size)
+
+}
+
+func ViewFromVisibleIntervals(visibles *IntervalList[*VisibleInterval], offset int64, size int64) (chunkViews *IntervalList[*ChunkView]) {
+
+	stop := offset + size
+	if size == math.MaxInt64 {
+		stop = math.MaxInt64
+	}
+	if stop < offset {
+		stop = math.MaxInt64
+	}
+
+	chunkViews = NewIntervalList[*ChunkView]()
+	for x := visibles.Front(); x != nil; x = x.Next {
+		chunk := x.Value
+
+		chunkStart, chunkStop := max(offset, chunk.start), min(stop, chunk.stop)
+
+		if chunkStart < chunkStop {
+			chunkView := &ChunkView{
+				FileId:        chunk.fileId,
+				OffsetInChunk: chunkStart - chunk.start + chunk.offsetInChunk,
+				ViewSize:      uint64(chunkStop - chunkStart),
+				ViewOffset:    chunkStart,
+				ChunkSize:     chunk.chunkSize,
+				CipherKey:     chunk.cipherKey,
+				IsGzipped:     chunk.isGzipped,
+				ModifiedTsNs:  chunk.modifiedTsNs,
+			}
+			chunkViews.AppendInterval(&Interval[*ChunkView]{
+				StartOffset: chunkStart,
+				StopOffset:  chunkStop,
+				TsNs:        chunk.modifiedTsNs,
+				Value:       chunkView,
+				Prev:        nil,
+				Next:        nil,
+			})
+		}
+	}
+
+	return chunkViews
+
+}
+
+func MergeIntoVisibles(visibles *IntervalList[*VisibleInterval], start int64, stop int64, chunk *filer_pb.FileChunk) {
+
+	newV := &VisibleInterval{
+		start:         start,
+		stop:          stop,
+		fileId:        chunk.GetFileIdString(),
+		modifiedTsNs:  chunk.ModifiedTsNs,
+		offsetInChunk: start - chunk.Offset, // the starting position in the chunk
+		chunkSize:     chunk.Size,           // size of the chunk
+		cipherKey:     chunk.CipherKey,
+		isGzipped:     chunk.IsCompressed,
+	}
+
+	visibles.InsertInterval(start, stop, chunk.ModifiedTsNs, newV)
+}
+
+func MergeIntoChunkViews(chunkViews *IntervalList[*ChunkView], start int64, stop int64, chunk *filer_pb.FileChunk) {
+
+	chunkView := &ChunkView{
+		FileId:        chunk.GetFileIdString(),
+		OffsetInChunk: start - chunk.Offset,
+		ViewSize:      uint64(stop - start),
+		ViewOffset:    start,
+		ChunkSize:     chunk.Size,
+		CipherKey:     chunk.CipherKey,
+		IsGzipped:     chunk.IsCompressed,
+		ModifiedTsNs:  chunk.ModifiedTsNs,
+	}
+
+	chunkViews.InsertInterval(start, stop, chunk.ModifiedTsNs, chunkView)
+}
+
+// NonOverlappingVisibleIntervals translates the file chunk into VisibleInterval in memory
+// If the file chunk content is a chunk manifest
+func NonOverlappingVisibleIntervals(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles *IntervalList[*VisibleInterval], err error) {
+
+	chunks, _, err = ResolveChunkManifest(ctx, lookupFileIdFn, chunks, startOffset, stopOffset, nil)
+	if err != nil {
+		return
+	}
+
+	visibles2 := readResolvedChunks(chunks, 0, math.MaxInt64)
+
+	return visibles2, err
+}
+
+// find non-overlapping visible intervals
+// visible interval map to one file chunk
+
+type VisibleInterval struct {
+	start         int64
+	stop          int64
+	modifiedTsNs  int64
+	fileId        string
+	offsetInChunk int64
+	chunkSize     uint64
+	cipherKey     []byte
+	isGzipped     bool
+}
+
+func (v *VisibleInterval) SetStartStop(start, stop int64) {
+	v.offsetInChunk += start - v.start
+	v.start, v.stop = start, stop
+}
+func (v *VisibleInterval) Clone() IntervalValue {
+	return &VisibleInterval{
+		start:         v.start,
+		stop:          v.stop,
+		modifiedTsNs:  v.modifiedTsNs,
+		fileId:        v.fileId,
+		offsetInChunk: v.offsetInChunk,
+		chunkSize:     v.chunkSize,
+		cipherKey:     v.cipherKey,
+		isGzipped:     v.isGzipped,
+	}
+}
+
+func min(x, y int64) int64 {
+	if x <= y {
+		return x
+	}
+	return y
+}
+func max(x, y int64) int64 {
+	if x <= y {
+		return y
+	}
+	return x
+}

weed/filer/filechunks2_test.go

@@ -0,0 +1,86 @@
+package filer
+
+import (
+	"context"
+	"log"
+	"slices"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDoMinusChunks(t *testing.T) {
+	// https://github.com/seaweedfs/seaweedfs/issues/3328
+
+	// clusterA and clusterB using filer.sync to sync file: hello.txt
+	// clusterA append a new line and then clusterB also append a new line
+	// clusterA append a new line again
+	chunksInA := []*filer_pb.FileChunk{
+		{Offset: 0, Size: 3, FileId: "11", ModifiedTsNs: 100},
+		{Offset: 3, Size: 3, FileId: "22", SourceFileId: "2", ModifiedTsNs: 200},
+		{Offset: 6, Size: 3, FileId: "33", ModifiedTsNs: 300},
+	}
+	chunksInB := []*filer_pb.FileChunk{
+		{Offset: 0, Size: 3, FileId: "1", SourceFileId: "11", ModifiedTsNs: 100},
+		{Offset: 3, Size: 3, FileId: "2", ModifiedTsNs: 200},
+		{Offset: 6, Size: 3, FileId: "3", SourceFileId: "33", ModifiedTsNs: 300},
+	}
+
+	// clusterB using command "echo 'content' > hello.txt" to overwrite file
+	// clusterA will receive two evenNotification, need to empty the whole file content first and add new content
+	// the first one is oldEntry is chunksInB and newEntry is empty fileChunks
+	firstOldEntry := chunksInB
+	var firstNewEntry []*filer_pb.FileChunk
+
+	// clusterA received the first one event, gonna empty the whole chunk, according the code in filer_sink 194
+	// we can get the deleted chunks and newChunks
+	firstDeletedChunks := DoMinusChunks(firstOldEntry, firstNewEntry)
+	log.Println("first deleted chunks:", firstDeletedChunks)
+	//firstNewEntry := DoMinusChunks(firstNewEntry, firstOldEntry)
+
+	// clusterA need to delete all chunks in firstDeletedChunks
+	emptiedChunksInA := DoMinusChunksBySourceFileId(chunksInA, firstDeletedChunks)
+	// chunksInA supposed to be empty by minus the deletedChunks but it just delete the chunk which sync from clusterB
+	log.Println("clusterA synced empty chunks event result:", emptiedChunksInA)
+	// clusterB emptied it's chunks and clusterA must sync the change and empty chunks too
+	assert.Equalf(t, firstNewEntry, emptiedChunksInA, "empty")
+}
+
+func TestCompactFileChunksRealCase(t *testing.T) {
+
+	chunks := []*filer_pb.FileChunk{
+		{FileId: "2,512f31f2c0700a", Offset: 0, Size: 25 - 0, ModifiedTsNs: 5320497},
+		{FileId: "6,512f2c2e24e9e8", Offset: 868352, Size: 917585 - 868352, ModifiedTsNs: 5320492},
+		{FileId: "7,514468dd5954ca", Offset: 884736, Size: 901120 - 884736, ModifiedTsNs: 5325928},
+		{FileId: "5,5144463173fe77", Offset: 917504, Size: 2297856 - 917504, ModifiedTsNs: 5325894},
+		{FileId: "4,51444c7ab54e2d", Offset: 2301952, Size: 2367488 - 2301952, ModifiedTsNs: 5325900},
+		{FileId: "4,514450e643ad22", Offset: 2371584, Size: 2420736 - 2371584, ModifiedTsNs: 5325904},
+		{FileId: "6,514456a5e9e4d7", Offset: 2449408, Size: 2490368 - 2449408, ModifiedTsNs: 5325910},
+		{FileId: "3,51444f8d53eebe", Offset: 2494464, Size: 2555904 - 2494464, ModifiedTsNs: 5325903},
+		{FileId: "4,5144578b097c7e", Offset: 2560000, Size: 2596864 - 2560000, ModifiedTsNs: 5325911},
+		{FileId: "3,51445500b6b4ac", Offset: 2637824, Size: 2678784 - 2637824, ModifiedTsNs: 5325909},
+		{FileId: "1,51446285e52a61", Offset: 2695168, Size: 2715648 - 2695168, ModifiedTsNs: 5325922},
+	}
+
+	printChunks("before", chunks)
+
+	compacted, garbage := CompactFileChunks(context.Background(), nil, chunks)
+
+	printChunks("compacted", compacted)
+	printChunks("garbage", garbage)
+
+}
+
+func printChunks(name string, chunks []*filer_pb.FileChunk) {
+	slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) int {
+		if a.Offset == b.Offset {
+			return int(a.ModifiedTsNs - b.ModifiedTsNs)
+		}
+		return int(a.Offset - b.Offset)
+	})
+	for _, chunk := range chunks {
+		glog.V(0).Infof("%s chunk %s [%10d,%10d)", name, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))
+	}
+}

weed/filer/filechunks_read.go

@@ -0,0 +1,129 @@
+package filer
+
+import (
+	"container/list"
+	"slices"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles *IntervalList[*VisibleInterval]) {
+
+	var points []*Point
+	for _, chunk := range chunks {
+		if chunk.IsChunkManifest {
+			println("This should not happen! A manifest chunk found:", chunk.GetFileIdString())
+		}
+		start, stop := max(chunk.Offset, startOffset), min(chunk.Offset+int64(chunk.Size), stopOffset)
+		if start >= stop {
+			continue
+		}
+		points = append(points, &Point{
+			x:       chunk.Offset,
+			ts:      chunk.ModifiedTsNs,
+			chunk:   chunk,
+			isStart: true,
+		})
+		points = append(points, &Point{
+			x:       chunk.Offset + int64(chunk.Size),
+			ts:      chunk.ModifiedTsNs,
+			chunk:   chunk,
+			isStart: false,
+		})
+	}
+	slices.SortFunc(points, func(a, b *Point) int {
+		if a.x != b.x {
+			return int(a.x - b.x)
+		}
+		if a.ts != b.ts {
+			return int(a.ts - b.ts)
+		}
+		if a.isStart {
+			return 1
+		}
+		if b.isStart {
+			return -1
+		}
+		return 0
+	})
+
+	var prevX int64
+	queue := list.New() // points with higher ts are at the tail
+	visibles = NewIntervalList[*VisibleInterval]()
+	var prevPoint *Point
+	for _, point := range points {
+		if queue.Len() > 0 {
+			prevPoint = queue.Back().Value.(*Point)
+		} else {
+			prevPoint = nil
+		}
+		if point.isStart {
+			if prevPoint != nil {
+				if point.x != prevX && prevPoint.ts < point.ts {
+					addToVisibles(visibles, prevX, prevPoint, point)
+					prevX = point.x
+				}
+			}
+			// insert into queue
+			if prevPoint == nil || prevPoint.ts < point.ts {
+				queue.PushBack(point)
+				prevX = point.x
+			} else {
+				for e := queue.Front(); e != nil; e = e.Next() {
+					if e.Value.(*Point).ts > point.ts {
+						queue.InsertBefore(point, e)
+						break
+					}
+				}
+			}
+		} else {
+			isLast := true
+			for e := queue.Back(); e != nil; e = e.Prev() {
+				if e.Value.(*Point).ts == point.ts {
+					queue.Remove(e)
+					break
+				}
+				isLast = false
+			}
+			if isLast && prevPoint != nil {
+				addToVisibles(visibles, prevX, prevPoint, point)
+				prevX = point.x
+			}
+		}
+	}
+
+	return
+}
+
+func addToVisibles(visibles *IntervalList[*VisibleInterval], prevX int64, startPoint *Point, point *Point) {
+	if prevX < point.x {
+		chunk := startPoint.chunk
+		visible := &VisibleInterval{
+			start:         prevX,
+			stop:          point.x,
+			fileId:        chunk.GetFileIdString(),
+			modifiedTsNs:  chunk.ModifiedTsNs,
+			offsetInChunk: prevX - chunk.Offset,
+			chunkSize:     chunk.Size,
+			cipherKey:     chunk.CipherKey,
+			isGzipped:     chunk.IsCompressed,
+		}
+		appendVisibleInterfal(visibles, visible)
+	}
+}
+
+func appendVisibleInterfal(visibles *IntervalList[*VisibleInterval], visible *VisibleInterval) {
+	visibles.AppendInterval(&Interval[*VisibleInterval]{
+		StartOffset: visible.start,
+		StopOffset:  visible.stop,
+		TsNs:        visible.modifiedTsNs,
+		Value:       visible,
+	})
+}
+
+type Point struct {
+	x       int64
+	ts      int64
+	chunk   *filer_pb.FileChunk
+	isStart bool
+}

weed/filer/filechunks_read_test.go

@@ -0,0 +1,281 @@
+package filer
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestReadResolvedChunks(t *testing.T) {
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId:       "a",
+			Offset:       0,
+			Size:         100,
+			ModifiedTsNs: 1,
+		},
+		{
+			FileId:       "b",
+			Offset:       50,
+			Size:         100,
+			ModifiedTsNs: 2,
+		},
+		{
+			FileId:       "c",
+			Offset:       200,
+			Size:         50,
+			ModifiedTsNs: 3,
+		},
+		{
+			FileId:       "d",
+			Offset:       250,
+			Size:         50,
+			ModifiedTsNs: 4,
+		},
+		{
+			FileId:       "e",
+			Offset:       175,
+			Size:         100,
+			ModifiedTsNs: 5,
+		},
+	}
+
+	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
+
+	fmt.Printf("resolved to %d visible intervales\n", visibles.Len())
+	for x := visibles.Front(); x != nil; x = x.Next {
+		visible := x.Value
+		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
+	}
+
+}
+
+func TestReadResolvedChunks2(t *testing.T) {
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId:       "c",
+			Offset:       200,
+			Size:         50,
+			ModifiedTsNs: 3,
+		},
+		{
+			FileId:       "e",
+			Offset:       200,
+			Size:         25,
+			ModifiedTsNs: 5,
+		},
+	}
+
+	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
+
+	fmt.Printf("resolved to %d visible intervales\n", visibles.Len())
+	for x := visibles.Front(); x != nil; x = x.Next {
+		visible := x.Value
+		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
+	}
+
+}
+
+func TestRandomizedReadResolvedChunks(t *testing.T) {
+
+	var limit int64 = 1024 * 1024
+	array := make([]int64, limit)
+	var chunks []*filer_pb.FileChunk
+	for ts := int64(0); ts < 1024; ts++ {
+		x := rand.Int63n(limit)
+		y := rand.Int63n(limit)
+		size := x - y
+		if size < 0 {
+			size = -size
+		}
+		if size > 1024 {
+			size = 1024
+		}
+		start := x
+		if start > y {
+			start = y
+		}
+		chunks = append(chunks, randomWrite(array, start, size, ts))
+	}
+
+	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
+
+	for x := visibles.Front(); x != nil; x = x.Next {
+		visible := x.Value
+		for i := visible.start; i < visible.stop; i++ {
+			if array[i] != visible.modifiedTsNs {
+				t.Errorf("position %d expected ts %d actual ts %d", i, array[i], visible.modifiedTsNs)
+			}
+		}
+	}
+
+	// fmt.Printf("visibles %d", len(visibles))
+
+}
+
+func randomWrite(array []int64, start int64, size int64, ts int64) *filer_pb.FileChunk {
+	for i := start; i < start+size; i++ {
+		array[i] = ts
+	}
+	// fmt.Printf("write [%d,%d) %d\n", start, start+size, ts)
+	return &filer_pb.FileChunk{
+		FileId:       "",
+		Offset:       start,
+		Size:         uint64(size),
+		ModifiedTsNs: ts,
+	}
+}
+
+func TestSequentialReadResolvedChunks(t *testing.T) {
+
+	var chunkSize int64 = 1024 * 1024 * 2
+	var chunks []*filer_pb.FileChunk
+	for ts := int64(0); ts < 13; ts++ {
+		chunks = append(chunks, &filer_pb.FileChunk{
+			FileId:       "",
+			Offset:       chunkSize * ts,
+			Size:         uint64(chunkSize),
+			ModifiedTsNs: 1,
+		})
+	}
+
+	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
+
+	fmt.Printf("visibles %d", visibles.Len())
+
+}
+
+func TestActualReadResolvedChunks(t *testing.T) {
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId:       "5,e7b96fef48",
+			Offset:       0,
+			Size:         2097152,
+			ModifiedTsNs: 1634447487595823000,
+		},
+		{
+			FileId:       "5,e5562640b9",
+			Offset:       2097152,
+			Size:         2097152,
+			ModifiedTsNs: 1634447487595826000,
+		},
+		{
+			FileId:       "5,df033e0fe4",
+			Offset:       4194304,
+			Size:         2097152,
+			ModifiedTsNs: 1634447487595827000,
+		},
+		{
+			FileId:       "7,eb08148a9b",
+			Offset:       6291456,
+			Size:         2097152,
+			ModifiedTsNs: 1634447487595827000,
+		},
+		{
+			FileId:       "7,e0f92d1604",
+			Offset:       8388608,
+			Size:         2097152,
+			ModifiedTsNs: 1634447487595828000,
+		},
+		{
+			FileId:       "7,e33cb63262",
+			Offset:       10485760,
+			Size:         2097152,
+			ModifiedTsNs: 1634447487595828000,
+		},
+		{
+			FileId:       "5,ea98e40e93",
+			Offset:       12582912,
+			Size:         2097152,
+			ModifiedTsNs: 1634447487595829000,
+		},
+		{
+			FileId:       "5,e165661172",
+			Offset:       14680064,
+			Size:         2097152,
+			ModifiedTsNs: 1634447487595829000,
+		},
+		{
+			FileId:       "3,e692097486",
+			Offset:       16777216,
+			Size:         2097152,
+			ModifiedTsNs: 1634447487595830000,
+		},
+		{
+			FileId:       "3,e28e2e3cbd",
+			Offset:       18874368,
+			Size:         2097152,
+			ModifiedTsNs: 1634447487595830000,
+		},
+		{
+			FileId:       "3,e443974d4e",
+			Offset:       20971520,
+			Size:         2097152,
+			ModifiedTsNs: 1634447487595830000,
+		},
+		{
+			FileId:       "2,e815bed597",
+			Offset:       23068672,
+			Size:         2097152,
+			ModifiedTsNs: 1634447487595831000,
+		},
+		{
+			FileId:       "5,e94715199e",
+			Offset:       25165824,
+			Size:         1974736,
+			ModifiedTsNs: 1634447487595832000,
+		},
+	}
+
+	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
+
+	for x := visibles.Front(); x != nil; x = x.Next {
+		visible := x.Value
+		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
+	}
+
+}
+
+func TestActualReadResolvedChunks2(t *testing.T) {
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId:       "1,e7b96fef48",
+			Offset:       0,
+			Size:         184320,
+			ModifiedTsNs: 1,
+		},
+		{
+			FileId:       "2,22562640b9",
+			Offset:       184320,
+			Size:         4096,
+			ModifiedTsNs: 2,
+		},
+		{
+			FileId:       "2,33562640b9",
+			Offset:       184320,
+			Size:         4096,
+			ModifiedTsNs: 4,
+		},
+		{
+			FileId:       "4,df033e0fe4",
+			Offset:       188416,
+			Size:         2097152,
+			ModifiedTsNs: 3,
+		},
+	}
+
+	visibles := readResolvedChunks(chunks, 0, math.MaxInt64)
+
+	for x := visibles.Front(); x != nil; x = x.Next {
+		visible := x.Value
+		fmt.Printf("[%d,%d) %s %d\n", visible.start, visible.stop, visible.fileId, visible.modifiedTsNs)
+	}
+
+}

weed/filer/filechunks_test.go

@@ -0,0 +1,571 @@
+package filer
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"math"
+	"math/rand/v2"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestCompactFileChunks(t *testing.T) {
+	chunks := []*filer_pb.FileChunk{
+		{Offset: 10, Size: 100, FileId: "abc", ModifiedTsNs: 50},
+		{Offset: 100, Size: 100, FileId: "def", ModifiedTsNs: 100},
+		{Offset: 200, Size: 100, FileId: "ghi", ModifiedTsNs: 200},
+		{Offset: 110, Size: 200, FileId: "jkl", ModifiedTsNs: 300},
+	}
+
+	compacted, garbage := CompactFileChunks(context.Background(), nil, chunks)
+
+	if len(compacted) != 3 {
+		t.Fatalf("unexpected compacted: %d", len(compacted))
+	}
+	if len(garbage) != 1 {
+		t.Fatalf("unexpected garbage: %d", len(garbage))
+	}
+
+}
+
+func TestCompactFileChunks2(t *testing.T) {
+
+	chunks := []*filer_pb.FileChunk{
+		{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 50},
+		{Offset: 100, Size: 100, FileId: "def", ModifiedTsNs: 100},
+		{Offset: 200, Size: 100, FileId: "ghi", ModifiedTsNs: 200},
+		{Offset: 0, Size: 100, FileId: "abcf", ModifiedTsNs: 300},
+		{Offset: 50, Size: 100, FileId: "fhfh", ModifiedTsNs: 400},
+		{Offset: 100, Size: 100, FileId: "yuyu", ModifiedTsNs: 500},
+	}
+
+	k := 3
+
+	for n := 0; n < k; n++ {
+		chunks = append(chunks, &filer_pb.FileChunk{
+			Offset: int64(n * 100), Size: 100, FileId: fmt.Sprintf("fileId%d", n), ModifiedTsNs: int64(n),
+		})
+		chunks = append(chunks, &filer_pb.FileChunk{
+			Offset: int64(n * 50), Size: 100, FileId: fmt.Sprintf("fileId%d", n+k), ModifiedTsNs: int64(n + k),
+		})
+	}
+
+	compacted, garbage := CompactFileChunks(context.Background(), nil, chunks)
+
+	if len(compacted) != 4 {
+		t.Fatalf("unexpected compacted: %d", len(compacted))
+	}
+	if len(garbage) != 8 {
+		t.Fatalf("unexpected garbage: %d", len(garbage))
+	}
+}
+
+func TestRandomFileChunksCompact(t *testing.T) {
+
+	data := make([]byte, 1024)
+
+	var chunks []*filer_pb.FileChunk
+	for i := 0; i < 15; i++ {
+		start, stop := rand.IntN(len(data)), rand.IntN(len(data))
+		if start > stop {
+			start, stop = stop, start
+		}
+		if start+16 < stop {
+			stop = start + 16
+		}
+		chunk := &filer_pb.FileChunk{
+			FileId:       strconv.Itoa(i),
+			Offset:       int64(start),
+			Size:         uint64(stop - start),
+			ModifiedTsNs: int64(i),
+			Fid:          &filer_pb.FileId{FileKey: uint64(i)},
+		}
+		chunks = append(chunks, chunk)
+		for x := start; x < stop; x++ {
+			data[x] = byte(i)
+		}
+	}
+
+	visibles, _ := NonOverlappingVisibleIntervals(context.Background(), nil, chunks, 0, math.MaxInt64)
+
+	for visible := visibles.Front(); visible != nil; visible = visible.Next {
+		v := visible.Value
+		for x := v.start; x < v.stop; x++ {
+			assert.Equal(t, strconv.Itoa(int(data[x])), v.fileId)
+		}
+	}
+
+}
+
+func TestIntervalMerging(t *testing.T) {
+
+	testcases := []struct {
+		Chunks   []*filer_pb.FileChunk
+		Expected []*VisibleInterval
+	}{
+		// case 0: normal
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
+				{Offset: 100, Size: 100, FileId: "asdf", ModifiedTsNs: 134},
+				{Offset: 200, Size: 100, FileId: "fsad", ModifiedTsNs: 353},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 100, fileId: "abc"},
+				{start: 100, stop: 200, fileId: "asdf"},
+				{start: 200, stop: 300, fileId: "fsad"},
+			},
+		},
+		// case 1: updates overwrite full chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
+				{Offset: 0, Size: 200, FileId: "asdf", ModifiedTsNs: 134},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 200, fileId: "asdf"},
+			},
+		},
+		// case 2: updates overwrite part of previous chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "a", ModifiedTsNs: 123},
+				{Offset: 0, Size: 70, FileId: "b", ModifiedTsNs: 134},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 70, fileId: "b"},
+				{start: 70, stop: 100, fileId: "a", offsetInChunk: 70},
+			},
+		},
+		// case 3: updates overwrite full chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
+				{Offset: 0, Size: 200, FileId: "asdf", ModifiedTsNs: 134},
+				{Offset: 50, Size: 250, FileId: "xxxx", ModifiedTsNs: 154},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 50, fileId: "asdf"},
+				{start: 50, stop: 300, fileId: "xxxx"},
+			},
+		},
+		// case 4: updates far away from prev chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
+				{Offset: 0, Size: 200, FileId: "asdf", ModifiedTsNs: 134},
+				{Offset: 250, Size: 250, FileId: "xxxx", ModifiedTsNs: 154},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 200, fileId: "asdf"},
+				{start: 250, stop: 500, fileId: "xxxx"},
+			},
+		},
+		// case 5: updates overwrite full chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "a", ModifiedTsNs: 123},
+				{Offset: 0, Size: 200, FileId: "d", ModifiedTsNs: 184},
+				{Offset: 70, Size: 150, FileId: "c", ModifiedTsNs: 143},
+				{Offset: 80, Size: 100, FileId: "b", ModifiedTsNs: 134},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 200, fileId: "d"},
+				{start: 200, stop: 220, fileId: "c", offsetInChunk: 130},
+			},
+		},
+		// case 6: same updates
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Fid: &filer_pb.FileId{FileKey: 1}, ModifiedTsNs: 123},
+				{Offset: 0, Size: 100, FileId: "axf", Fid: &filer_pb.FileId{FileKey: 2}, ModifiedTsNs: 124},
+				{Offset: 0, Size: 100, FileId: "xyz", Fid: &filer_pb.FileId{FileKey: 3}, ModifiedTsNs: 125},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 100, fileId: "xyz"},
+			},
+		},
+		// case 7: real updates
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 2097152, FileId: "7,0294cbb9892b", ModifiedTsNs: 123},
+				{Offset: 0, Size: 3145728, FileId: "3,029565bf3092", ModifiedTsNs: 130},
+				{Offset: 2097152, Size: 3145728, FileId: "6,029632f47ae2", ModifiedTsNs: 140},
+				{Offset: 5242880, Size: 3145728, FileId: "2,029734c5aa10", ModifiedTsNs: 150},
+				{Offset: 8388608, Size: 3145728, FileId: "5,02982f80de50", ModifiedTsNs: 160},
+				{Offset: 11534336, Size: 2842193, FileId: "7,0299ad723803", ModifiedTsNs: 170},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 2097152, fileId: "3,029565bf3092"},
+				{start: 2097152, stop: 5242880, fileId: "6,029632f47ae2"},
+				{start: 5242880, stop: 8388608, fileId: "2,029734c5aa10"},
+				{start: 8388608, stop: 11534336, fileId: "5,02982f80de50"},
+				{start: 11534336, stop: 14376529, fileId: "7,0299ad723803"},
+			},
+		},
+		// case 8: real bug
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 77824, FileId: "4,0b3df938e301", ModifiedTsNs: 123},
+				{Offset: 471040, Size: 472225 - 471040, FileId: "6,0b3e0650019c", ModifiedTsNs: 130},
+				{Offset: 77824, Size: 208896 - 77824, FileId: "4,0b3f0c7202f0", ModifiedTsNs: 140},
+				{Offset: 208896, Size: 339968 - 208896, FileId: "2,0b4031a72689", ModifiedTsNs: 150},
+				{Offset: 339968, Size: 471040 - 339968, FileId: "3,0b416a557362", ModifiedTsNs: 160},
+			},
+			Expected: []*VisibleInterval{
+				{start: 0, stop: 77824, fileId: "4,0b3df938e301"},
+				{start: 77824, stop: 208896, fileId: "4,0b3f0c7202f0"},
+				{start: 208896, stop: 339968, fileId: "2,0b4031a72689"},
+				{start: 339968, stop: 471040, fileId: "3,0b416a557362"},
+				{start: 471040, stop: 472225, fileId: "6,0b3e0650019c"},
+			},
+		},
+	}
+
+	for i, testcase := range testcases {
+		log.Printf("++++++++++ merged test case %d ++++++++++++++++++++", i)
+		intervals, _ := NonOverlappingVisibleIntervals(context.Background(), nil, testcase.Chunks, 0, math.MaxInt64)
+		x := -1
+		for visible := intervals.Front(); visible != nil; visible = visible.Next {
+			x++
+			interval := visible.Value
+			log.Printf("test case %d, interval start=%d, stop=%d, fileId=%s",
+				i, interval.start, interval.stop, interval.fileId)
+		}
+		x = -1
+		for visible := intervals.Front(); visible != nil; visible = visible.Next {
+			x++
+			interval := visible.Value
+			if interval.start != testcase.Expected[x].start {
+				t.Fatalf("failed on test case %d, interval %d, start %d, expect %d",
+					i, x, interval.start, testcase.Expected[x].start)
+			}
+			if interval.stop != testcase.Expected[x].stop {
+				t.Fatalf("failed on test case %d, interval %d, stop %d, expect %d",
+					i, x, interval.stop, testcase.Expected[x].stop)
+			}
+			if interval.fileId != testcase.Expected[x].fileId {
+				t.Fatalf("failed on test case %d, interval %d, chunkId %s, expect %s",
+					i, x, interval.fileId, testcase.Expected[x].fileId)
+			}
+			if interval.offsetInChunk != testcase.Expected[x].offsetInChunk {
+				t.Fatalf("failed on test case %d, interval %d, offsetInChunk %d, expect %d",
+					i, x, interval.offsetInChunk, testcase.Expected[x].offsetInChunk)
+			}
+		}
+		if intervals.Len() != len(testcase.Expected) {
+			t.Fatalf("failed to compact test case %d, len %d expected %d", i, intervals.Len(), len(testcase.Expected))
+		}
+
+	}
+
+}
+
+func TestChunksReading(t *testing.T) {
+
+	testcases := []struct {
+		Chunks   []*filer_pb.FileChunk
+		Offset   int64
+		Size     int64
+		Expected []*ChunkView
+	}{
+		// case 0: normal
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
+				{Offset: 100, Size: 100, FileId: "asdf", ModifiedTsNs: 134},
+				{Offset: 200, Size: 100, FileId: "fsad", ModifiedTsNs: 353},
+			},
+			Offset: 0,
+			Size:   250,
+			Expected: []*ChunkView{
+				{OffsetInChunk: 0, ViewSize: 100, FileId: "abc", ViewOffset: 0},
+				{OffsetInChunk: 0, ViewSize: 100, FileId: "asdf", ViewOffset: 100},
+				{OffsetInChunk: 0, ViewSize: 50, FileId: "fsad", ViewOffset: 200},
+			},
+		},
+		// case 1: updates overwrite full chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
+				{Offset: 0, Size: 200, FileId: "asdf", ModifiedTsNs: 134},
+			},
+			Offset: 50,
+			Size:   100,
+			Expected: []*ChunkView{
+				{OffsetInChunk: 50, ViewSize: 100, FileId: "asdf", ViewOffset: 50},
+			},
+		},
+		// case 2: updates overwrite part of previous chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 3, Size: 100, FileId: "a", ModifiedTsNs: 123},
+				{Offset: 10, Size: 50, FileId: "b", ModifiedTsNs: 134},
+			},
+			Offset: 30,
+			Size:   40,
+			Expected: []*ChunkView{
+				{OffsetInChunk: 20, ViewSize: 30, FileId: "b", ViewOffset: 30},
+				{OffsetInChunk: 57, ViewSize: 10, FileId: "a", ViewOffset: 60},
+			},
+		},
+		// case 3: updates overwrite full chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
+				{Offset: 0, Size: 200, FileId: "asdf", ModifiedTsNs: 134},
+				{Offset: 50, Size: 250, FileId: "xxxx", ModifiedTsNs: 154},
+			},
+			Offset: 0,
+			Size:   200,
+			Expected: []*ChunkView{
+				{OffsetInChunk: 0, ViewSize: 50, FileId: "asdf", ViewOffset: 0},
+				{OffsetInChunk: 0, ViewSize: 150, FileId: "xxxx", ViewOffset: 50},
+			},
+		},
+		// case 4: updates far away from prev chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
+				{Offset: 0, Size: 200, FileId: "asdf", ModifiedTsNs: 134},
+				{Offset: 250, Size: 250, FileId: "xxxx", ModifiedTsNs: 154},
+			},
+			Offset: 0,
+			Size:   400,
+			Expected: []*ChunkView{
+				{OffsetInChunk: 0, ViewSize: 200, FileId: "asdf", ViewOffset: 0},
+				{OffsetInChunk: 0, ViewSize: 150, FileId: "xxxx", ViewOffset: 250},
+			},
+		},
+		// case 5: updates overwrite full chunks
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "a", ModifiedTsNs: 123},
+				{Offset: 0, Size: 200, FileId: "c", ModifiedTsNs: 184},
+				{Offset: 70, Size: 150, FileId: "b", ModifiedTsNs: 143},
+				{Offset: 80, Size: 100, FileId: "xxxx", ModifiedTsNs: 134},
+			},
+			Offset: 0,
+			Size:   220,
+			Expected: []*ChunkView{
+				{OffsetInChunk: 0, ViewSize: 200, FileId: "c", ViewOffset: 0},
+				{OffsetInChunk: 130, ViewSize: 20, FileId: "b", ViewOffset: 200},
+			},
+		},
+		// case 6: same updates
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Fid: &filer_pb.FileId{FileKey: 1}, ModifiedTsNs: 123},
+				{Offset: 0, Size: 100, FileId: "def", Fid: &filer_pb.FileId{FileKey: 2}, ModifiedTsNs: 124},
+				{Offset: 0, Size: 100, FileId: "xyz", Fid: &filer_pb.FileId{FileKey: 3}, ModifiedTsNs: 125},
+			},
+			Offset: 0,
+			Size:   100,
+			Expected: []*ChunkView{
+				{OffsetInChunk: 0, ViewSize: 100, FileId: "xyz", ViewOffset: 0},
+			},
+		},
+		// case 7: edge cases
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
+				{Offset: 100, Size: 100, FileId: "asdf", ModifiedTsNs: 134},
+				{Offset: 200, Size: 100, FileId: "fsad", ModifiedTsNs: 353},
+			},
+			Offset: 0,
+			Size:   200,
+			Expected: []*ChunkView{
+				{OffsetInChunk: 0, ViewSize: 100, FileId: "abc", ViewOffset: 0},
+				{OffsetInChunk: 0, ViewSize: 100, FileId: "asdf", ViewOffset: 100},
+			},
+		},
+		// case 8: edge cases
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 123},
+				{Offset: 90, Size: 200, FileId: "asdf", ModifiedTsNs: 134},
+				{Offset: 190, Size: 300, FileId: "fsad", ModifiedTsNs: 353},
+			},
+			Offset: 0,
+			Size:   300,
+			Expected: []*ChunkView{
+				{OffsetInChunk: 0, ViewSize: 90, FileId: "abc", ViewOffset: 0},
+				{OffsetInChunk: 0, ViewSize: 100, FileId: "asdf", ViewOffset: 90},
+				{OffsetInChunk: 0, ViewSize: 110, FileId: "fsad", ViewOffset: 190},
+			},
+		},
+		// case 9: edge cases
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 43175947, FileId: "2,111fc2cbfac1", ModifiedTsNs: 1},
+				{Offset: 43175936, Size: 52981771 - 43175936, FileId: "2,112a36ea7f85", ModifiedTsNs: 2},
+				{Offset: 52981760, Size: 72564747 - 52981760, FileId: "4,112d5f31c5e7", ModifiedTsNs: 3},
+				{Offset: 72564736, Size: 133255179 - 72564736, FileId: "1,113245f0cdb6", ModifiedTsNs: 4},
+				{Offset: 133255168, Size: 137269259 - 133255168, FileId: "3,1141a70733b5", ModifiedTsNs: 5},
+				{Offset: 137269248, Size: 153578836 - 137269248, FileId: "1,114201d5bbdb", ModifiedTsNs: 6},
+			},
+			Offset: 0,
+			Size:   153578836,
+			Expected: []*ChunkView{
+				{OffsetInChunk: 0, ViewSize: 43175936, FileId: "2,111fc2cbfac1", ViewOffset: 0},
+				{OffsetInChunk: 0, ViewSize: 52981760 - 43175936, FileId: "2,112a36ea7f85", ViewOffset: 43175936},
+				{OffsetInChunk: 0, ViewSize: 72564736 - 52981760, FileId: "4,112d5f31c5e7", ViewOffset: 52981760},
+				{OffsetInChunk: 0, ViewSize: 133255168 - 72564736, FileId: "1,113245f0cdb6", ViewOffset: 72564736},
+				{OffsetInChunk: 0, ViewSize: 137269248 - 133255168, FileId: "3,1141a70733b5", ViewOffset: 133255168},
+				{OffsetInChunk: 0, ViewSize: 153578836 - 137269248, FileId: "1,114201d5bbdb", ViewOffset: 137269248},
+			},
+		},
+	}
+
+	for i, testcase := range testcases {
+		if i != 2 {
+			// continue
+		}
+		log.Printf("++++++++++ read test case %d ++++++++++++++++++++", i)
+		chunks := ViewFromChunks(context.Background(), nil, testcase.Chunks, testcase.Offset, testcase.Size)
+		x := -1
+		for c := chunks.Front(); c != nil; c = c.Next {
+			x++
+			chunk := c.Value
+			log.Printf("read case %d, chunk %d, offset=%d, size=%d, fileId=%s",
+				i, x, chunk.OffsetInChunk, chunk.ViewSize, chunk.FileId)
+			if chunk.OffsetInChunk != testcase.Expected[x].OffsetInChunk {
+				t.Fatalf("failed on read case %d, chunk %s, Offset %d, expect %d",
+					i, chunk.FileId, chunk.OffsetInChunk, testcase.Expected[x].OffsetInChunk)
+			}
+			if chunk.ViewSize != testcase.Expected[x].ViewSize {
+				t.Fatalf("failed on read case %d, chunk %s, ViewSize %d, expect %d",
+					i, chunk.FileId, chunk.ViewSize, testcase.Expected[x].ViewSize)
+			}
+			if chunk.FileId != testcase.Expected[x].FileId {
+				t.Fatalf("failed on read case %d, chunk %d, FileId %s, expect %s",
+					i, x, chunk.FileId, testcase.Expected[x].FileId)
+			}
+			if chunk.ViewOffset != testcase.Expected[x].ViewOffset {
+				t.Fatalf("failed on read case %d, chunk %d, ViewOffset %d, expect %d",
+					i, x, chunk.ViewOffset, testcase.Expected[x].ViewOffset)
+			}
+		}
+		if chunks.Len() != len(testcase.Expected) {
+			t.Fatalf("failed to read test case %d, len %d expected %d", i, chunks.Len(), len(testcase.Expected))
+		}
+	}
+
+}
+
+func BenchmarkCompactFileChunks(b *testing.B) {
+
+	var chunks []*filer_pb.FileChunk
+
+	k := 1024
+
+	for n := 0; n < k; n++ {
+		chunks = append(chunks, &filer_pb.FileChunk{
+			Offset: int64(n * 100), Size: 100, FileId: fmt.Sprintf("fileId%d", n), ModifiedTsNs: int64(n),
+		})
+		chunks = append(chunks, &filer_pb.FileChunk{
+			Offset: int64(n * 50), Size: 100, FileId: fmt.Sprintf("fileId%d", n+k), ModifiedTsNs: int64(n + k),
+		})
+	}
+
+	for n := 0; n < b.N; n++ {
+		CompactFileChunks(context.Background(), nil, chunks)
+	}
+}
+
+func addVisibleInterval(visibles *IntervalList[*VisibleInterval], x *VisibleInterval) {
+	visibles.AppendInterval(&Interval[*VisibleInterval]{
+		StartOffset: x.start,
+		StopOffset:  x.stop,
+		TsNs:        x.modifiedTsNs,
+		Value:       x,
+	})
+}
+
+func TestViewFromVisibleIntervals(t *testing.T) {
+	visibles := NewIntervalList[*VisibleInterval]()
+	addVisibleInterval(visibles, &VisibleInterval{
+		start:  0,
+		stop:   25,
+		fileId: "fid1",
+	})
+	addVisibleInterval(visibles, &VisibleInterval{
+		start:  4096,
+		stop:   8192,
+		fileId: "fid2",
+	})
+	addVisibleInterval(visibles, &VisibleInterval{
+		start:  16384,
+		stop:   18551,
+		fileId: "fid3",
+	})
+
+	views := ViewFromVisibleIntervals(visibles, 0, math.MaxInt32)
+
+	if views.Len() != visibles.Len() {
+		assert.Equal(t, visibles.Len(), views.Len(), "ViewFromVisibleIntervals error")
+	}
+
+}
+
+func TestViewFromVisibleIntervals2(t *testing.T) {
+	visibles := NewIntervalList[*VisibleInterval]()
+	addVisibleInterval(visibles, &VisibleInterval{
+		start:  344064,
+		stop:   348160,
+		fileId: "fid1",
+	})
+	addVisibleInterval(visibles, &VisibleInterval{
+		start:  348160,
+		stop:   356352,
+		fileId: "fid2",
+	})
+
+	views := ViewFromVisibleIntervals(visibles, 0, math.MaxInt32)
+
+	if views.Len() != visibles.Len() {
+		assert.Equal(t, visibles.Len(), views.Len(), "ViewFromVisibleIntervals error")
+	}
+
+}
+
+func TestViewFromVisibleIntervals3(t *testing.T) {
+	visibles := NewIntervalList[*VisibleInterval]()
+	addVisibleInterval(visibles, &VisibleInterval{
+		start:  1000,
+		stop:   2000,
+		fileId: "fid1",
+	})
+	addVisibleInterval(visibles, &VisibleInterval{
+		start:  3000,
+		stop:   4000,
+		fileId: "fid2",
+	})
+
+	views := ViewFromVisibleIntervals(visibles, 1700, 1500)
+
+	if views.Len() != visibles.Len() {
+		assert.Equal(t, visibles.Len(), views.Len(), "ViewFromVisibleIntervals error")
+	}
+
+}
+
+func TestCompactFileChunks3(t *testing.T) {
+	chunks := []*filer_pb.FileChunk{
+		{Offset: 0, Size: 100, FileId: "abc", ModifiedTsNs: 50},
+		{Offset: 100, Size: 100, FileId: "ghi", ModifiedTsNs: 50},
+		{Offset: 200, Size: 100, FileId: "jkl", ModifiedTsNs: 100},
+		{Offset: 300, Size: 100, FileId: "def", ModifiedTsNs: 200},
+	}
+
+	compacted, _ := CompactFileChunks(context.Background(), nil, chunks)
+
+	if len(compacted) != 4 {
+		t.Fatalf("unexpected compacted: %d", len(compacted))
+	}
+}

weed/filer/filer.go

@@ -0,0 +1,758 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+	"github.com/seaweedfs/seaweedfs/weed/filer/empty_folder_cleanup"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+	"golang.org/x/sync/singleflight"
+)
+
+const (
+	LogFlushInterval = time.Minute
+	PaginationSize   = 1024
+	FilerStoreId     = "filer.store.id"
+)
+
+var (
+	OS_UID = uint32(os.Getuid())
+	OS_GID = uint32(os.Getgid())
+)
+
+type Filer struct {
+	UniqueFilerId                 int32
+	UniqueFilerEpoch              int32
+	Store                         VirtualFilerStore
+	MasterClient                  *wdclient.MasterClient
+	FileIdDeletionQueue           *util.UnboundedQueue
+	GrpcDialOption                grpc.DialOption
+	DirBucketsPath                string
+	Cipher                        bool
+	LocalMetaLogBuffer            *log_buffer.LogBuffer
+	metaLogCollection             string
+	metaLogReplication            string
+	DefaultDiskType               string
+	MetaAggregator                *MetaAggregator
+	Signature                     int32
+	FilerConf                     *FilerConf
+	placementOverlay              PlacementOverlay
+	RemoteStorage                 *FilerRemoteStorage
+	BuildGuardedRemoteClient      RemoteStorageClientBuilder
+	AllowUntrustedRemoteEndpoints bool
+	lazyFetchGroup                singleflight.Group
+	lazyListGroup                 singleflight.Group
+	Dlm                           *lock_manager.DistributedLockManager
+	MaxFilenameLength             uint32
+	deletionQuit                  chan struct{}
+	DeletionRetryQueue            *DeletionRetryQueue
+	EmptyFolderCleaner            *empty_folder_cleanup.EmptyFolderCleaner
+	EmptyFolderCleanupDelay       time.Duration
+	persistedLogCache             *persistedLogCache
+	metaLogInflight               metaLogInflight
+}
+
+func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, maxFilenameLength uint32, notifyFn func()) *Filer {
+	f := &Filer{
+		MasterClient:        wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, "", masters),
+		FileIdDeletionQueue: util.NewUnboundedQueue(),
+		GrpcDialOption:      grpcDialOption,
+		FilerConf:           NewFilerConf(),
+		RemoteStorage:       NewFilerRemoteStorage(),
+		UniqueFilerId:       util.RandomInt32(),
+		Dlm:                 lock_manager.NewDistributedLockManager(filerHost),
+		MaxFilenameLength:   maxFilenameLength,
+		deletionQuit:        make(chan struct{}),
+		DeletionRetryQueue:  NewDeletionRetryQueue(),
+		persistedLogCache:   newPersistedLogCache(persistedLogCacheMaxBytes),
+	}
+	if f.UniqueFilerId < 0 {
+		f.UniqueFilerId = -f.UniqueFilerId
+	}
+
+	// ReadFromDiskFn is intentionally nil here.  SubscribeLocalMetadata already
+	// manages disk reads explicitly with shouldReadFromDisk / lastCheckedFlushTsNs
+	// tracking.  Setting ReadFromDiskFn would cause LoopProcessLogData to issue a
+	// redundant ReadPersistedLogBuffer call (ListDirectoryEntries + readahead
+	// goroutine) on every 250ms health-check tick when a subscriber encounters
+	// ResumeFromDiskError, adding significant CPU and GC pressure even when idle.
+	f.LocalMetaLogBuffer = log_buffer.NewLogBuffer("local", LogFlushInterval, f.logFlushFunc, nil, notifyFn)
+	f.metaLogCollection = collection
+	f.metaLogReplication = replication
+
+	if newPlacementOverlay != nil {
+		f.placementOverlay = newPlacementOverlay(f)
+	}
+
+	go f.loopProcessingDeletion()
+
+	return f
+}
+
+func (f *Filer) buildRemoteStorageClient(ctx context.Context, remoteConf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	if f.BuildGuardedRemoteClient != nil {
+		return f.BuildGuardedRemoteClient(ctx, remoteConf, f.AllowUntrustedRemoteEndpoints)
+	}
+	return remote_storage.GetRemoteStorage(remoteConf)
+}
+
+func (f *Filer) MaybeBootstrapFromOnePeer(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, snapshotTime time.Time) (err error) {
+	if len(existingNodes) == 0 {
+		return
+	}
+	sort.Slice(existingNodes, func(i, j int) bool {
+		return existingNodes[i].CreatedAtNs < existingNodes[j].CreatedAtNs
+	})
+	earliestNode := existingNodes[0]
+	if pb.ServerAddress(earliestNode.Address).Equals(self) {
+		return
+	}
+
+	glog.V(0).Infof("bootstrap from %v clientId:%d", earliestNode.Address, f.UniqueFilerId)
+
+	return pb.WithFilerClient(false, f.UniqueFilerId, pb.ServerAddress(earliestNode.Address), f.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		return filer_pb.StreamBfs(client, "/", snapshotTime.UnixNano(), func(parentPath util.FullPath, entry *filer_pb.Entry) error {
+			return f.Store.InsertEntry(context.Background(), FromPbEntry(string(parentPath), entry))
+		})
+	})
+
+}
+
+func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, startFrom time.Time) {
+
+	var snapshot []pb.ServerAddress
+	for _, node := range existingNodes {
+		address := pb.ServerAddress(node.Address)
+		snapshot = append(snapshot, address)
+	}
+	f.Dlm.LockRing.SetSnapshot(snapshot, 0)
+	glog.V(0).Infof("%s aggregate from peers %+v", self, snapshot)
+
+	// Initialize the empty folder cleaner using the same LockRing as Dlm for consistent hashing
+	f.EmptyFolderCleaner = empty_folder_cleanup.NewEmptyFolderCleaner(f, f.Dlm.LockRing, self, f.DirBucketsPath, f.EmptyFolderCleanupDelay)
+
+	f.MetaAggregator = NewMetaAggregator(f, self, f.GrpcDialOption)
+	// The ring starts empty while peer history sits on disk: mark the pre-startFrom
+	// range evicted so a cursor there reads disk, not the ring's earliest entry.
+	f.MetaAggregator.MetaLogBuffer.MarkEvictedThrough(startFrom.UnixNano())
+	f.MasterClient.SetOnPeerUpdateFn(func(update *master_pb.ClusterNodeUpdate, startFrom time.Time) {
+		if update.NodeType != cluster.FilerType {
+			return
+		}
+		// Lock ring is now managed by the master via LockRingUpdate,
+		// so we no longer call AddServer/RemoveServer here.
+		f.MetaAggregator.OnPeerUpdate(update, startFrom)
+	})
+	f.MasterClient.SetOnLockRingUpdateFn(func(update *master_pb.LockRingUpdate) {
+		var servers []pb.ServerAddress
+		for _, s := range update.Servers {
+			servers = append(servers, pb.ServerAddress(s))
+		}
+		glog.V(0).Infof("LockRing: applying master ring update v%d: %v", update.Version, servers)
+		f.Dlm.LockRing.SetSnapshot(servers, update.Version)
+	})
+
+	// Subscribe to the local filer first: its events reach the aggregated
+	// buffer only through this subscription, and the peer watermarks must
+	// account for it before any remote peer - a remotes-only watermark set
+	// would claim completeness without self. existingNodes can omit self
+	// (master registration races this bootstrap); duplicate adds are no-ops.
+	f.MetaAggregator.OnPeerUpdate(&master_pb.ClusterNodeUpdate{
+		NodeType: cluster.FilerType,
+		Address:  string(self),
+		IsAdd:    true,
+	}, startFrom)
+	for _, peerUpdate := range existingNodes {
+		f.MetaAggregator.OnPeerUpdate(peerUpdate, startFrom)
+	}
+
+}
+
+func (f *Filer) ListExistingPeerUpdates(ctx context.Context) (existingNodes []*master_pb.ClusterNodeUpdate) {
+	return cluster.ListExistingPeerUpdates(f.GetMaster(ctx), f.GrpcDialOption, f.MasterClient.FilerGroup, cluster.FilerType)
+}
+
+func (f *Filer) SetStore(store FilerStore) (isFresh bool) {
+	f.Store = NewFilerStoreWrapper(store)
+
+	return f.setOrLoadFilerStoreSignature(store)
+}
+
+func (f *Filer) setOrLoadFilerStoreSignature(store FilerStore) (isFresh bool) {
+	storeIdBytes, err := store.KvGet(context.Background(), []byte(FilerStoreId))
+	if err == ErrKvNotFound || err == nil && len(storeIdBytes) == 0 {
+		f.Signature = util.RandomInt32()
+		storeIdBytes = make([]byte, 4)
+		util.Uint32toBytes(storeIdBytes, uint32(f.Signature))
+		if err = store.KvPut(context.Background(), []byte(FilerStoreId), storeIdBytes); err != nil {
+			glog.Fatalf("set %s=%d : %v", FilerStoreId, f.Signature, err)
+		}
+		glog.V(0).Infof("create %s to %d", FilerStoreId, f.Signature)
+		return true
+	} else if err == nil && len(storeIdBytes) == 4 {
+		f.Signature = int32(util.BytesToUint32(storeIdBytes))
+		glog.V(0).Infof("existing %s = %d", FilerStoreId, f.Signature)
+	} else {
+		glog.Fatalf("read %v=%v : %v", FilerStoreId, string(storeIdBytes), err)
+	}
+	return false
+}
+
+func (f *Filer) GetStore() (store FilerStore) {
+	return f.Store
+}
+
+func (fs *Filer) GetMaster(ctx context.Context) pb.ServerAddress {
+	return fs.MasterClient.GetMaster(ctx)
+}
+
+func (f *Filer) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return f.Store.BeginTransaction(ctx)
+}
+
+func (f *Filer) CommitTransaction(ctx context.Context) error {
+	return f.Store.CommitTransaction(ctx)
+}
+
+func (f *Filer) RollbackTransaction(ctx context.Context) error {
+	return f.Store.RollbackTransaction(ctx)
+}
+
+// CreateEntry creates or replaces an entry. When existing is non-nil the caller
+// has already fetched the current entry at this path under a path lock, and it
+// is reused instead of looking the store up again; pass nil to have CreateEntry
+// look it up itself.
+func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool, maxFilenameLength uint32) error {
+
+	if string(entry.FullPath) == "/" {
+		return nil
+	}
+
+	if entry.FullPath.IsLongerFileName(maxFilenameLength) {
+		return filer_pb.ErrEntryNameTooLong
+	}
+
+	if entry.IsDirectory() {
+		entry.Attr.TtlSec = 0
+	}
+
+	if entry.Attr.Atime.IsZero() {
+		entry.Attr.Atime = entryInitialAtime(entry.Attr)
+	}
+
+	oldEntry := existing
+	if oldEntry == nil {
+		oldEntry, _ = f.FindEntry(ctx, entry.FullPath)
+	}
+
+	/*
+		if !hasWritePermission(lastDirectoryEntry, entry) {
+			glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d",
+				lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid)
+			return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath)
+		}
+	*/
+
+	if oldEntry == nil {
+		f.ensureEntryInode(entry)
+
+		glog.V(4).InfofCtx(ctx, "InsertEntry %s: new entry: %v", entry.FullPath, entry.Name())
+		if err := f.Store.InsertEntry(ctx, entry); err != nil {
+			glog.ErrorfCtx(ctx, "insert entry %s: %v", entry.FullPath, err)
+			return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
+		}
+
+		// Parents go after the entry: one checked first can be taken by the
+		// empty-folder cleaner before the entry lands, and nothing would look again.
+		if !skipCreateParentDir {
+			dirParts := strings.Split(string(entry.FullPath), "/")
+			if err := f.ensureParentDirectoryEntry(ctx, entry, dirParts, len(dirParts)-1, isFromOtherCluster); err != nil {
+				// The entry stays: deleting by path would destroy a concurrent create
+				// that already succeeded through the update branch, and the update keeps
+				// the inode and crtime while mtime only survives the store to the second.
+				// It is not announced - the aggregator replicates creates into peer
+				// stores, so a failed write would land on all of them rather than on the
+				// one filer whose next write into that folder repairs it.
+				glog.ErrorfCtx(ctx, "create parent directories of %s: %v", entry.FullPath, err)
+				return err
+			}
+		}
+
+		if !entry.IsDirectory() {
+			stats.FilerObjectSizeBytesHistogram.Observe(float64(entry.Size()))
+		}
+	} else {
+		if o_excl {
+			glog.V(3).InfofCtx(ctx, "EEXIST: entry %s already exists", entry.FullPath)
+			return fmt.Errorf("%s: %w", entry.FullPath, filer_pb.ErrEntryAlreadyExists)
+		}
+		glog.V(4).InfofCtx(ctx, "UpdateEntry %s: old entry: %v", entry.FullPath, oldEntry.Name())
+		if err := f.UpdateEntry(ctx, oldEntry, entry); err != nil {
+			if errors.Is(err, filer_pb.ErrExistingIsDirectory) || errors.Is(err, filer_pb.ErrExistingIsFile) {
+				glog.V(2).InfofCtx(ctx, "update entry %s: %v", entry.FullPath, err)
+			} else {
+				glog.ErrorfCtx(ctx, "update entry %s: %v", entry.FullPath, err)
+			}
+			return fmt.Errorf("update entry %s: %w", entry.FullPath, err)
+		}
+	}
+
+	f.NotifyUpdateEvent(ctx, oldEntry, entry, true, isFromOtherCluster, signatures)
+
+	f.deleteChunksIfNotNew(ctx, oldEntry, entry)
+
+	glog.V(4).InfofCtx(ctx, "CreateEntry %s: created", entry.FullPath)
+
+	return nil
+}
+
+func (f *Filer) ensureParentDirectoryEntry(ctx context.Context, entry *Entry, dirParts []string, level int, isFromOtherCluster bool) (err error) {
+
+	if level == 0 {
+		return nil
+	}
+
+	dirPath := "/" + util.Join(dirParts[:level]...)
+	// fmt.Printf("%d dirPath: %+v\n", level, dirPath)
+
+	// check the store directly
+	glog.V(4).InfofCtx(ctx, "find uncached directory: %s", dirPath)
+	dirEntry, findErr := f.FindEntry(ctx, util.FullPath(dirPath))
+	if findErr != nil && !errors.Is(findErr, filer_pb.ErrNotFound) {
+		return findErr
+	}
+
+	// no such existing directory
+	if dirEntry == nil {
+
+		// fmt.Printf("dirParts: %v %v %v\n", dirParts[0], dirParts[1], dirParts[2])
+		// dirParts[0] == "" and dirParts[1] == "buckets"
+		isUnderBuckets := len(dirParts) >= 3 && dirParts[1] == "buckets"
+		if isUnderBuckets {
+			if !strings.HasPrefix(dirParts[2], ".") {
+				if err := s3bucket.VerifyS3BucketName(dirParts[2]); err != nil {
+					return fmt.Errorf("invalid bucket name %s: %v", dirParts[2], err)
+				}
+			}
+		}
+
+		// ensure parent directory
+		if err = f.ensureParentDirectoryEntry(ctx, entry, dirParts, level-1, isFromOtherCluster); err != nil {
+			return err
+		}
+
+		// create the directory
+		now := time.Now()
+
+		dirEntry = &Entry{
+			FullPath: util.FullPath(dirPath),
+			Attr: Attr{
+				Mtime:      now,
+				Crtime:     now,
+				Mode:       os.ModeDir | entry.Mode | 0111,
+				Uid:        entry.Uid,
+				Gid:        entry.Gid,
+				UserName:   entry.UserName,
+				GroupNames: entry.GroupNames,
+			},
+		}
+		f.ensureEntryInode(dirEntry)
+		if isUnderBuckets && level > 3 {
+			// Parent directories under buckets are created automatically; no additional logging.
+		}
+
+		glog.V(2).InfofCtx(ctx, "create directory: %s %v", dirPath, dirEntry.Mode)
+		mkdirErr := f.Store.InsertEntry(ctx, dirEntry)
+		if mkdirErr != nil {
+			if fEntry, err := f.FindEntry(ctx, util.FullPath(dirPath)); err == filer_pb.ErrNotFound || fEntry == nil {
+				glog.V(3).InfofCtx(ctx, "mkdir %s: %v", dirPath, mkdirErr)
+				return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
+			}
+		} else {
+			if !strings.HasPrefix("/"+util.Join(dirParts[:]...), SystemLogDir) {
+				f.NotifyUpdateEvent(ctx, nil, dirEntry, false, isFromOtherCluster, nil)
+			}
+		}
+
+	} else if !dirEntry.IsDirectory() {
+		// S3 allows both "foo/bar" (object) and "foo/bar/xyzzy" (another
+		// object) to coexist because S3 has a flat key space. Promote the
+		// existing file to a directory, preserving its content/chunks so
+		// the original object data remains accessible.
+		glog.V(2).InfofCtx(ctx, "promoting %s from file to directory for %s", dirPath, entry.FullPath)
+		dirEntry.Attr.Mode |= os.ModeDir | 0111
+		// Expiring the entry now deletes the directory row and strands the keys under
+		// it, so the prefix object gives up its lazy TTL. The lifecycle worker still
+		// expires it, through the delete that leaves the directory behind.
+		dirEntry.Attr.TtlSec = 0
+		// An empty object leaves no chunks, content or mime behind, so without the
+		// mark the promotion would hide it.
+		if dirEntry.Extended == nil {
+			dirEntry.Extended = make(map[string][]byte)
+		}
+		dirEntry.Extended[s3_constants.SeaweedFSPrefixObject] = []byte("true")
+		if updateErr := f.Store.UpdateEntry(ctx, dirEntry); updateErr != nil {
+			return fmt.Errorf("promote %s to directory: %v", dirPath, updateErr)
+		}
+		f.NotifyUpdateEvent(ctx, nil, dirEntry, false, isFromOtherCluster, nil)
+	}
+
+	return nil
+}
+
+// restorableModeBits is everything but the type bits: ModePerm alone would drop
+// setgid, setuid and sticky, quietly changing group inheritance and delete semantics.
+const restorableModeBits = os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky
+
+// DirectoryAttributes reads what a directory would need to be recreated as it is.
+func (f *Filer) DirectoryAttributes(ctx context.Context, dirPath util.FullPath) (attrs empty_folder_cleanup.DirectoryAttributes, err error) {
+	entry, err := f.FindEntry(ctx, dirPath)
+	if err != nil {
+		return attrs, err
+	}
+	if entry == nil {
+		return attrs, filer_pb.ErrNotFound
+	}
+	return empty_folder_cleanup.DirectoryAttributes{
+		Mode:       entry.Mode & restorableModeBits,
+		Uid:        entry.Uid,
+		Gid:        entry.Gid,
+		UserName:   entry.UserName,
+		GroupNames: entry.GroupNames,
+	}, nil
+}
+
+// EnsureDirectoryEntry recreates dirPath, and any missing ancestor, for entries that
+// outlived the directory holding them. dirPath comes back with the attributes it was
+// deleted with, so a restore cannot hand back a directory more permissive than the one
+// it replaces - including when someone else already put it back with wider ones.
+func (f *Filer) EnsureDirectoryEntry(ctx context.Context, dirPath util.FullPath, attrs empty_folder_cleanup.DirectoryAttributes) error {
+	existing, err := f.FindEntry(ctx, dirPath)
+	if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
+		return err
+	}
+	if existing != nil {
+		// A writer recreating its own missing parent infers the mode from the entry it
+		// is inserting, so it can come back wider. Intersect rather than replace, or a
+		// mode holding a bit the saved one lacks would be granted the ones it lacks.
+		kept := existing.Mode & attrs.Mode & restorableModeBits
+		if existing.Mode&restorableModeBits == kept {
+			return nil
+		}
+		narrowed := existing.ShallowClone()
+		narrowed.Mode = existing.Mode&^restorableModeBits | kept
+		glog.V(1).InfofCtx(ctx, "restore directory %s: narrowing %v to %v", dirPath, existing.Mode, narrowed.Mode)
+		if err := f.UpdateEntry(ctx, existing, narrowed); err != nil {
+			return err
+		}
+		f.NotifyUpdateEvent(ctx, existing, narrowed, false, false, nil)
+		return nil
+	}
+
+	holder := &Entry{FullPath: dirPath, Attr: Attr{
+		Mode: attrs.Mode, Uid: attrs.Uid, Gid: attrs.Gid,
+		UserName: attrs.UserName, GroupNames: attrs.GroupNames,
+	}}
+	dirParts := strings.Split(string(dirPath), "/")
+	if err := f.ensureParentDirectoryEntry(ctx, holder, dirParts, len(dirParts)-1, false); err != nil {
+		return err
+	}
+
+	now := time.Now()
+	dirEntry := &Entry{FullPath: dirPath, Attr: Attr{
+		Mtime: now, Crtime: now,
+		Mode:     os.ModeDir | attrs.Mode,
+		Uid:      attrs.Uid,
+		Gid:      attrs.Gid,
+		UserName: attrs.UserName, GroupNames: attrs.GroupNames,
+	}}
+	f.ensureEntryInode(dirEntry)
+	if err := f.Store.InsertEntry(ctx, dirEntry); err != nil {
+		return fmt.Errorf("restore directory %s: %v", dirPath, err)
+	}
+	f.NotifyUpdateEvent(ctx, nil, dirEntry, false, false, nil)
+
+	return nil
+}
+
+func (f *Filer) UpdateEntry(ctx context.Context, oldEntry, entry *Entry) (err error) {
+	if oldEntry != nil {
+		entry.Attr.Crtime = oldEntry.Attr.Crtime
+		if oldEntry.Attr.Inode != 0 {
+			// Object identity must not change on in-place updates.
+			entry.Attr.Inode = oldEntry.Attr.Inode
+		} else {
+			f.ensureEntryInode(entry)
+		}
+		// A type conflict is reported through the sentinel, and callers act on it -
+		// an S3 write of a key other keys are nested under retries as a prefix object -
+		// so it is the caller's outcome that decides whether anything went wrong.
+		if oldEntry.IsDirectory() && !entry.IsDirectory() {
+			glog.V(2).InfofCtx(ctx, "existing %s is a directory", oldEntry.FullPath)
+			return fmt.Errorf("%s: %w", oldEntry.FullPath, filer_pb.ErrExistingIsDirectory)
+		}
+		if !oldEntry.IsDirectory() && entry.IsDirectory() {
+			glog.V(2).InfofCtx(ctx, "existing %s is a file", oldEntry.FullPath)
+			return fmt.Errorf("%s: %w", oldEntry.FullPath, filer_pb.ErrExistingIsFile)
+		}
+	}
+	if entry.Attr.Atime.IsZero() {
+		entry.Attr.Atime = entryInitialAtime(entry.Attr)
+	}
+	return f.Store.UpdateEntry(ctx, entry)
+}
+
+func entryInitialAtime(attr Attr) time.Time {
+	if !attr.Mtime.IsZero() {
+		return attr.Mtime
+	}
+	return attr.Crtime
+}
+
+var (
+	Root = &Entry{
+		FullPath: "/",
+		Attr: Attr{
+			Mtime:  time.Now(),
+			Crtime: time.Now(),
+			Mode:   os.ModeDir | 0755,
+			Uid:    OS_UID,
+			Gid:    OS_GID,
+		},
+	}
+)
+
+func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, err error) {
+
+	if string(p) == "/" {
+		return Root, nil
+	}
+	entry, err = f.Store.FindEntry(ctx, p)
+	// A directory is deleted here one row at a time, which would strand whatever is
+	// under it, so a TTL an older build left on one is not acted on.
+	if entry != nil && entry.TtlSec > 0 && !entry.IsDirectory() {
+		if entry.IsExpireS3Enabled() {
+			if entry.GetS3ExpireTime().Before(time.Now()) && !entry.IsS3Versioning() {
+				if delErr := f.doDeleteEntryMetaAndData(ctx, entry, true, false, nil); delErr != nil {
+					glog.ErrorfCtx(ctx, "FindEntry doDeleteEntryMetaAndData %s failed: %v", entry.FullPath, delErr)
+				}
+				return nil, filer_pb.ErrNotFound
+			}
+		} else if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
+			f.Store.DeleteOneEntry(ctx, entry)
+			return nil, filer_pb.ErrNotFound
+		}
+	}
+
+	if entry == nil && (err == nil || errors.Is(err, filer_pb.ErrNotFound)) {
+		if lazy, lazyErr := f.maybeLazyFetchFromRemote(ctx, p); lazyErr != nil {
+			glog.V(1).InfofCtx(ctx, "FindEntry lazy fetch %s: %v", p, lazyErr)
+		} else if lazy != nil {
+			return lazy, nil
+		}
+	}
+
+	return entry, err
+}
+
+func (f *Filer) doListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (expiredCount int64, lastFileName string, err error) {
+	f.maybeLazyListFromRemote(ctx, p)
+
+	// Collect expired entries during iteration to avoid deadlock with DB connection pool
+	var expiredEntries []*Entry
+	var s3ExpiredEntries []*Entry
+	var hasValidEntries bool
+
+	lastFileName, err = f.Store.ListDirectoryPrefixedEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) (bool, error) {
+		select {
+		case <-ctx.Done():
+			glog.Errorf("Context is done.")
+			return false, fmt.Errorf("context canceled: %w", ctx.Err())
+		default:
+			if entry.TtlSec > 0 && !entry.IsDirectory() {
+				if entry.IsExpireS3Enabled() {
+					if entry.GetS3ExpireTime().Before(time.Now()) && !entry.IsS3Versioning() {
+						// Collect for deletion after iteration completes to avoid DB deadlock
+						s3ExpiredEntries = append(s3ExpiredEntries, entry)
+						expiredCount++
+						return true, nil
+					}
+				} else if entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
+					// Collect for deletion after iteration completes to avoid DB deadlock
+					expiredEntries = append(expiredEntries, entry)
+					expiredCount++
+					return true, nil
+				}
+			}
+			// Track that we found at least one valid (non-expired) entry
+			hasValidEntries = true
+			return eachEntryFunc(entry)
+		}
+	})
+	if err != nil {
+		return expiredCount, lastFileName, err
+	}
+
+	// Delete expired entries after iteration completes to avoid DB connection deadlock
+	if len(s3ExpiredEntries) > 0 || len(expiredEntries) > 0 {
+		for _, entry := range s3ExpiredEntries {
+			if delErr := f.doDeleteEntryMetaAndData(ctx, entry, true, false, nil); delErr != nil {
+				glog.ErrorfCtx(ctx, "doListDirectoryEntries doDeleteEntryMetaAndData %s failed: %v", entry.FullPath, delErr)
+			}
+		}
+		for _, entry := range expiredEntries {
+			if delErr := f.Store.DeleteOneEntry(ctx, entry); delErr != nil {
+				glog.ErrorfCtx(ctx, "doListDirectoryEntries DeleteOneEntry %s failed: %v", entry.FullPath, delErr)
+			}
+		}
+
+		// After expiring entries, the directory might be empty.
+		// Attempt to clean it up and any empty parent directories.
+		if !hasValidEntries && p != "/" && startFileName == "" {
+			stopAtPath := util.FullPath(f.DirBucketsPath)
+			f.DeleteEmptyParentDirectories(ctx, p, stopAtPath)
+		}
+	}
+
+	return
+}
+
+// DeleteEmptyParentDirectories recursively checks and deletes parent directories if they become empty.
+// It stops at root "/" or at stopAtPath (if provided).
+// This is useful for cleaning up directories after deleting files or expired entries.
+//
+// IMPORTANT: For safety, dirPath must be under stopAtPath (when stopAtPath is provided).
+// This prevents accidental deletion of directories outside the intended scope (e.g., outside bucket paths).
+//
+// Example usage:
+//
+//	// After deleting /bucket/dir/subdir/file.txt, clean up empty parent directories
+//	// but stop at the bucket path
+//	parentPath := util.FullPath("/bucket/dir/subdir")
+//	filer.DeleteEmptyParentDirectories(ctx, parentPath, util.FullPath("/bucket"))
+//
+// Example with gRPC client:
+//
+//	if err := pb_filer_client.WithFilerClient(ctx, func(client filer_pb.SeaweedFilerClient) error {
+//	    return filer_pb.Traverse(ctx, filer, parentPath, "", func(entry *filer_pb.Entry) error {
+//	        // Process entries...
+//	    })
+//	}); err == nil {
+//	    filer.DeleteEmptyParentDirectories(ctx, parentPath, stopPath)
+//	}
+func (f *Filer) DeleteEmptyParentDirectories(ctx context.Context, dirPath util.FullPath, stopAtPath util.FullPath) {
+	if dirPath == "/" || dirPath == stopAtPath {
+		return
+	}
+
+	// Safety check: if stopAtPath is provided, dirPath must be under it (root "/" allows everything)
+	stopStr := string(stopAtPath)
+	if stopAtPath != "" && stopStr != "/" && !strings.HasPrefix(string(dirPath)+"/", stopStr+"/") {
+		glog.V(1).InfofCtx(ctx, "DeleteEmptyParentDirectories: %s is not under %s, skipping", dirPath, stopAtPath)
+		return
+	}
+
+	// Additional safety: prevent deletion of bucket-level directories
+	// This protects /buckets/mybucket from being deleted even if empty
+	baseDepth := strings.Count(f.DirBucketsPath, "/")
+	dirDepth := strings.Count(string(dirPath), "/")
+	if dirDepth <= baseDepth+1 {
+		glog.V(2).InfofCtx(ctx, "DeleteEmptyParentDirectories: skipping deletion of bucket-level directory %s", dirPath)
+		return
+	}
+
+	// Check if directory is empty
+	isEmpty, err := f.IsDirectoryEmpty(ctx, dirPath)
+	if err != nil {
+		glog.V(3).InfofCtx(ctx, "DeleteEmptyParentDirectories: error checking %s: %v", dirPath, err)
+		return
+	}
+
+	if !isEmpty {
+		// Directory is not empty, stop checking upward
+		glog.V(3).InfofCtx(ctx, "DeleteEmptyParentDirectories: directory %s is not empty, stopping cleanup", dirPath)
+		return
+	}
+
+	// Directory is empty, try to delete it
+	glog.V(2).InfofCtx(ctx, "DeleteEmptyParentDirectories: deleting empty directory %s", dirPath)
+	parentDir, _ := dirPath.DirAndName()
+	if dirEntry, findErr := f.FindEntry(ctx, dirPath); findErr == nil {
+		if delErr := f.doDeleteEntryMetaAndData(ctx, dirEntry, false, false, nil); delErr == nil {
+			// Successfully deleted, continue checking upwards
+			f.DeleteEmptyParentDirectories(ctx, util.FullPath(parentDir), stopAtPath)
+		} else {
+			// Failed to delete, stop cleanup
+			glog.V(3).InfofCtx(ctx, "DeleteEmptyParentDirectories: failed to delete %s: %v", dirPath, delErr)
+		}
+	}
+}
+
+// IsDirectoryEmpty checks if a directory contains any entries
+func (f *Filer) IsDirectoryEmpty(ctx context.Context, dirPath util.FullPath) (bool, error) {
+	isEmpty := true
+	_, err := f.Store.ListDirectoryPrefixedEntries(ctx, dirPath, "", true, 1, "", func(entry *Entry) (bool, error) {
+		isEmpty = false
+		return false, nil // Stop after first entry
+	})
+	return isEmpty, err
+}
+
+func (f *Filer) Shutdown() {
+	close(f.deletionQuit)
+	if f.EmptyFolderCleaner != nil {
+		f.EmptyFolderCleaner.Stop()
+	}
+	f.LocalMetaLogBuffer.ShutdownLogBuffer()
+	f.Store.Shutdown()
+}
+
+func (f *Filer) GetEntryAttributes(ctx context.Context, p util.FullPath) (map[string][]byte, error) {
+	entry, err := f.FindEntry(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+	if entry == nil {
+		return nil, nil
+	}
+	return entry.Extended, nil
+}
+
+func (f *Filer) IsDirectoryKeyObject(ctx context.Context, p util.FullPath) (bool, error) {
+	entry, err := f.FindEntry(ctx, p)
+	if err != nil {
+		if errors.Is(err, filer_pb.ErrNotFound) {
+			return false, nil
+		}
+		return false, err
+	}
+	if entry == nil {
+		return false, nil
+	}
+	// Mirror filer_pb.Entry.IsDirectoryKeyObject so the cleaner keeps a promoted file's data.
+	_, isPrefixObject := entry.Extended[s3_constants.SeaweedFSPrefixObject]
+	return entry.IsDirectory() && (entry.Mime != "" || len(entry.GetChunks()) > 0 || len(entry.Content) > 0 || entry.IsInRemoteOnly() || isPrefixObject), nil
+}

weed/filer/filer_buckets.go

@@ -0,0 +1,21 @@
+package filer
+
+import (
+	"strings"
+)
+
+func (f *Filer) IsBucket(entry *Entry) bool {
+	if !entry.IsDirectory() {
+		return false
+	}
+	parent, dirName := entry.FullPath.DirAndName()
+	if parent != f.DirBucketsPath {
+		return false
+	}
+	if strings.HasPrefix(dirName, ".") {
+		return false
+	}
+
+	return true
+
+}

weed/filer/filer_conf.go

@@ -0,0 +1,548 @@
+package filer
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/viant/ptrie"
+	jsonpb "google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
+)
+
+const (
+	DirectoryEtcRoot      = "/etc/"
+	DirectoryEtcSeaweedFS = "/etc/seaweedfs"
+	DirectoryEtcRemote    = "/etc/remote"
+	FilerConfName         = "filer.conf"
+	IamConfigDirectory    = "/etc/iam"
+	IamIdentityFile       = "identity.json"
+	IamPoliciesFile       = "policies.json"
+)
+
+// FilerConfVersion is stamped into every configuration this build writes.
+// Version 0 predates worm presence: it was written with EmitUnpopulated, so every
+// rule carries an explicit "worm": false that meant nothing. Reading one back as an
+// override would silently lift worm off nested paths, so it is dropped to unset.
+const FilerConfVersion = 1
+
+type FilerConf struct {
+	rules ptrie.Trie[*filer_pb.FilerConf_PathConf]
+}
+
+func ReadFilerConf(filerGrpcAddress pb.ServerAddress, grpcDialOption grpc.DialOption, masterClient *wdclient.MasterClient) (*FilerConf, error) {
+	return ReadFilerConfFromFilers([]pb.ServerAddress{filerGrpcAddress}, grpcDialOption, masterClient)
+}
+
+// ReadFilerConfFromFilers reads filer configuration with multi-filer failover support
+func ReadFilerConfFromFilers(filerGrpcAddresses []pb.ServerAddress, grpcDialOption grpc.DialOption, masterClient *wdclient.MasterClient) (*FilerConf, error) {
+	var data []byte
+	if err := pb.WithOneOfGrpcFilerClients(false, filerGrpcAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		if masterClient != nil {
+			var buf bytes.Buffer
+			if err := ReadEntry(masterClient, client, DirectoryEtcSeaweedFS, FilerConfName, &buf); err != nil {
+				return err
+			}
+			data = buf.Bytes()
+			return nil
+		}
+		content, err := ReadInsideFiler(context.Background(), client, DirectoryEtcSeaweedFS, FilerConfName)
+		if err != nil {
+			return err
+		}
+		data = content
+		return nil
+	}); err != nil && err != filer_pb.ErrNotFound {
+		return nil, fmt.Errorf("read %s/%s: %v", DirectoryEtcSeaweedFS, FilerConfName, err)
+	}
+
+	fc := NewFilerConf()
+	if len(data) > 0 {
+		if err := fc.LoadFromBytes(data); err != nil {
+			return nil, fmt.Errorf("parse %s/%s: %v", DirectoryEtcSeaweedFS, FilerConfName, err)
+		}
+	}
+	return fc, nil
+}
+
+func NewFilerConf() (fc *FilerConf) {
+	fc = &FilerConf{
+		rules: ptrie.New[*filer_pb.FilerConf_PathConf](),
+	}
+	return fc
+}
+
+func (fc *FilerConf) loadFromFiler(filer *Filer) (err error) {
+	filerConfPath := util.NewFullPath(DirectoryEtcSeaweedFS, FilerConfName)
+	entry, err := filer.FindEntry(context.Background(), filerConfPath)
+	if err != nil {
+		if err == filer_pb.ErrNotFound {
+			return nil
+		}
+		glog.Errorf("read filer conf entry %s: %v", filerConfPath, err)
+		return
+	}
+
+	if len(entry.Content) > 0 {
+		return fc.LoadFromBytes(entry.Content)
+	}
+
+	return fc.loadFromChunks(filer, entry.Content, entry.GetChunks(), entry.Size())
+}
+
+func (fc *FilerConf) loadFromChunks(filer *Filer, content []byte, chunks []*filer_pb.FileChunk, size uint64) (err error) {
+	if len(content) == 0 {
+		content, err = filer.readEntry(chunks, size)
+		if err != nil {
+			glog.Errorf("read filer conf content: %v", err)
+			return
+		}
+	}
+
+	return fc.LoadFromBytes(content)
+}
+
+func (fc *FilerConf) LoadFromBytes(data []byte) (err error) {
+	conf := &filer_pb.FilerConf{}
+
+	if err := jsonpb.Unmarshal(data, conf); err != nil {
+		return err
+	}
+
+	return fc.doLoadConf(conf)
+}
+
+func (fc *FilerConf) doLoadConf(conf *filer_pb.FilerConf) (err error) {
+	for _, location := range conf.Locations {
+		if conf.Version < FilerConfVersion && location.Worm != nil && !*location.Worm {
+			location.Worm = nil
+		}
+		err = fc.SetLocationConf(location)
+		if err != nil {
+			// this is not recoverable
+			return nil
+		}
+	}
+	return nil
+}
+
+func (fc *FilerConf) GetLocationConf(locationPrefix string) (locConf *filer_pb.FilerConf_PathConf, found bool) {
+	return fc.rules.Get([]byte(locationPrefix))
+}
+
+func (fc *FilerConf) SetLocationConf(locConf *filer_pb.FilerConf_PathConf) (err error) {
+	err = fc.rules.Put([]byte(locConf.LocationPrefix), locConf)
+	if err != nil {
+		glog.Errorf("put location prefix: %v", err)
+	}
+	return
+}
+
+func (fc *FilerConf) AddLocationConf(locConf *filer_pb.FilerConf_PathConf) (err error) {
+	existingConf, found := fc.rules.Get([]byte(locConf.LocationPrefix))
+	if found {
+		mergePathConf(existingConf, locConf)
+		locConf = existingConf
+	}
+	err = fc.rules.Put([]byte(locConf.LocationPrefix), locConf)
+	if err != nil {
+		glog.Errorf("put location prefix: %v", err)
+	}
+	return
+}
+
+func (fc *FilerConf) DeleteLocationConf(locationPrefix string) {
+	rules := ptrie.New[*filer_pb.FilerConf_PathConf]()
+	fc.rules.Walk(func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
+		if string(key) == locationPrefix {
+			return true
+		}
+		key = bytes.Clone(key)
+		_ = rules.Put(key, value)
+		return true
+	})
+	fc.rules = rules
+}
+
+// emptyPathConf is a singleton for paths with no matching rules
+// Callers must NOT mutate the returned value
+var emptyPathConf = &filer_pb.FilerConf_PathConf{}
+
+func (fc *FilerConf) MatchStorageRule(path string) (pathConf *filer_pb.FilerConf_PathConf) {
+	// Convert once to avoid allocation in multi-match case
+	pathBytes := []byte(path)
+
+	// Fast path: check if any rules match before allocating
+	// This avoids allocation for paths with no configured rules (common case)
+	var firstMatch *filer_pb.FilerConf_PathConf
+	matchCount := 0
+
+	fc.rules.MatchPrefix(pathBytes, func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
+		matchCount++
+		if matchCount == 1 {
+			firstMatch = value
+			return true // continue to check for more matches
+		}
+		// Stop after 2 matches - we only need to know if there are multiple
+		return false
+	})
+
+	// No rules match - return singleton (callers must NOT mutate)
+	if matchCount == 0 {
+		return emptyPathConf
+	}
+
+	// Single rule matches - return directly (callers must NOT mutate)
+	if matchCount == 1 {
+		return firstMatch
+	}
+
+	// Multiple rules match - need to merge (allocate new)
+	pathConf = &filer_pb.FilerConf_PathConf{}
+	fc.rules.MatchPrefix(pathBytes, func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
+		mergePathConf(pathConf, value)
+		return true
+	})
+	return pathConf
+}
+
+// ClonePathConf creates a mutable copy of an existing PathConf.
+// Use this when you need to modify a config (e.g., before calling SetLocationConf).
+//
+// IMPORTANT: Keep in sync with filer_pb.FilerConf_PathConf fields.
+// When adding new fields to the protobuf, update this function accordingly.
+func ClonePathConf(src *filer_pb.FilerConf_PathConf) *filer_pb.FilerConf_PathConf {
+	if src == nil {
+		return &filer_pb.FilerConf_PathConf{}
+	}
+	var worm *bool
+	if src.Worm != nil {
+		worm = proto.Bool(*src.Worm)
+	}
+	return &filer_pb.FilerConf_PathConf{
+		LocationPrefix:           src.LocationPrefix,
+		Collection:               src.Collection,
+		Replication:              src.Replication,
+		Ttl:                      src.Ttl,
+		DiskType:                 src.DiskType,
+		Fsync:                    src.Fsync,
+		VolumeGrowthCount:        src.VolumeGrowthCount,
+		ReadOnly:                 src.ReadOnly,
+		MaxFileNameLength:        src.MaxFileNameLength,
+		DataCenter:               src.DataCenter,
+		Rack:                     src.Rack,
+		DataNode:                 src.DataNode,
+		DisableChunkDeletion:     src.DisableChunkDeletion,
+		Worm:                     worm,
+		WormGracePeriodSeconds:   src.WormGracePeriodSeconds,
+		WormRetentionTimeSeconds: src.WormRetentionTimeSeconds,
+	}
+}
+
+// ApplyBucketQuotaReadOnly sets read-only when usedSize exceeds quota and clears it
+// once back under, reporting whether the flag changed. A non-positive quota is left
+// untouched so a manually locked bucket is never reopened.
+func (fc *FilerConf) ApplyBucketQuotaReadOnly(locationPrefix string, usedSize, quota float64) (readOnly, changed bool) {
+	if quota <= 0 {
+		return fc.MatchStorageRule(locationPrefix).ReadOnly, false
+	}
+
+	locConf := ClonePathConf(fc.MatchStorageRule(locationPrefix))
+	locConf.LocationPrefix = locationPrefix
+	wasReadOnly := locConf.ReadOnly
+
+	if wasReadOnly {
+		if usedSize < quota {
+			locConf.ReadOnly = false
+		}
+	} else {
+		if usedSize > quota {
+			locConf.ReadOnly = true
+		}
+	}
+
+	if locConf.ReadOnly == wasReadOnly {
+		return wasReadOnly, false
+	}
+	fc.SetLocationConf(locConf)
+	return locConf.ReadOnly, true
+}
+
+// ClearReadOnly clears the read-only flag on the rule at exactly locationPrefix,
+// reporting whether the flag was set. This is the explicit unlock for a flag that
+// ApplyBucketQuotaReadOnly can no longer clear once the quota is gone.
+func (fc *FilerConf) ClearReadOnly(locationPrefix string) (changed bool) {
+	locConf, found := fc.GetLocationConf(locationPrefix)
+	if !found || !locConf.ReadOnly {
+		return false
+	}
+	locConf.ReadOnly = false
+	fc.SetLocationConf(locConf)
+	return true
+}
+
+// ClearBucketReadOnly lifts the read-only flag that quota enforcement may have
+// left on the bucket's path rule, saving the updated configuration back to the
+// filer. It reports whether anything was cleared.
+func ClearBucketReadOnly(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketsPath, bucketName string) (changed bool, err error) {
+	data, err := ReadInsideFiler(ctx, client, DirectoryEtcSeaweedFS, FilerConfName)
+	if err == filer_pb.ErrNotFound || (err == nil && len(data) == 0) {
+		return false, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("read %s/%s: %v", DirectoryEtcSeaweedFS, FilerConfName, err)
+	}
+	fc := NewFilerConf()
+	if err = fc.LoadFromBytes(data); err != nil {
+		return false, fmt.Errorf("parse %s/%s: %v", DirectoryEtcSeaweedFS, FilerConfName, err)
+	}
+	// join the rule key exactly as s3.bucket.quota.enforce writes it, so the
+	// exact-match lookup finds the rule even for a non-canonical bucketsPath
+	if !fc.ClearReadOnly(bucketsPath + "/" + bucketName + "/") {
+		return false, nil
+	}
+	var buf bytes.Buffer
+	if err = fc.ToText(&buf); err != nil {
+		return false, err
+	}
+	if err = SaveInsideFiler(ctx, client, DirectoryEtcSeaweedFS, FilerConfName, buf.Bytes()); err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// filerConfSnapshot is a read of filer.conf plus enough of the entry (or its
+// absence) to make a follow-up write conditional via
+// saveFilerConfConditionally, instead of blindly overwriting whatever is
+// there by the time the write happens.
+type filerConfSnapshot struct {
+	fc    *FilerConf
+	entry *filer_pb.Entry // nil if filer.conf did not exist at read time
+}
+
+func readFilerConfSnapshot(ctx context.Context, client filer_pb.SeaweedFilerClient) (*filerConfSnapshot, error) {
+	resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
+		Directory: DirectoryEtcSeaweedFS,
+		Name:      FilerConfName,
+	})
+	fc := NewFilerConf()
+	if errors.Is(err, filer_pb.ErrNotFound) {
+		return &filerConfSnapshot{fc: fc}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read %s/%s: %v", DirectoryEtcSeaweedFS, FilerConfName, err)
+	}
+	if len(resp.Entry.Content) > 0 {
+		if err := fc.LoadFromBytes(resp.Entry.Content); err != nil {
+			return nil, fmt.Errorf("parse %s/%s: %v", DirectoryEtcSeaweedFS, FilerConfName, err)
+		}
+	}
+	return &filerConfSnapshot{fc: fc, entry: resp.Entry}, nil
+}
+
+// saveFilerConfConditionally writes snap.fc back to filer.conf, conditioned
+// on nothing having created or modified the file since snap was read. The
+// filer evaluates the condition under its per-path lock, so a racing writer
+// fails the precondition instead of silently losing its change.
+//
+// The condition is an exact content check (IF_ETAG_MATCH against the MD5
+// every writer stamps, see SaveInsideFiler), which mtime's one-second
+// resolution cannot match. A filer.conf written before that stamp existed
+// carries no hash, so that one write falls back to IF_UNMODIFIED_SINCE and
+// self-heals from the next write on.
+func saveFilerConfConditionally(ctx context.Context, client filer_pb.SeaweedFilerClient, snap *filerConfSnapshot) error {
+	var buf bytes.Buffer
+	if err := snap.fc.ToText(&buf); err != nil {
+		return err
+	}
+	content := buf.Bytes()
+	contentMd5 := md5.Sum(content)
+
+	if snap.entry == nil {
+		err := filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
+			Directory: DirectoryEtcSeaweedFS,
+			Entry: &filer_pb.Entry{
+				Name:        FilerConfName,
+				IsDirectory: false,
+				Attributes: &filer_pb.FuseAttributes{
+					Mtime:    time.Now().Unix(),
+					Crtime:   time.Now().Unix(),
+					FileMode: uint32(0644),
+					FileSize: uint64(len(content)),
+					Md5:      contentMd5[:],
+				},
+				Content: content,
+			},
+			Condition: &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{
+				{Kind: filer_pb.WriteCondition_IF_NOT_EXISTS},
+			}},
+		})
+		if err != nil {
+			return fmt.Errorf("create %s/%s: %w", DirectoryEtcSeaweedFS, FilerConfName, err)
+		}
+		return nil
+	}
+
+	entry := snap.entry
+	var condition *filer_pb.WriteCondition
+	if entry.Attributes != nil && len(entry.Attributes.Md5) > 0 {
+		condition = &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{
+			{Kind: filer_pb.WriteCondition_IF_ETAG_MATCH, Etags: []string{fmt.Sprintf("%x", entry.Attributes.Md5)}},
+		}}
+	} else {
+		var unmodifiedSince int64
+		if entry.Attributes != nil {
+			unmodifiedSince = entry.Attributes.Mtime
+		} else {
+			entry.Attributes = &filer_pb.FuseAttributes{}
+		}
+		condition = &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{
+			{Kind: filer_pb.WriteCondition_IF_UNMODIFIED_SINCE, UnixTime: unmodifiedSince},
+		}}
+	}
+
+	entry.Content = content
+	entry.Attributes.Mtime = time.Now().Unix()
+	entry.Attributes.FileSize = uint64(len(content))
+	entry.Attributes.Md5 = contentMd5[:]
+	err := filer_pb.UpdateEntry(ctx, client, &filer_pb.UpdateEntryRequest{
+		Directory: DirectoryEtcSeaweedFS,
+		Entry:     entry,
+		Condition: condition,
+	})
+	if err != nil {
+		return fmt.Errorf("update %s/%s: %w", DirectoryEtcSeaweedFS, FilerConfName, err)
+	}
+	return nil
+}
+
+// ClearBucketLifecycleDayTTLs removes any day-TTL filer.conf rules a legacy
+// PutBucketLifecycleConfiguration handler installed under the bucket's path.
+// Per-write TTL is now driven by the LifecycleTTLResolver built off the
+// stored lifecycle XML, so a lingering day-TTL rule would double-stamp
+// expiration (volume server expires under the old rule) or contradict a
+// newly saved XML. The write is conditioned on filer.conf being unchanged
+// since it was read here, so a concurrent writer fails this call rather than
+// silently losing one side's change.
+func ClearBucketLifecycleDayTTLs(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketsPath, bucketName, collection string) error {
+	snap, err := readFilerConfSnapshot(ctx, client)
+	if err != nil {
+		return err
+	}
+	if snap.entry == nil {
+		return nil
+	}
+
+	changed := false
+	bucketPrefix := fmt.Sprintf("%s/%s/", bucketsPath, bucketName)
+	for prefix, ttl := range snap.fc.GetCollectionTtls(collection) {
+		if !strings.HasPrefix(prefix, bucketPrefix) || !strings.HasSuffix(ttl, "d") {
+			continue
+		}
+		locConf, found := snap.fc.GetLocationConf(prefix)
+		if !found {
+			continue
+		}
+		// Logged either way: this is a one-way migration, and the prefix and
+		// TTL are all an operator needs to put a rule back by hand.
+		if isLifecycleOwnedPathConf(locConf) {
+			glog.V(0).Infof("lifecycle migration: dropping legacy day-TTL rule %s ttl=%s", prefix, ttl)
+			snap.fc.DeleteLocationConf(prefix)
+		} else {
+			glog.V(0).Infof("lifecycle migration: clearing legacy day-TTL %s on operator rule %s", ttl, prefix)
+			updated := ClonePathConf(locConf)
+			updated.Ttl = ""
+			if err := snap.fc.SetLocationConf(updated); err != nil {
+				return err
+			}
+		}
+		changed = true
+	}
+	if !changed {
+		return nil
+	}
+
+	return saveFilerConfConditionally(ctx, client, snap)
+}
+
+// isLifecycleOwnedPathConf reports whether a rule looks like one the removed
+// PutBucketLifecycleConfiguration add path created, which set only the
+// routing and TTL fields. That path merged onto whatever already sat at the
+// prefix, so anything else here - a disk type, WORM settings, a read-only
+// flag, a placement pin - is an operator's, and deleting the whole rule to
+// retire its TTL would take their configuration with it.
+func isLifecycleOwnedPathConf(c *filer_pb.FilerConf_PathConf) bool {
+	return c.GetDiskType() == "" && !c.GetFsync() && !c.GetReadOnly() &&
+		c.GetDataCenter() == "" && c.GetRack() == "" && c.GetDataNode() == "" &&
+		c.GetMaxFileNameLength() == 0 && !c.GetDisableChunkDeletion() &&
+		c.Worm == nil && c.GetWormGracePeriodSeconds() == 0 && c.GetWormRetentionTimeSeconds() == 0
+}
+
+func (fc *FilerConf) GetCollectionTtls(collection string) (ttls map[string]string) {
+	ttls = make(map[string]string)
+	fc.rules.Walk(func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
+		if value.Collection == collection {
+			ttls[value.LocationPrefix] = value.GetTtl()
+		}
+		return true
+	})
+	return ttls
+}
+
+// merge if values in b is not empty, merge them into a
+func mergePathConf(a, b *filer_pb.FilerConf_PathConf) {
+	a.Collection = util.Nvl(b.Collection, a.Collection)
+	a.Replication = util.Nvl(b.Replication, a.Replication)
+	a.Ttl = util.Nvl(b.Ttl, a.Ttl)
+	a.DiskType = util.Nvl(b.DiskType, a.DiskType)
+	a.Fsync = b.Fsync || a.Fsync
+	if b.VolumeGrowthCount > 0 {
+		a.VolumeGrowthCount = b.VolumeGrowthCount
+	}
+	a.ReadOnly = b.ReadOnly || a.ReadOnly
+	if b.MaxFileNameLength > 0 {
+		a.MaxFileNameLength = b.MaxFileNameLength
+	}
+	a.DataCenter = util.Nvl(b.DataCenter, a.DataCenter)
+	a.Rack = util.Nvl(b.Rack, a.Rack)
+	a.DataNode = util.Nvl(b.DataNode, a.DataNode)
+	a.DisableChunkDeletion = b.DisableChunkDeletion || a.DisableChunkDeletion
+	// worm merges on presence, so a nested rule can turn it off. readOnly, fsync and
+	// disableChunkDeletion stay OR'ed on purpose: a nested rule must not be able to
+	// lift a lock the bucket set.
+	if b.Worm != nil {
+		// copy the value: a is often a scratch conf while b is a live trie entry
+		a.Worm = proto.Bool(*b.Worm)
+	}
+	if b.WormRetentionTimeSeconds > 0 {
+		a.WormRetentionTimeSeconds = b.WormRetentionTimeSeconds
+	}
+	if b.WormGracePeriodSeconds > 0 {
+		a.WormGracePeriodSeconds = b.WormGracePeriodSeconds
+	}
+}
+
+func (fc *FilerConf) ToProto() *filer_pb.FilerConf {
+	m := &filer_pb.FilerConf{Version: FilerConfVersion}
+	fc.rules.Walk(func(key []byte, value *filer_pb.FilerConf_PathConf) bool {
+		m.Locations = append(m.Locations, value)
+		return true
+	})
+	return m
+}
+
+func (fc *FilerConf) ToText(writer io.Writer) error {
+	return ProtoToText(writer, fc.ToProto())
+}

weed/filer/filer_conf_test.go

@@ -0,0 +1,591 @@
+package filer
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"reflect"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/protobuf/proto"
+)
+
+func TestFilerConf(t *testing.T) {
+
+	fc := NewFilerConf()
+
+	conf := &filer_pb.FilerConf{Locations: []*filer_pb.FilerConf_PathConf{
+		{
+			LocationPrefix: "/buckets/abc",
+			Collection:     "abc",
+		},
+		{
+			LocationPrefix: "/buckets/abcd",
+			Collection:     "abcd",
+		},
+		{
+			LocationPrefix: "/buckets/",
+			Replication:    "001",
+		},
+		{
+			LocationPrefix: "/buckets",
+			ReadOnly:       false,
+		},
+		{
+			LocationPrefix: "/buckets/xxx",
+			ReadOnly:       true,
+		},
+		{
+			LocationPrefix: "/buckets/xxx/yyy",
+			ReadOnly:       false,
+		},
+	}}
+	fc.doLoadConf(conf)
+
+	assert.Equal(t, "abc", fc.MatchStorageRule("/buckets/abc/jasdf").Collection)
+	assert.Equal(t, "abcd", fc.MatchStorageRule("/buckets/abcd/jasdf").Collection)
+	assert.Equal(t, "001", fc.MatchStorageRule("/buckets/abc/jasdf").Replication)
+
+	assert.Equal(t, true, fc.MatchStorageRule("/buckets/xxx/yyy/zzz").ReadOnly)
+	assert.Equal(t, false, fc.MatchStorageRule("/buckets/other").ReadOnly)
+
+}
+
+func TestWormInheritance(t *testing.T) {
+	fc := NewFilerConf()
+	fc.doLoadConf(&filer_pb.FilerConf{
+		Version: FilerConfVersion,
+		Locations: []*filer_pb.FilerConf_PathConf{
+			{LocationPrefix: "/buckets/b/", Worm: proto.Bool(true), Ttl: "7d"},
+			{LocationPrefix: "/buckets/b/quiet/", Collection: "quiet"},
+			{LocationPrefix: "/buckets/b/scratch/", Worm: proto.Bool(false)},
+			{LocationPrefix: "/buckets/b/scratch/keep/", Worm: proto.Bool(true)},
+		},
+	})
+
+	// a rule that says nothing about worm keeps inheriting it, along with the ttl
+	rule := fc.MatchStorageRule("/buckets/b/quiet/x")
+	assert.True(t, rule.GetWorm())
+	assert.Equal(t, "7d", rule.Ttl)
+
+	// an explicit false turns it off, and a deeper rule turns it back on
+	assert.False(t, fc.MatchStorageRule("/buckets/b/scratch/x").GetWorm())
+	assert.True(t, fc.MatchStorageRule("/buckets/b/scratch/keep/x").GetWorm())
+
+	// paths with no rule at all are unaffected
+	assert.False(t, fc.MatchStorageRule("/buckets/other/x").GetWorm())
+}
+
+// TestWormLegacyFalseIsNotAnOverride pins that the explicit "worm": false every
+// version 0 configuration carries does not read as a per-path opt-out.
+func TestWormLegacyFalseIsNotAnOverride(t *testing.T) {
+	const conf = `{
+	  "locations": [
+	    {"locationPrefix": "/buckets/b/", "worm": true},
+	    {"locationPrefix": "/buckets/b/sub/", "collection": "sub", "worm": false}
+	  ]
+	}`
+
+	fc := NewFilerConf()
+	assert.NoError(t, fc.LoadFromBytes([]byte(conf)))
+	assert.True(t, fc.MatchStorageRule("/buckets/b/sub/x").GetWorm())
+
+	// the same file at the current version means what it says
+	fc = NewFilerConf()
+	assert.NoError(t, fc.LoadFromBytes([]byte(`{"version": 1,`+conf[1:])))
+	assert.False(t, fc.MatchStorageRule("/buckets/b/sub/x").GetWorm())
+}
+
+// TestWormSurvivesRoundTrip guards the write side: an explicit false has to be
+// stamped along with a version that says to honor it.
+func TestWormSurvivesRoundTrip(t *testing.T) {
+	fc := NewFilerConf()
+	fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/", Worm: proto.Bool(true)})
+	fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/sub/", Worm: proto.Bool(false)})
+	fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/other/", Ttl: "7d"})
+
+	var buf bytes.Buffer
+	assert.NoError(t, fc.ToText(&buf))
+
+	reloaded := NewFilerConf()
+	assert.NoError(t, reloaded.LoadFromBytes(buf.Bytes()))
+	assert.False(t, reloaded.MatchStorageRule("/buckets/b/sub/x").GetWorm())
+	assert.True(t, reloaded.MatchStorageRule("/buckets/b/other/x").GetWorm())
+}
+
+// TestClonePathConf verifies that ClonePathConf copies all exported fields.
+// Uses reflection to automatically detect new fields added to the protobuf,
+// ensuring the test fails if ClonePathConf is not updated for new fields.
+func TestClonePathConf(t *testing.T) {
+	// Create a fully-populated PathConf with non-zero values for all fields
+	src := &filer_pb.FilerConf_PathConf{
+		LocationPrefix:           "/test/path",
+		Collection:               "test_collection",
+		Replication:              "001",
+		Ttl:                      "7d",
+		DiskType:                 "ssd",
+		Fsync:                    true,
+		VolumeGrowthCount:        5,
+		ReadOnly:                 true,
+		MaxFileNameLength:        255,
+		DataCenter:               "dc1",
+		Rack:                     "rack1",
+		DataNode:                 "node1",
+		DisableChunkDeletion:     true,
+		Worm:                     proto.Bool(true),
+		WormGracePeriodSeconds:   3600,
+		WormRetentionTimeSeconds: 86400,
+	}
+
+	clone := ClonePathConf(src)
+
+	// Verify it's a different object
+	assert.NotSame(t, src, clone, "ClonePathConf should return a new object, not the same pointer")
+
+	// Use reflection to compare all exported fields
+	// This will automatically catch any new fields added to the protobuf
+	srcVal := reflect.ValueOf(src).Elem()
+	cloneVal := reflect.ValueOf(clone).Elem()
+	srcType := srcVal.Type()
+
+	for i := 0; i < srcType.NumField(); i++ {
+		field := srcType.Field(i)
+
+		// Skip unexported fields (protobuf internal fields like sizeCache, unknownFields)
+		if !field.IsExported() {
+			continue
+		}
+
+		srcField := srcVal.Field(i)
+		cloneField := cloneVal.Field(i)
+
+		// Compare field values
+		if !reflect.DeepEqual(srcField.Interface(), cloneField.Interface()) {
+			t.Errorf("Field %s not copied correctly: src=%v, clone=%v",
+				field.Name, srcField.Interface(), cloneField.Interface())
+		}
+	}
+
+	// Additionally verify that all exported fields in src are non-zero
+	// This ensures we're testing with fully populated data
+	for i := 0; i < srcType.NumField(); i++ {
+		field := srcType.Field(i)
+		if !field.IsExported() {
+			continue
+		}
+
+		srcField := srcVal.Field(i)
+		if srcField.IsZero() {
+			t.Errorf("Test setup error: field %s has zero value, update test to set a non-zero value", field.Name)
+		}
+	}
+
+	// Verify mutation of clone doesn't affect source
+	clone.Collection = "modified"
+	clone.ReadOnly = false
+	*clone.Worm = false
+	assert.Equal(t, "test_collection", src.Collection, "Modifying clone should not affect source Collection")
+	assert.Equal(t, true, src.ReadOnly, "Modifying clone should not affect source ReadOnly")
+	assert.Equal(t, true, src.GetWorm(), "Modifying clone should not affect source Worm")
+}
+
+func TestClonePathConfNil(t *testing.T) {
+	clone := ClonePathConf(nil)
+	assert.NotNil(t, clone, "ClonePathConf(nil) should return a non-nil empty PathConf")
+	assert.Equal(t, "", clone.LocationPrefix, "ClonePathConf(nil) should return empty PathConf")
+}
+
+func TestApplyBucketQuotaReadOnly(t *testing.T) {
+	const prefix = "/buckets/b/"
+
+	// over quota: flips to read-only
+	fc := NewFilerConf()
+	readOnly, changed := fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
+	assert.True(t, changed)
+	assert.True(t, readOnly)
+	assert.True(t, fc.MatchStorageRule(prefix).ReadOnly)
+
+	// still over quota: no change
+	_, changed = fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
+	assert.False(t, changed)
+
+	// back under quota: flips to writable
+	readOnly, changed = fc.ApplyBucketQuotaReadOnly(prefix, 50, 100)
+	assert.True(t, changed)
+	assert.False(t, readOnly)
+	assert.False(t, fc.MatchStorageRule(prefix).ReadOnly)
+
+	// quota disabled leaves the flag untouched, so manual locks survive
+	fc = NewFilerConf()
+	fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
+	readOnly, changed = fc.ApplyBucketQuotaReadOnly(prefix, 150, -1)
+	assert.False(t, changed)
+	assert.True(t, readOnly)
+
+	// under quota and not read-only: no rule churn
+	fc = NewFilerConf()
+	_, changed = fc.ApplyBucketQuotaReadOnly(prefix, 50, 100)
+	assert.False(t, changed)
+}
+
+func TestClearReadOnly(t *testing.T) {
+	const prefix = "/buckets/b/"
+
+	fc := NewFilerConf()
+	assert.False(t, fc.ClearReadOnly(prefix), "no rule to clear")
+
+	// locked by quota enforcement, then quota removed: still clearable
+	fc.ApplyBucketQuotaReadOnly(prefix, 150, 100)
+	assert.True(t, fc.ClearReadOnly(prefix))
+	assert.False(t, fc.MatchStorageRule(prefix).ReadOnly)
+	assert.False(t, fc.ClearReadOnly(prefix), "already writable")
+
+	// clearing the flag keeps the rule's other settings
+	fc = NewFilerConf()
+	fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: prefix, Ttl: "7d", ReadOnly: true})
+	assert.True(t, fc.ClearReadOnly(prefix))
+	rule := fc.MatchStorageRule(prefix)
+	assert.False(t, rule.ReadOnly)
+	assert.Equal(t, "7d", rule.Ttl)
+}
+
+// fakeFilerConfClient is a minimal in-memory filer_pb.SeaweedFilerClient that
+// only supports the single-file round trip ReadInsideFiler/SaveInsideFiler
+// need: lookup, create-if-absent, update. Embedding the interface satisfies
+// the rest of it; calling any other method panics on the nil embedded value.
+type fakeFilerConfClient struct {
+	filer_pb.SeaweedFilerClient
+
+	mu      sync.Mutex
+	entries map[string]*filer_pb.Entry // key: dir+"/"+name
+}
+
+func newFakeFilerConfClient() *fakeFilerConfClient {
+	return &fakeFilerConfClient{entries: make(map[string]*filer_pb.Entry)}
+}
+
+func (c *fakeFilerConfClient) key(dir, name string) string { return dir + "/" + name }
+
+func (c *fakeFilerConfClient) LookupDirectoryEntry(_ context.Context, in *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	e, ok := c.entries[c.key(in.Directory, in.Name)]
+	if !ok {
+		return nil, filer_pb.ErrNotFound
+	}
+	// A real gRPC round trip always hands back an independent copy (proto
+	// marshal/unmarshal), so mutating what the caller gets back (as
+	// saveFilerConfConditionally does before re-sending it) must not affect
+	// what conditionHoldsLocked below compares against.
+	return &filer_pb.LookupDirectoryEntryResponse{Entry: cloneFakeEntry(e)}, nil
+}
+
+func cloneFakeEntry(e *filer_pb.Entry) *filer_pb.Entry {
+	if e == nil {
+		return nil
+	}
+	// proto.Clone rather than a struct copy: filer_pb.Entry embeds
+	// protoimpl.MessageState (a sync.Mutex), which a plain `*e` copy would
+	// duplicate by value — exactly what go vet's copylocks check flags.
+	return proto.Clone(e).(*filer_pb.Entry)
+}
+
+func (c *fakeFilerConfClient) CreateEntry(_ context.Context, in *filer_pb.CreateEntryRequest, _ ...grpc.CallOption) (*filer_pb.CreateEntryResponse, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	key := c.key(in.Directory, in.Entry.Name)
+	if !conditionHoldsLocked(in.Condition, c.entries[key]) {
+		return nil, errors.New("precondition failed")
+	}
+	c.entries[key] = in.Entry
+	return &filer_pb.CreateEntryResponse{}, nil
+}
+
+func (c *fakeFilerConfClient) UpdateEntry(_ context.Context, in *filer_pb.UpdateEntryRequest, _ ...grpc.CallOption) (*filer_pb.UpdateEntryResponse, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	key := c.key(in.Directory, in.Entry.Name)
+	if !conditionHoldsLocked(in.Condition, c.entries[key]) {
+		return nil, errors.New("precondition failed")
+	}
+	c.entries[key] = in.Entry
+	return &filer_pb.UpdateEntryResponse{}, nil
+}
+
+// conditionHoldsLocked is a minimal stand-in for the filer server's
+// writeConditionSatisfied (weed/server/filer_grpc_server_condition.go),
+// covering just the clause kinds saveFilerConfConditionally uses, so tests
+// can exercise the CAS path without a real filer server.
+func conditionHoldsLocked(cond *filer_pb.WriteCondition, current *filer_pb.Entry) bool {
+	if cond == nil {
+		return true
+	}
+	for _, clause := range cond.Clauses {
+		switch clause.Kind {
+		case filer_pb.WriteCondition_IF_NOT_EXISTS:
+			if current != nil {
+				return false
+			}
+		case filer_pb.WriteCondition_IF_UNMODIFIED_SINCE:
+			if current != nil && current.Attributes != nil && current.Attributes.Mtime > clause.UnixTime {
+				return false
+			}
+		case filer_pb.WriteCondition_IF_ETAG_MATCH:
+			if current == nil {
+				return false
+			}
+			stored := ""
+			if current.Attributes != nil {
+				stored = fmt.Sprintf("%x", current.Attributes.Md5)
+			}
+			matched := false
+			for _, want := range clause.Etags {
+				if want == stored {
+					matched = true
+					break
+				}
+			}
+			if !matched {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+// putFilerConf seeds the fake client's filer.conf with the given rules.
+func putFilerConf(t *testing.T, client *fakeFilerConfClient, rules ...*filer_pb.FilerConf_PathConf) {
+	t.Helper()
+	fc := NewFilerConf()
+	for _, r := range rules {
+		require.NoError(t, fc.SetLocationConf(r))
+	}
+	var buf bytes.Buffer
+	require.NoError(t, fc.ToText(&buf))
+	client.entries[client.key(DirectoryEtcSeaweedFS, FilerConfName)] = &filer_pb.Entry{
+		Name:       FilerConfName,
+		Content:    buf.Bytes(),
+		Attributes: &filer_pb.FuseAttributes{},
+	}
+}
+
+// readFilerConfText returns the current filer.conf content stored in the fake
+// client, or "" if none exists yet.
+func readFilerConfText(client *fakeFilerConfClient) string {
+	e, ok := client.entries[client.key(DirectoryEtcSeaweedFS, FilerConfName)]
+	if !ok {
+		return ""
+	}
+	return string(e.Content)
+}
+
+func TestClearBucketLifecycleDayTTLs_NoFilerConf(t *testing.T) {
+	client := newFakeFilerConfClient()
+
+	require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
+	assert.Empty(t, readFilerConfText(client))
+}
+
+func TestClearBucketLifecycleDayTTLs_NoMatchingRule(t *testing.T) {
+	client := newFakeFilerConfClient()
+	putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/buckets/other/",
+		Collection:     "other",
+		Ttl:            "7d",
+	})
+
+	require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
+	assert.Contains(t, readFilerConfText(client), "other")
+}
+
+func TestClearBucketLifecycleDayTTLs_RemovesDayTTLUnderBucket(t *testing.T) {
+	client := newFakeFilerConfClient()
+	putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/buckets/mybucket/",
+		Collection:     "mybucket",
+		Ttl:            "7d",
+	})
+
+	require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
+
+	reloaded := NewFilerConf()
+	require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
+	_, found := reloaded.GetLocationConf("/buckets/mybucket/")
+	assert.False(t, found, "day-TTL rule should have been removed")
+}
+
+func TestClearBucketLifecycleDayTTLs_KeepsNonDayTTL(t *testing.T) {
+	client := newFakeFilerConfClient()
+	putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/buckets/mybucket/",
+		Collection:     "mybucket",
+		Ttl:            "7m", // minutes, not days: not a legacy lifecycle TTL rule
+	})
+
+	require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
+
+	reloaded := NewFilerConf()
+	require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
+	_, found := reloaded.GetLocationConf("/buckets/mybucket/")
+	assert.True(t, found, "non-day TTL rule should be left alone")
+}
+
+func TestClearBucketLifecycleDayTTLs_KeepsOperatorSettingsAndClearsOnlyTheTTL(t *testing.T) {
+	// The removed add path merged its TTL onto whatever already sat at the
+	// prefix, so a rule carrying operator settings must survive with only its
+	// TTL retired - deleting it outright would drop the disk type and the
+	// read-only flag with it.
+	client := newFakeFilerConfClient()
+	putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/buckets/mybucket/logs/",
+		Collection:     "mybucket",
+		Ttl:            "30d",
+		DiskType:       "hdd",
+		ReadOnly:       true,
+	})
+
+	require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
+
+	reloaded := NewFilerConf()
+	require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
+	rule, found := reloaded.GetLocationConf("/buckets/mybucket/logs/")
+	require.True(t, found, "a rule carrying operator settings must not be deleted")
+	assert.Equal(t, "", rule.Ttl, "the legacy day TTL should be gone")
+	assert.Equal(t, "hdd", rule.DiskType, "the operator's disk type must survive")
+	assert.True(t, rule.ReadOnly, "the operator's read-only flag must survive")
+}
+
+func TestClearBucketLifecycleDayTTLs_KeepsWormSettings(t *testing.T) {
+	worm := true
+	client := newFakeFilerConfClient()
+	putFilerConf(t, client, &filer_pb.FilerConf_PathConf{
+		LocationPrefix:           "/buckets/mybucket/locked/",
+		Collection:               "mybucket",
+		Ttl:                      "7d",
+		Worm:                     &worm,
+		WormRetentionTimeSeconds: 3600,
+	})
+
+	require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
+
+	reloaded := NewFilerConf()
+	require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
+	rule, found := reloaded.GetLocationConf("/buckets/mybucket/locked/")
+	require.True(t, found, "a WORM rule must not be deleted to retire a TTL")
+	assert.Equal(t, "", rule.Ttl)
+	assert.True(t, rule.GetWorm())
+	assert.Equal(t, uint64(3600), rule.WormRetentionTimeSeconds)
+}
+
+func TestClearBucketLifecycleDayTTLs_KeepsOtherBucketsAndCollections(t *testing.T) {
+	client := newFakeFilerConfClient()
+	putFilerConf(t, client,
+		&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/mybucket/", Collection: "mybucket", Ttl: "7d"},
+		&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/other/", Collection: "other", Ttl: "7d"},
+		// nested under the target bucket's path but tagged to a different
+		// collection (e.g. a filer-group-prefixed name): must not be swept up
+		// by this bucket's cleanup, since GetCollectionTtls filters by collection.
+		&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/mybucket/nested/", Collection: "group_mybucket", Ttl: "7d"},
+	)
+
+	require.NoError(t, ClearBucketLifecycleDayTTLs(context.Background(), client, "/buckets", "mybucket", "mybucket"))
+
+	reloaded := NewFilerConf()
+	require.NoError(t, reloaded.LoadFromBytes([]byte(readFilerConfText(client))))
+	_, found := reloaded.GetLocationConf("/buckets/mybucket/")
+	assert.False(t, found, "the target bucket's day-TTL rule should be gone")
+
+	text := readFilerConfText(client)
+	assert.Contains(t, text, "other")
+	assert.Contains(t, text, "group_mybucket")
+}
+
+// bumpMtime models a writer that landed after unmodifiedSince, so an
+// IF_UNMODIFIED_SINCE precondition built on that timestamp no longer holds.
+func bumpMtime(client *fakeFilerConfClient, unmodifiedSince int64) {
+	client.mu.Lock()
+	defer client.mu.Unlock()
+	e := client.entries[client.key(DirectoryEtcSeaweedFS, FilerConfName)]
+	if e == nil {
+		return
+	}
+	if e.Attributes == nil {
+		e.Attributes = &filer_pb.FuseAttributes{}
+	}
+	e.Attributes.Mtime = unmodifiedSince + 1
+}
+
+func TestSaveFilerConfConditionally_RejectsUpdateModifiedConcurrently(t *testing.T) {
+	client := newFakeFilerConfClient()
+	putFilerConf(t, client, &filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/a/", Collection: "a", Ttl: "7d"})
+
+	snap, err := readFilerConfSnapshot(context.Background(), client)
+	require.NoError(t, err)
+	require.NotNil(t, snap.entry)
+	require.Empty(t, snap.entry.Attributes.Md5, "test setup should model a pre-existing filer.conf with no stamped Md5")
+
+	// Someone else writes filer.conf after our read but before our write.
+	bumpMtime(client, snap.entry.Attributes.Mtime)
+
+	err = saveFilerConfConditionally(context.Background(), client, snap)
+	assert.Error(t, err, "expected a concurrent modification to fail the conditional write")
+}
+
+// TestSaveFilerConfConditionally_ExactContentCheckRejectsConcurrentModification
+// exercises the primary path: once an entry has been written through this
+// function (so it carries a content Md5), a later write with a stale
+// snapshot is rejected by the exact IF_ETAG_MATCH check even when the
+// concurrent writer happened to leave mtime unchanged — something a
+// mtime-only precondition would have missed.
+func TestSaveFilerConfConditionally_ExactContentCheckRejectsConcurrentModification(t *testing.T) {
+	client := newFakeFilerConfClient()
+	putFilerConf(t, client, &filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/a/", Collection: "a", Ttl: "7d"})
+
+	// First write through saveFilerConfConditionally stamps Md5, moving
+	// subsequent reads onto the exact-content check.
+	snap, err := readFilerConfSnapshot(context.Background(), client)
+	require.NoError(t, err)
+	require.NoError(t, saveFilerConfConditionally(context.Background(), client, snap))
+
+	snap2, err := readFilerConfSnapshot(context.Background(), client)
+	require.NoError(t, err)
+	require.NotEmpty(t, snap2.entry.Attributes.Md5, "expected Md5 to have been stamped by the prior write")
+
+	// A concurrent writer goes through SaveInsideFiler, the path every
+	// other filer.conf writer uses. It lands in the same wall-clock second,
+	// so only the content hash can catch it.
+	concurrentFc := NewFilerConf()
+	require.NoError(t, concurrentFc.LoadFromBytes([]byte(readFilerConfText(client))))
+	require.NoError(t, concurrentFc.SetLocationConf(&filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/buckets/other/", Collection: "other", Ttl: "3d",
+	}))
+	var buf bytes.Buffer
+	require.NoError(t, concurrentFc.ToText(&buf))
+	require.NoError(t, SaveInsideFiler(context.Background(), client, DirectoryEtcSeaweedFS, FilerConfName, buf.Bytes()))
+	client.entries[client.key(DirectoryEtcSeaweedFS, FilerConfName)].Attributes.Mtime = snap2.entry.Attributes.Mtime
+
+	err = saveFilerConfConditionally(context.Background(), client, snap2)
+	assert.Error(t, err, "expected the exact-content check to reject a write whose baseline content hash no longer matches")
+	assert.Contains(t, readFilerConfText(client), "/buckets/other/", "the concurrent writer's content must survive")
+}
+
+func TestSaveFilerConfConditionally_RejectsCreateWhenCreatedConcurrently(t *testing.T) {
+	client := newFakeFilerConfClient()
+
+	snap, err := readFilerConfSnapshot(context.Background(), client)
+	require.NoError(t, err)
+	require.Nil(t, snap.entry, "filer.conf should not exist yet")
+
+	// Someone else creates filer.conf after our read but before our write.
+	putFilerConf(t, client, &filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/", Collection: "b", Ttl: "3d"})
+
+	err = saveFilerConfConditionally(context.Background(), client, snap)
+	assert.Error(t, err, "expected a concurrent create to fail the conditional write")
+	assert.Contains(t, readFilerConfText(client), "/buckets/b/", "the concurrent writer's content must survive")
+}

weed/filer/filer_delete_collection_test.go

@@ -0,0 +1,183 @@
+package filer
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+)
+
+// collectionDeleteMaster is just enough of a master for a MasterClient to
+// consider itself connected, and records every CollectionDelete it is asked for.
+type collectionDeleteMaster struct {
+	master_pb.UnimplementedSeaweedServer
+	calls chan collectionDeleteCall
+}
+
+type collectionDeleteCall struct {
+	name string
+	// budget is the time the RPC arrived with, or 0 when it carried no deadline.
+	budget time.Duration
+}
+
+// KeepConnected is what MasterClient waits on before it reports a master: one
+// response is enough, then the stream idles until the server is torn down.
+func (m *collectionDeleteMaster) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error {
+	if _, err := stream.Recv(); err != nil {
+		return err
+	}
+	if err := stream.Send(&master_pb.KeepConnectedResponse{}); err != nil {
+		return err
+	}
+	<-stream.Context().Done()
+	return nil
+}
+
+func (m *collectionDeleteMaster) CollectionDelete(ctx context.Context, req *master_pb.CollectionDeleteRequest) (*master_pb.CollectionDeleteResponse, error) {
+	call := collectionDeleteCall{name: req.Name}
+	if deadline, ok := ctx.Deadline(); ok {
+		call.budget = time.Until(deadline)
+	}
+	m.calls <- call
+	return &master_pb.CollectionDeleteResponse{}, nil
+}
+
+// hookedStore is the stub store with one extra seam: a callback that runs once
+// an entry has actually been removed, so a test can interleave an event -- a
+// client hanging up, say -- between the store write and whatever follows it.
+type hookedStore struct {
+	*stubFilerStore
+	onDeleteEntry func(util.FullPath)
+}
+
+func (s *hookedStore) DeleteEntry(ctx context.Context, p util.FullPath) error {
+	if err := s.stubFilerStore.DeleteEntry(ctx, p); err != nil {
+		return err
+	}
+	if s.onDeleteEntry != nil {
+		s.onDeleteEntry(p)
+	}
+	return nil
+}
+
+// newFilerWithFakeMaster builds a filer backed by the stub store, with its
+// MasterClient connected to a fake master the test can observe.
+func newFilerWithFakeMaster(t *testing.T) (*Filer, *hookedStore, *collectionDeleteMaster) {
+	t.Helper()
+
+	master := &collectionDeleteMaster{calls: make(chan collectionDeleteCall, 4)}
+
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	grpcServer := grpc.NewServer()
+	master_pb.RegisterSeaweedServer(grpcServer, master)
+	go func() { _ = grpcServer.Serve(lis) }()
+	t.Cleanup(grpcServer.Stop)
+
+	_, port, err := net.SplitHostPort(lis.Addr().String())
+	if err != nil {
+		t.Fatalf("split listener address: %v", err)
+	}
+	masterAddress := pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", port))
+
+	mc := wdclient.NewMasterClient(
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		"test", cluster.FilerType, pb.ServerAddress("localhost:0"), "", "",
+		*pb.NewServiceDiscoveryFromMap(map[string]pb.ServerAddress{"m": masterAddress}),
+	)
+
+	connecting, stopConnecting := context.WithCancel(context.Background())
+	t.Cleanup(stopConnecting)
+	go mc.KeepConnectedToMaster(connecting)
+
+	waiting, cancelWaiting := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancelWaiting()
+	mc.WaitUntilConnected(waiting)
+	if waiting.Err() != nil {
+		t.Fatal("the master client never connected to the fake master")
+	}
+
+	store := &hookedStore{stubFilerStore: newStubFilerStore()}
+	f := &Filer{
+		DirBucketsPath:      "/buckets",
+		RemoteStorage:       NewFilerRemoteStorage(),
+		Store:               NewFilerStoreWrapper(store),
+		FilerConf:           NewFilerConf(),
+		MaxFilenameLength:   255,
+		MasterClient:        mc,
+		FileIdDeletionQueue: util.NewUnboundedQueue(),
+		deletionQuit:        make(chan struct{}),
+		LocalMetaLogBuffer: log_buffer.NewLogBuffer("test", time.Minute,
+			func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, func() {}),
+	}
+	return f, store, master
+}
+
+// Deleting a bucket entry deletes its collection, and the request hanging up
+// partway must not stop that: the bucket's metadata is already gone, so skipping
+// it strands the bucket's volumes with nothing left to come back for them. The
+// cleanup still has to carry a deadline of its own, or a master that is down
+// parks this handler and every client retry behind it parks another.
+//
+// The cancellation lands between the store removing the entry and the collection
+// delete, which is where a client disconnect actually bites: FilerStoreWrapper
+// refuses a context that is already dead on entry, so an up-front cancellation
+// fails the delete long before this point instead.
+func TestDeleteEntryMetaAndDataDeletesCollectionWhenTheRequestIsCancelledMidDelete(t *testing.T) {
+	f, store, master := newFilerWithFakeMaster(t)
+
+	const bucket = "bucket-a"
+	bucketPath := util.FullPath(f.DirBucketsPath + "/" + bucket)
+	if err := store.InsertEntry(context.Background(), &Entry{
+		FullPath: bucketPath,
+		Attr:     Attr{Mode: os.ModeDir | 0755},
+	}); err != nil {
+		t.Fatalf("seed the bucket entry: %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	// The client hangs up just as the bucket entry comes out of the store.
+	store.onDeleteEntry = func(util.FullPath) { cancel() }
+
+	if err := f.DeleteEntryMetaAndData(ctx, bucketPath, true, false, true, false, nil, 0); err != nil {
+		t.Fatalf("DeleteEntryMetaAndData: %v", err)
+	}
+	if ctx.Err() == nil {
+		t.Fatal("test setup: the request was never cancelled, so nothing was exercised")
+	}
+
+	var call collectionDeleteCall
+	select {
+	case call = <-master.calls:
+	case <-time.After(20 * time.Second):
+		t.Fatal("CollectionDelete never reached the master")
+	}
+	if call.name != bucket {
+		t.Errorf("master was asked to delete collection %q, want %q", call.name, bucket)
+	}
+	if call.budget <= 0 {
+		t.Error("CollectionDelete arrived with no deadline; a master that stops answering would hold this handler open")
+	}
+	if call.budget > collectionDeleteTimeout {
+		t.Errorf("CollectionDelete budget = %v, want at most collectionDeleteTimeout %v", call.budget, collectionDeleteTimeout)
+	}
+
+	if store.getEntry(string(bucketPath)) != nil {
+		t.Error("the bucket entry survived the delete")
+	}
+}

weed/filer/filer_delete_entry.go

@@ -0,0 +1,244 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const (
+	MsgFailDelNonEmptyFolder = "fail to delete non-empty folder"
+)
+
+// ErrNonEmptyFolder is a non-recursive delete refused because the folder still
+// has children. The marker leads the message the filer builds for it and is
+// never wrapped on the way out, so the path that follows it, which the client
+// chose, cannot forge one.
+var ErrNonEmptyFolder = errors.New(MsgFailDelNonEmptyFolder)
+
+// DeleteEntryError turns the text of DeleteEntryResponse.Error back into an
+// error carrying the condition the filer reported. Call it on the response
+// field, before formatting a path around it.
+func DeleteEntryError(msg string) error {
+	if strings.HasPrefix(msg, MsgFailDelNonEmptyFolder) {
+		return &deleteEntryError{msg: msg, cause: ErrNonEmptyFolder}
+	}
+	return errors.New(msg)
+}
+
+// IsNonEmptyFolderError is for callers holding a delete failure that has not
+// been wrapped yet: the sentinel when it survived, the leading marker when the
+// error only crossed the wire as text.
+func IsNonEmptyFolderError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return errors.Is(err, ErrNonEmptyFolder) || strings.HasPrefix(err.Error(), MsgFailDelNonEmptyFolder)
+}
+
+type deleteEntryError struct {
+	msg   string
+	cause error
+}
+
+func (e *deleteEntryError) Error() string { return e.msg }
+func (e *deleteEntryError) Unwrap() error { return e.cause }
+
+type OnChunksFunc func([]*filer_pb.FileChunk) error
+type OnHardLinkIdsFunc func([]HardLinkId) error
+
+func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isFromOtherCluster bool, signatures []int32, ifNotModifiedAfter int64) (err error) {
+	if p == "/" {
+		return nil
+	}
+
+	entry, findErr := f.FindEntry(ctx, p)
+	if findErr != nil {
+		return findErr
+	}
+	if ifNotModifiedAfter > 0 && entry.Attr.Mtime.Unix() > ifNotModifiedAfter {
+		return nil
+	}
+	isDeleteCollection := f.IsBucket(entry)
+	if entry.IsDirectory() {
+		// delete the folder children, not including the folder itself
+		err = f.doBatchDeleteFolderMetaAndData(ctx, entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks && !isDeleteCollection, isDeleteCollection, isFromOtherCluster, signatures, func(hardLinkIds []HardLinkId) error {
+			// A case not handled:
+			// what if the chunk is in a different collection?
+			if shouldDeleteChunks {
+				f.maybeDeleteHardLinks(ctx, hardLinkIds)
+			}
+			return nil
+		})
+		if err != nil {
+			glog.V(2).InfofCtx(ctx, "delete directory %s: %v", p, err)
+			if errors.Is(err, ErrNonEmptyFolder) {
+				return err
+			}
+			return fmt.Errorf("delete directory %s: %v", p, err)
+		}
+	}
+
+	// delete the file or folder
+	err = f.doDeleteEntryMetaAndData(ctx, entry, shouldDeleteChunks, isFromOtherCluster, signatures)
+	if err != nil {
+		return fmt.Errorf("delete file %s: %v", p, err)
+	}
+
+	if shouldDeleteChunks && !isDeleteCollection {
+		if len(entry.HardLinkId) != 0 && entry.HardLinkCounter > 1 {
+			// if the file is a hard link and there are other hard links, do not delete the chunks
+		} else {
+			f.DeleteChunks(ctx, p, entry.GetChunks())
+		}
+	}
+
+	if isDeleteCollection {
+		collectionName := entry.Name()
+		// the entry is already gone: a caller that hung up must not leave the
+		// collection behind, so this cleanup outlives the request -- bounded all
+		// the same, or a master that is down parks this handler indefinitely and
+		// every client retry behind it parks another
+		collectionCtx, cancelCollection := context.WithTimeout(context.WithoutCancel(ctx), collectionDeleteTimeout)
+		f.DoDeleteCollection(collectionCtx, collectionName)
+		cancelCollection()
+		// drop bucket-labeled series held by this process; the S3 gateway
+		// only cleans its own registry
+		stats.DeleteBucketMetrics(collectionName)
+	}
+
+	return nil
+}
+
+func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isDeletingBucket, isFromOtherCluster bool, signatures []int32, onHardLinkIdsFn OnHardLinkIdsFunc) (err error) {
+
+	//collect all the chunks of this layer and delete them together at the end
+	var chunksToDelete []*filer_pb.FileChunk
+	lastFileName := ""
+	includeLastFile := false
+	listedChildren := !isDeletingBucket || !f.Store.CanDropWholeBucket()
+	if listedChildren {
+		for {
+			entries, _, err := f.ListDirectoryEntries(ctx, entry.FullPath, lastFileName, includeLastFile, PaginationSize, "", "", "")
+			if err != nil {
+				glog.ErrorfCtx(ctx, "list folder %s: %v", entry.FullPath, err)
+				return fmt.Errorf("list folder %s: %v", entry.FullPath, err)
+			}
+			if lastFileName == "" && !isRecursive && len(entries) > 0 {
+				// only for first iteration in the loop
+				glog.V(2).InfofCtx(ctx, "deleting a folder %s has children: %+v ...", entry.FullPath, entries[0].Name())
+				return fmt.Errorf("%w: %s", ErrNonEmptyFolder, entry.FullPath)
+			}
+
+			for _, sub := range entries {
+				lastFileName = sub.Name()
+				if sub.IsDirectory() {
+					subIsDeletingBucket := f.IsBucket(sub)
+					err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, subIsDeletingBucket, isFromOtherCluster, nil, onHardLinkIdsFn)
+				} else {
+					if !isFromOtherCluster {
+						if _, remoteErr := f.maybeDeleteFromRemote(ctx, sub); remoteErr != nil {
+							glog.Warningf("remote delete child %s: %v", sub.FullPath, remoteErr)
+							if !ignoreRecursiveError {
+								err = remoteErr
+							}
+						}
+					}
+					if err != nil && !ignoreRecursiveError {
+						break
+					}
+					f.NotifyUpdateEvent(ctx, sub, nil, shouldDeleteChunks, isFromOtherCluster, nil)
+					if len(sub.HardLinkId) != 0 {
+						// hard link chunk data are deleted separately
+						err = onHardLinkIdsFn([]HardLinkId{sub.HardLinkId})
+					} else {
+						if shouldDeleteChunks {
+							chunksToDelete = append(chunksToDelete, sub.GetChunks()...)
+						}
+					}
+				}
+				if err != nil && !ignoreRecursiveError {
+					return err
+				}
+			}
+
+			if len(entries) < PaginationSize {
+				break
+			}
+		}
+	}
+
+	glog.V(3).InfofCtx(ctx, "deleting directory %v delete chunks: %v", entry.FullPath, shouldDeleteChunks)
+
+	// a non-recursive delete already proved the folder empty above, so sweeping the
+	// children now can only remove entries that raced in after that listing
+	if isRecursive || !listedChildren {
+		if storeDeletionErr := f.Store.DeleteFolderChildren(ctx, entry.FullPath); storeDeletionErr != nil {
+			return fmt.Errorf("filer store delete: %w", storeDeletionErr)
+		}
+	}
+
+	f.NotifyUpdateEvent(ctx, entry, nil, shouldDeleteChunks, isFromOtherCluster, signatures)
+	f.DeleteChunks(ctx, entry.FullPath, chunksToDelete)
+
+	return nil
+}
+
+func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shouldDeleteChunks bool, isFromOtherCluster bool, signatures []int32) (err error) {
+
+	glog.V(3).InfofCtx(ctx, "deleting entry %v, delete chunks: %v", entry.FullPath, shouldDeleteChunks)
+
+	if !isFromOtherCluster {
+		if _, remoteDeletionErr := f.maybeDeleteFromRemote(ctx, entry); remoteDeletionErr != nil {
+			return remoteDeletionErr
+		}
+	}
+
+	if storeDeletionErr := f.Store.DeleteOneEntry(ctx, entry); storeDeletionErr != nil {
+		return fmt.Errorf("filer store delete: %w", storeDeletionErr)
+	}
+
+	if !entry.IsDirectory() {
+		f.NotifyUpdateEvent(ctx, entry, nil, shouldDeleteChunks, isFromOtherCluster, signatures)
+	}
+
+	return nil
+}
+
+// collectionDeleteTimeout bounds the collection delete a bucket entry's own
+// delete leaves behind, which carries no deadline of its own. It bounds the wait
+// and not the work: the master keeps deleting on its own fan-out once asked, so
+// giving up costs the confirmation. Short enough that the S3 client waiting on
+// the bucket delete, which pays this and then the gateway's own follow-up
+// DeleteCollection, still has retry budget left.
+const collectionDeleteTimeout = 15 * time.Second
+
+func (f *Filer) DoDeleteCollection(ctx context.Context, collectionName string) (err error) {
+
+	return f.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {
+		_, err := client.CollectionDelete(ctx, &master_pb.CollectionDeleteRequest{
+			Name: collectionName,
+		})
+		if err != nil {
+			glog.Infof("delete collection %s: %v", collectionName, err)
+		}
+		return err
+	})
+
+}
+
+func (f *Filer) maybeDeleteHardLinks(ctx context.Context, hardLinkIds []HardLinkId) {
+	for _, hardLinkId := range hardLinkIds {
+		if err := f.Store.DeleteHardLink(ctx, hardLinkId); err != nil {
+			glog.ErrorfCtx(ctx, "delete hard link id %d : %v", hardLinkId, err)
+		}
+	}
+}

weed/filer/filer_delete_entry_test.go

@@ -0,0 +1,29 @@
+package filer
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+)
+
+// The path in a non-empty-folder failure is the client's, so the marker has to
+// lead the message for a caller reading it off the wire to trust it.
+func TestNonEmptyFolderClassification(t *testing.T) {
+	err := fmt.Errorf("%w: %s", ErrNonEmptyFolder, "/buckets/b/photos")
+	if !IsNonEmptyFolderError(err) {
+		t.Errorf("expected the sentinel to be recognized: %v", err)
+	}
+	if !errors.Is(DeleteEntryError(err.Error()), ErrNonEmptyFolder) {
+		t.Errorf("expected the wire text to classify: %v", err)
+	}
+
+	// an entry named after the marker cannot forge one: every other delete
+	// failure the filer builds leads with its own wrapper
+	spoofed := "delete file /buckets/b/" + MsgFailDelNonEmptyFolder + ": filer store delete: disk full"
+	if errors.Is(DeleteEntryError(spoofed), ErrNonEmptyFolder) {
+		t.Errorf("expected no forgery from the entry name: %v", spoofed)
+	}
+	if IsNonEmptyFolderError(errors.New(spoofed)) {
+		t.Errorf("expected no forgery from the entry name: %v", spoofed)
+	}
+}

weed/filer/filer_deletion.go

@@ -0,0 +1,637 @@
+package filer
+
+import (
+	"container/heap"
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+)
+
+const (
+	// Maximum number of retry attempts for failed deletions
+	MaxRetryAttempts = 10
+	// Initial retry delay (will be doubled with each attempt)
+	InitialRetryDelay = 5 * time.Minute
+	// Maximum retry delay
+	MaxRetryDelay = 6 * time.Hour
+	// Interval for checking retry queue for ready items
+	DeletionRetryPollInterval = 1 * time.Minute
+	// Maximum number of items to process per retry iteration
+	DeletionRetryBatchSize = 1000
+	// Maximum number of error details to include in log messages
+	MaxLoggedErrorDetails = 10
+	// Interval for polling the deletion queue for new items
+	// Using a prime number to de-synchronize with other periodic tasks
+	DeletionPollInterval = 1123 * time.Millisecond
+	// Maximum number of file IDs to delete per batch (roughly 20 bytes per file ID)
+	DeletionBatchSize = 100000
+)
+
+// retryablePatterns contains transient conditions specific to the deletion
+// pipeline, on top of the network and service failures util.IsTransientErrorMessage
+// already covers.
+//
+// Context cancellation counts as retryable here but not in util: this decides
+// whether to requeue the deletion, not whether to retry a call, and a cancelled
+// batch leaves the file still needing deletion.
+var retryablePatterns = []string{
+	"is read only",              // Volume temporarily read-only (tiering, maintenance)
+	"error reading from server", // Network I/O errors
+	"closed network connection", // Network connection closed unexpectedly
+	"timeout",                   // Operation timeout (network or server)
+	"deadline exceeded",         // Context deadline exceeded
+	"context canceled",          // Context cancellation
+	"lookup error",              // Volume lookup failures
+	"lookup failed",             // Volume server discovery issues
+	"too many requests",         // Rate limiting / backpressure
+	"try again",                 // Explicit retry suggestion
+}
+
+// DeletionRetryItem represents a file deletion that failed and needs to be retried
+type DeletionRetryItem struct {
+	FileId      string
+	RetryCount  int
+	NextRetryAt time.Time
+	LastError   string
+	heapIndex   int  // index in the heap (for heap.Interface)
+	inFlight    bool // true when item is being processed, prevents duplicate additions
+}
+
+// retryHeap implements heap.Interface for DeletionRetryItem
+// Items are ordered by NextRetryAt (earliest first)
+type retryHeap []*DeletionRetryItem
+
+// Compile-time assertion that retryHeap implements heap.Interface
+var _ heap.Interface = (*retryHeap)(nil)
+
+func (h retryHeap) Len() int { return len(h) }
+
+func (h retryHeap) Less(i, j int) bool {
+	return h[i].NextRetryAt.Before(h[j].NextRetryAt)
+}
+
+func (h retryHeap) Swap(i, j int) {
+	h[i], h[j] = h[j], h[i]
+	h[i].heapIndex = i
+	h[j].heapIndex = j
+}
+
+func (h *retryHeap) Push(x any) {
+	item := x.(*DeletionRetryItem)
+	item.heapIndex = len(*h)
+	*h = append(*h, item)
+}
+
+func (h *retryHeap) Pop() any {
+	old := *h
+	n := len(old)
+	item := old[n-1]
+	old[n-1] = nil      // avoid memory leak
+	item.heapIndex = -1 // mark as removed
+	*h = old[0 : n-1]
+	return item
+}
+
+// DeletionRetryQueue manages the queue of failed deletions that need to be retried.
+// Uses a min-heap ordered by NextRetryAt for efficient retrieval of ready items.
+//
+// LIMITATION: Current implementation stores retry queue in memory only.
+// On filer restart, all pending retries are lost. With MaxRetryDelay up to 6 hours,
+// process restarts during this window will cause retry state loss.
+//
+// TODO: Consider persisting retry queue to durable storage for production resilience:
+//   - Option 1: Leverage existing Filer store (KV operations)
+//   - Option 2: Periodic snapshots to disk with recovery on startup
+//   - Option 3: Write-ahead log for retry queue mutations
+//   - Trade-offs: Performance vs durability, complexity vs reliability
+//
+// For now, accepting in-memory storage as pragmatic initial implementation.
+// Lost retries will be eventually consistent as files remain in deletion queue.
+type DeletionRetryQueue struct {
+	heap      retryHeap
+	itemIndex map[string]*DeletionRetryItem // for O(1) lookup by FileId
+	lock      sync.Mutex
+}
+
+// NewDeletionRetryQueue creates a new retry queue
+func NewDeletionRetryQueue() *DeletionRetryQueue {
+	q := &DeletionRetryQueue{
+		heap:      make(retryHeap, 0),
+		itemIndex: make(map[string]*DeletionRetryItem),
+	}
+	heap.Init(&q.heap)
+	return q
+}
+
+// calculateBackoff calculates the exponential backoff delay for a given retry count.
+// Uses exponential backoff formula: InitialRetryDelay * 2^(retryCount-1)
+// The first retry (retryCount=1) uses InitialRetryDelay, second uses 2x, third uses 4x, etc.
+// Includes overflow protection and caps at MaxRetryDelay.
+func calculateBackoff(retryCount int) time.Duration {
+	// The first retry is attempt 1, but shift should start at 0
+	if retryCount <= 1 {
+		return InitialRetryDelay
+	}
+
+	shiftAmount := uint(retryCount - 1)
+
+	// time.Duration is an int64. A left shift of 63 or more will result in a
+	// negative number or zero. The multiplication can also overflow much earlier
+	// (around a shift of 25 for a 5-minute initial delay).
+	// The `delay <= 0` check below correctly catches all these overflow cases.
+	delay := InitialRetryDelay << shiftAmount
+
+	if delay <= 0 || delay > MaxRetryDelay {
+		return MaxRetryDelay
+	}
+
+	return delay
+}
+
+// AddOrUpdate adds a new failed deletion or updates an existing one
+// Time complexity: O(log N) for insertion/update
+func (q *DeletionRetryQueue) AddOrUpdate(fileId string, errorMsg string) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	// Check if item already exists (including in-flight items)
+	if item, exists := q.itemIndex[fileId]; exists {
+		// Item is already in the queue or being processed. Just update the error.
+		// The existing retry schedule should proceed.
+		// RetryCount is only incremented in RequeueForRetry when an actual retry is performed.
+		item.LastError = errorMsg
+		if item.inFlight {
+			glog.V(2).Infof("retry for %s in-flight: attempt %d, will preserve retry state", fileId, item.RetryCount)
+		} else {
+			glog.V(2).Infof("retry for %s already scheduled: attempt %d, next retry in %v", fileId, item.RetryCount, time.Until(item.NextRetryAt))
+		}
+		return
+	}
+
+	// Add new item
+	delay := InitialRetryDelay
+	item := &DeletionRetryItem{
+		FileId:      fileId,
+		RetryCount:  1,
+		NextRetryAt: time.Now().Add(delay),
+		LastError:   errorMsg,
+		inFlight:    false,
+	}
+	heap.Push(&q.heap, item)
+	q.itemIndex[fileId] = item
+	glog.V(2).Infof("added retry for %s: next retry in %v", fileId, delay)
+}
+
+// RequeueForRetry re-adds a previously failed item back to the queue with incremented retry count.
+// This method MUST be used when re-queuing items from processRetryBatch to preserve retry state.
+// Time complexity: O(log N) for insertion
+func (q *DeletionRetryQueue) RequeueForRetry(item *DeletionRetryItem, errorMsg string) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	// Increment retry count
+	item.RetryCount++
+	item.LastError = errorMsg
+
+	// Calculate next retry time with exponential backoff
+	delay := calculateBackoff(item.RetryCount)
+	item.NextRetryAt = time.Now().Add(delay)
+	item.inFlight = false // Clear in-flight flag
+	glog.V(2).Infof("requeued retry for %s: attempt %d, next retry in %v", item.FileId, item.RetryCount, delay)
+
+	// Re-add to heap (item still in itemIndex)
+	heap.Push(&q.heap, item)
+}
+
+// GetReadyItems returns items that are ready to be retried and marks them as in-flight
+// Time complexity: O(K log N) where K is the number of ready items
+// Items are processed in order of NextRetryAt (earliest first)
+func (q *DeletionRetryQueue) GetReadyItems(maxItems int) []*DeletionRetryItem {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	now := time.Now()
+	var readyItems []*DeletionRetryItem
+
+	// Peek at items from the top of the heap (earliest NextRetryAt)
+	for len(q.heap) > 0 && len(readyItems) < maxItems {
+		item := q.heap[0]
+
+		// If the earliest item is not ready yet, no other items are ready either
+		if item.NextRetryAt.After(now) {
+			break
+		}
+
+		// Remove from heap but keep in itemIndex with inFlight flag
+		heap.Pop(&q.heap)
+
+		if item.RetryCount <= MaxRetryAttempts {
+			item.inFlight = true // Mark as being processed
+			readyItems = append(readyItems, item)
+		} else {
+			// Max attempts reached, log and discard completely
+			delete(q.itemIndex, item.FileId)
+			glog.Warningf("max retry attempts (%d) reached for %s, last error: %s", MaxRetryAttempts, item.FileId, item.LastError)
+		}
+	}
+
+	return readyItems
+}
+
+// Remove removes an item from the queue (called when deletion succeeds or fails permanently)
+// Time complexity: O(1)
+func (q *DeletionRetryQueue) Remove(item *DeletionRetryItem) {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+
+	// Item was already removed from heap by GetReadyItems, just remove from index
+	delete(q.itemIndex, item.FileId)
+}
+
+// Size returns the current size of the retry queue
+func (q *DeletionRetryQueue) Size() int {
+	q.lock.Lock()
+	defer q.lock.Unlock()
+	return len(q.heap)
+}
+
+func LookupByMasterClientFn(masterClient *wdclient.MasterClient) func(vids []string) (map[string]*operation.LookupResult, error) {
+	return func(vids []string) (map[string]*operation.LookupResult, error) {
+		m := make(map[string]*operation.LookupResult)
+		for _, vid := range vids {
+			locs, _ := masterClient.GetVidLocations(vid)
+			var locations []operation.Location
+			for _, loc := range locs {
+				locations = append(locations, operation.Location{
+					Url:       loc.Url,
+					PublicUrl: loc.PublicUrl,
+					GrpcPort:  loc.GrpcPort,
+				})
+			}
+			m[vid] = &operation.LookupResult{
+				VolumeOrFileId: vid,
+				Locations:      locations,
+			}
+		}
+		return m, nil
+	}
+}
+
+func (f *Filer) loopProcessingDeletion() {
+
+	lookupFunc := LookupByMasterClientFn(f.MasterClient)
+
+	// Start retry processor in a separate goroutine
+	go f.loopProcessingDeletionRetry(lookupFunc)
+
+	ticker := time.NewTicker(DeletionPollInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-f.deletionQuit:
+			glog.V(0).Infof("deletion processor shutting down")
+			return
+		case <-ticker.C:
+			f.FileIdDeletionQueue.Consume(func(fileIds []string) {
+				for i := 0; i < len(fileIds); i += DeletionBatchSize {
+					end := i + DeletionBatchSize
+					if end > len(fileIds) {
+						end = len(fileIds)
+					}
+					toDeleteFileIds := fileIds[i:end]
+					f.processDeletionBatch(toDeleteFileIds, lookupFunc)
+				}
+			})
+		}
+	}
+}
+
+// processDeletionBatch handles deletion of a batch of file IDs and processes results.
+// It classifies errors into retryable and permanent categories, adds retryable failures
+// to the retry queue, and logs appropriate messages.
+func (f *Filer) processDeletionBatch(toDeleteFileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) {
+	// Deduplicate file IDs to prevent incorrect retry count increments for the same file ID within a single batch.
+	uniqueFileIdsSlice := make([]string, 0, len(toDeleteFileIds))
+	processed := make(map[string]struct{}, len(toDeleteFileIds))
+	for _, fileId := range toDeleteFileIds {
+		if _, found := processed[fileId]; !found {
+			processed[fileId] = struct{}{}
+			uniqueFileIdsSlice = append(uniqueFileIdsSlice, fileId)
+		}
+	}
+
+	if len(uniqueFileIdsSlice) == 0 {
+		return
+	}
+
+	// Delete files and classify outcomes
+	outcomes := deleteFilesAndClassify(f.GrpcDialOption, uniqueFileIdsSlice, lookupFunc)
+
+	// Process outcomes
+	var successCount, notFoundCount, retryableErrorCount, permanentErrorCount int
+	var errorDetails []string
+
+	for _, fileId := range uniqueFileIdsSlice {
+		outcome := outcomes[fileId]
+
+		switch outcome.status {
+		case deletionOutcomeSuccess:
+			successCount++
+		case deletionOutcomeNotFound:
+			notFoundCount++
+		case deletionOutcomeRetryable, deletionOutcomeNoResult:
+			retryableErrorCount++
+			f.DeletionRetryQueue.AddOrUpdate(fileId, outcome.errorMsg)
+			if len(errorDetails) < MaxLoggedErrorDetails {
+				errorDetails = append(errorDetails, fileId+": "+outcome.errorMsg+" (will retry)")
+			}
+		case deletionOutcomePermanent:
+			permanentErrorCount++
+			if len(errorDetails) < MaxLoggedErrorDetails {
+				errorDetails = append(errorDetails, fileId+": "+outcome.errorMsg+" (permanent)")
+			}
+		}
+	}
+
+	if successCount > 0 || notFoundCount > 0 {
+		glog.V(2).Infof("deleted %d files successfully, %d already deleted (not found)", successCount, notFoundCount)
+	}
+
+	totalErrors := retryableErrorCount + permanentErrorCount
+	if totalErrors > 0 {
+		logMessage := fmt.Sprintf("failed to delete %d/%d files (%d retryable, %d permanent)",
+			totalErrors, len(uniqueFileIdsSlice), retryableErrorCount, permanentErrorCount)
+		if len(errorDetails) > 0 {
+			if totalErrors > MaxLoggedErrorDetails {
+				logMessage += fmt.Sprintf(" (showing first %d)", len(errorDetails))
+			}
+			glog.V(0).Infof("%s: %v", logMessage, strings.Join(errorDetails, "; "))
+		} else {
+			glog.V(0).Info(logMessage)
+		}
+	}
+
+	if f.DeletionRetryQueue.Size() > 0 {
+		glog.V(2).Infof("retry queue size: %d", f.DeletionRetryQueue.Size())
+	}
+}
+
+const (
+	deletionOutcomeSuccess   = "success"
+	deletionOutcomeNotFound  = "not_found"
+	deletionOutcomeRetryable = "retryable"
+	deletionOutcomePermanent = "permanent"
+	deletionOutcomeNoResult  = "no_result"
+)
+
+// deletionOutcome represents the result of classifying deletion results for a file
+type deletionOutcome struct {
+	status   string // One of the deletionOutcome* constants
+	errorMsg string
+}
+
+// deleteFilesAndClassify performs deletion and classifies outcomes for a list of file IDs
+func deleteFilesAndClassify(grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) map[string]deletionOutcome {
+	// Perform deletion
+	results := operation.DeleteFileIdsWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)
+
+	// Group results by file ID to handle multiple results for replicated volumes
+	resultsByFileId := make(map[string][]*volume_server_pb.DeleteResult)
+	for _, result := range results {
+		resultsByFileId[result.FileId] = append(resultsByFileId[result.FileId], result)
+	}
+
+	// Classify outcome for each file
+	outcomes := make(map[string]deletionOutcome, len(fileIds))
+	for _, fileId := range fileIds {
+		outcomes[fileId] = classifyDeletionOutcome(fileId, resultsByFileId)
+	}
+
+	return outcomes
+}
+
+// classifyDeletionOutcome examines all deletion results for a file ID and determines the overall outcome
+// Uses a single pass through results with early return for permanent errors (highest priority)
+// Priority: Permanent > Retryable > Success > Not Found
+func classifyDeletionOutcome(fileId string, resultsByFileId map[string][]*volume_server_pb.DeleteResult) deletionOutcome {
+	fileIdResults, found := resultsByFileId[fileId]
+	if !found || len(fileIdResults) == 0 {
+		return deletionOutcome{
+			status:   deletionOutcomeNoResult,
+			errorMsg: "no deletion result from volume server",
+		}
+	}
+
+	var firstRetryableError string
+	hasSuccess := false
+
+	for _, res := range fileIdResults {
+		if res.Error == "" {
+			hasSuccess = true
+			continue
+		}
+		if strings.Contains(res.Error, storage.ErrorDeleted.Error()) || res.Error == "not found" {
+			continue
+		}
+
+		if isRetryableError(res.Error) {
+			if firstRetryableError == "" {
+				firstRetryableError = res.Error
+			}
+		} else {
+			// Permanent error takes highest precedence - return immediately
+			return deletionOutcome{status: deletionOutcomePermanent, errorMsg: res.Error}
+		}
+	}
+
+	if firstRetryableError != "" {
+		return deletionOutcome{status: deletionOutcomeRetryable, errorMsg: firstRetryableError}
+	}
+
+	if hasSuccess {
+		return deletionOutcome{status: deletionOutcomeSuccess, errorMsg: ""}
+	}
+
+	// If we are here, all results were "not found"
+	return deletionOutcome{status: deletionOutcomeNotFound, errorMsg: ""}
+}
+
+// isRetryableError determines if an error is retryable based on its message.
+//
+// Current implementation uses string matching which is brittle and may break
+// if error messages change in dependencies. This is acceptable for the initial
+// implementation but should be improved in the future.
+//
+// TODO: Consider these improvements for more robust error handling:
+//   - Pass DeleteResult instead of just error string to access Status codes
+//   - Use HTTP status codes (503 Service Unavailable, 429 Too Many Requests, etc.)
+//   - Implement structured error types that can be checked with errors.Is/errors.As
+//   - Extract and check gRPC status codes for better classification
+//   - Add error wrapping in the deletion pipeline to preserve error context
+//
+// For now, we use conservative string matching for known transient error patterns.
+func isRetryableError(errorMsg string) bool {
+	// Empty errors are not retryable
+	if errorMsg == "" {
+		return false
+	}
+
+	if util.IsTransientErrorMessage(errorMsg) {
+		return true
+	}
+
+	errorLower := strings.ToLower(errorMsg)
+	for _, pattern := range retryablePatterns {
+		if strings.Contains(errorLower, pattern) {
+			return true
+		}
+	}
+	return false
+}
+
+// loopProcessingDeletionRetry processes the retry queue for failed deletions
+func (f *Filer) loopProcessingDeletionRetry(lookupFunc func([]string) (map[string]*operation.LookupResult, error)) {
+
+	ticker := time.NewTicker(DeletionRetryPollInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-f.deletionQuit:
+			glog.V(0).Infof("retry processor shutting down, %d items remaining in queue", f.DeletionRetryQueue.Size())
+			return
+		case <-ticker.C:
+			// Process all ready items in batches until queue is empty
+			totalProcessed := 0
+			for {
+				readyItems := f.DeletionRetryQueue.GetReadyItems(DeletionRetryBatchSize)
+				if len(readyItems) == 0 {
+					break
+				}
+
+				f.processRetryBatch(readyItems, lookupFunc)
+				totalProcessed += len(readyItems)
+			}
+
+			if totalProcessed > 0 {
+				glog.V(1).Infof("retried deletion of %d files", totalProcessed)
+			}
+		}
+	}
+}
+
+// processRetryBatch attempts to retry deletion of files and processes results.
+// Successfully deleted items are removed from tracking, retryable failures are
+// re-queued with updated retry counts, and permanent errors are logged and discarded.
+func (f *Filer) processRetryBatch(readyItems []*DeletionRetryItem, lookupFunc func([]string) (map[string]*operation.LookupResult, error)) {
+	// Extract file IDs from retry items
+	fileIds := make([]string, 0, len(readyItems))
+	for _, item := range readyItems {
+		fileIds = append(fileIds, item.FileId)
+	}
+
+	// Delete files and classify outcomes
+	outcomes := deleteFilesAndClassify(f.GrpcDialOption, fileIds, lookupFunc)
+
+	// Process outcomes - iterate over readyItems to ensure all items are accounted for
+	var successCount, notFoundCount, retryCount, permanentErrorCount int
+	for _, item := range readyItems {
+		outcome := outcomes[item.FileId]
+
+		switch outcome.status {
+		case deletionOutcomeSuccess:
+			successCount++
+			f.DeletionRetryQueue.Remove(item) // Remove from queue (success)
+			glog.V(2).Infof("retry successful for %s after %d attempts", item.FileId, item.RetryCount)
+		case deletionOutcomeNotFound:
+			notFoundCount++
+			f.DeletionRetryQueue.Remove(item) // Remove from queue (already deleted)
+		case deletionOutcomeRetryable, deletionOutcomeNoResult:
+			retryCount++
+			if outcome.status == deletionOutcomeNoResult {
+				glog.Warningf("no deletion result for retried file %s, re-queuing to avoid loss", item.FileId)
+			}
+			f.DeletionRetryQueue.RequeueForRetry(item, outcome.errorMsg)
+		case deletionOutcomePermanent:
+			permanentErrorCount++
+			f.DeletionRetryQueue.Remove(item) // Remove from queue (permanent failure)
+			glog.Warningf("permanent error on retry for %s after %d attempts: %s", item.FileId, item.RetryCount, outcome.errorMsg)
+		}
+	}
+
+	if successCount > 0 || notFoundCount > 0 {
+		glog.V(1).Infof("retry: deleted %d files successfully, %d already deleted", successCount, notFoundCount)
+	}
+	if retryCount > 0 {
+		glog.V(1).Infof("retry: %d files still failing, will retry again later", retryCount)
+	}
+	if permanentErrorCount > 0 {
+		glog.Warningf("retry: %d files failed with permanent errors", permanentErrorCount)
+	}
+}
+
+func (f *Filer) DeleteUncommittedChunks(ctx context.Context, chunks []*filer_pb.FileChunk) {
+	f.doDeleteChunks(ctx, chunks)
+}
+
+func (f *Filer) DeleteChunks(ctx context.Context, fullpath util.FullPath, chunks []*filer_pb.FileChunk) {
+	rule := f.FilerConf.MatchStorageRule(string(fullpath))
+	if rule.DisableChunkDeletion {
+		return
+	}
+	f.doDeleteChunks(ctx, chunks)
+}
+
+func (f *Filer) doDeleteChunks(ctx context.Context, chunks []*filer_pb.FileChunk) {
+	for _, chunk := range chunks {
+		if !chunk.IsChunkManifest {
+			f.FileIdDeletionQueue.EnQueue(chunk.GetFileIdString())
+			continue
+		}
+		dataChunks, manifestResolveErr := ResolveOneChunkManifest(ctx, f.MasterClient.LookupFileId, chunk, f.MasterClient)
+		if manifestResolveErr != nil {
+			glog.V(0).InfofCtx(ctx, "failed to resolve manifest %s: %v", chunk.FileId, manifestResolveErr)
+		}
+		for _, dChunk := range dataChunks {
+			f.FileIdDeletionQueue.EnQueue(dChunk.GetFileIdString())
+		}
+		f.FileIdDeletionQueue.EnQueue(chunk.GetFileIdString())
+	}
+}
+
+func (f *Filer) DeleteChunksNotRecursive(chunks []*filer_pb.FileChunk) {
+	for _, chunk := range chunks {
+		f.FileIdDeletionQueue.EnQueue(chunk.GetFileIdString())
+	}
+}
+
+func (f *Filer) deleteChunksIfNotNew(ctx context.Context, oldEntry, newEntry *Entry) {
+	var oldChunks, newChunks []*filer_pb.FileChunk
+	if oldEntry != nil {
+		oldChunks = oldEntry.GetChunks()
+	}
+	if newEntry != nil {
+		newChunks = newEntry.GetChunks()
+	}
+
+	toDelete, err := MinusChunks(ctx, f.MasterClient.GetLookupFileIdFunction(), oldChunks, newChunks, f.MasterClient)
+	if err != nil {
+		glog.ErrorfCtx(ctx, "Failed to resolve old entry chunks when delete old entry chunks. new: %s, old: %s", newChunks, oldChunks)
+		return
+	}
+	f.DeleteChunksNotRecursive(toDelete)
+}

weed/filer/filer_deletion_test.go

@@ -0,0 +1,308 @@
+package filer
+
+import (
+	"container/heap"
+	"testing"
+	"time"
+)
+
+func TestDeletionRetryQueue_AddAndRetrieve(t *testing.T) {
+	queue := NewDeletionRetryQueue()
+
+	// Add items
+	queue.AddOrUpdate("file1", "is read only")
+	queue.AddOrUpdate("file2", "connection reset")
+
+	if queue.Size() != 2 {
+		t.Errorf("Expected queue size 2, got %d", queue.Size())
+	}
+
+	// Items not ready yet (initial delay is 5 minutes)
+	readyItems := queue.GetReadyItems(10)
+	if len(readyItems) != 0 {
+		t.Errorf("Expected 0 ready items, got %d", len(readyItems))
+	}
+
+	// Size should remain unchanged
+	if queue.Size() != 2 {
+		t.Errorf("Expected queue size 2 after checking ready items, got %d", queue.Size())
+	}
+}
+
+func TestDeletionRetryQueue_ExponentialBackoff(t *testing.T) {
+	queue := NewDeletionRetryQueue()
+
+	// Create an item
+	item := &DeletionRetryItem{
+		FileId:      "test-file",
+		RetryCount:  0,
+		NextRetryAt: time.Now(),
+		LastError:   "test error",
+	}
+
+	// Requeue multiple times to test backoff
+	delays := []time.Duration{}
+
+	for i := 0; i < 5; i++ {
+		beforeTime := time.Now()
+		queue.RequeueForRetry(item, "error")
+
+		// Calculate expected delay for this retry count
+		expectedDelay := InitialRetryDelay * time.Duration(1<<uint(i))
+		if expectedDelay > MaxRetryDelay {
+			expectedDelay = MaxRetryDelay
+		}
+
+		// Verify NextRetryAt is approximately correct
+		actualDelay := item.NextRetryAt.Sub(beforeTime)
+		delays = append(delays, actualDelay)
+
+		// Allow small timing variance
+		timeDiff := actualDelay - expectedDelay
+		if timeDiff < 0 {
+			timeDiff = -timeDiff
+		}
+		if timeDiff > 100*time.Millisecond {
+			t.Errorf("Retry %d: expected delay ~%v, got %v (diff: %v)", i+1, expectedDelay, actualDelay, timeDiff)
+		}
+
+		// Verify retry count incremented
+		if item.RetryCount != i+1 {
+			t.Errorf("Expected RetryCount %d, got %d", i+1, item.RetryCount)
+		}
+
+		// Reset the heap for the next isolated test iteration
+		queue.lock.Lock()
+		queue.heap = retryHeap{}
+		queue.lock.Unlock()
+	}
+
+	t.Logf("Exponential backoff delays: %v", delays)
+}
+
+func TestDeletionRetryQueue_OverflowProtection(t *testing.T) {
+	queue := NewDeletionRetryQueue()
+
+	// Create an item with very high retry count
+	item := &DeletionRetryItem{
+		FileId:      "test-file",
+		RetryCount:  60, // High count that would cause overflow without protection
+		NextRetryAt: time.Now(),
+		LastError:   "test error",
+	}
+
+	// Should not panic and should cap at MaxRetryDelay
+	queue.RequeueForRetry(item, "error")
+
+	delay := time.Until(item.NextRetryAt)
+	if delay > MaxRetryDelay+time.Second {
+		t.Errorf("Delay exceeded MaxRetryDelay: %v > %v", delay, MaxRetryDelay)
+	}
+}
+
+func TestDeletionRetryQueue_MaxAttemptsReached(t *testing.T) {
+	queue := NewDeletionRetryQueue()
+
+	// Add item
+	queue.AddOrUpdate("file1", "error")
+
+	// Manually set retry count to max
+	queue.lock.Lock()
+	item, exists := queue.itemIndex["file1"]
+	if !exists {
+		queue.lock.Unlock()
+		t.Fatal("Item not found in queue")
+	}
+	item.RetryCount = MaxRetryAttempts
+	item.NextRetryAt = time.Now().Add(-1 * time.Second) // Ready now
+	heap.Fix(&queue.heap, item.heapIndex)
+	queue.lock.Unlock()
+
+	// Try to get ready items - should be returned for the last retry (attempt #10)
+	readyItems := queue.GetReadyItems(10)
+	if len(readyItems) != 1 {
+		t.Fatalf("Expected 1 item for last retry, got %d", len(readyItems))
+	}
+
+	// Requeue it, which will increment its retry count beyond the max
+	queue.RequeueForRetry(readyItems[0], "final error")
+
+	// Manually make it ready again
+	queue.lock.Lock()
+	item, exists = queue.itemIndex["file1"]
+	if !exists {
+		queue.lock.Unlock()
+		t.Fatal("Item not found in queue after requeue")
+	}
+	item.NextRetryAt = time.Now().Add(-1 * time.Second)
+	heap.Fix(&queue.heap, item.heapIndex)
+	queue.lock.Unlock()
+
+	// Now it should be discarded (retry count is 11, exceeds max of 10)
+	readyItems = queue.GetReadyItems(10)
+	if len(readyItems) != 0 {
+		t.Errorf("Expected 0 items (max attempts exceeded), got %d", len(readyItems))
+	}
+
+	// Should be removed from queue
+	if queue.Size() != 0 {
+		t.Errorf("Expected queue size 0 after max attempts exceeded, got %d", queue.Size())
+	}
+}
+
+func TestCalculateBackoff(t *testing.T) {
+	testCases := []struct {
+		retryCount    int
+		expectedDelay time.Duration
+		description   string
+	}{
+		{1, InitialRetryDelay, "first retry"},
+		{2, InitialRetryDelay * 2, "second retry"},
+		{3, InitialRetryDelay * 4, "third retry"},
+		{4, InitialRetryDelay * 8, "fourth retry"},
+		{5, InitialRetryDelay * 16, "fifth retry"},
+		{10, MaxRetryDelay, "capped at max delay"},
+		{65, MaxRetryDelay, "overflow protection (shift > 63)"},
+		{100, MaxRetryDelay, "very high retry count"},
+	}
+
+	for _, tc := range testCases {
+		result := calculateBackoff(tc.retryCount)
+		if result != tc.expectedDelay {
+			t.Errorf("%s (retry %d): expected %v, got %v",
+				tc.description, tc.retryCount, tc.expectedDelay, result)
+		}
+	}
+}
+
+func TestIsRetryableError(t *testing.T) {
+	testCases := []struct {
+		error       string
+		retryable   bool
+		description string
+	}{
+		{"volume 123 is read only", true, "read-only volume"},
+		{"connection reset by peer", true, "connection reset"},
+		{"timeout exceeded", true, "timeout"},
+		{"deadline exceeded", true, "deadline exceeded"},
+		{"context canceled", true, "context canceled"},
+		{"lookup error: volume not found", true, "lookup error"},
+		{"connection refused", true, "connection refused"},
+		{"too many requests", true, "rate limiting"},
+		{"service unavailable", true, "service unavailable"},
+		{"i/o timeout", true, "I/O timeout"},
+		{"broken pipe", true, "broken pipe"},
+		{"not found", false, "not found (not retryable)"},
+		{"invalid file id", false, "invalid input (not retryable)"},
+		{"", false, "empty error"},
+	}
+
+	for _, tc := range testCases {
+		result := isRetryableError(tc.error)
+		if result != tc.retryable {
+			t.Errorf("%s: expected retryable=%v, got %v for error: %q",
+				tc.description, tc.retryable, result, tc.error)
+		}
+	}
+}
+
+func TestDeletionRetryQueue_HeapOrdering(t *testing.T) {
+	queue := NewDeletionRetryQueue()
+
+	now := time.Now()
+
+	// Add items with different retry times (out of order)
+	items := []*DeletionRetryItem{
+		{FileId: "file3", RetryCount: 1, NextRetryAt: now.Add(30 * time.Second), LastError: "error3"},
+		{FileId: "file1", RetryCount: 1, NextRetryAt: now.Add(10 * time.Second), LastError: "error1"},
+		{FileId: "file2", RetryCount: 1, NextRetryAt: now.Add(20 * time.Second), LastError: "error2"},
+	}
+
+	// Add items directly (simulating internal state)
+	for _, item := range items {
+		queue.lock.Lock()
+		queue.itemIndex[item.FileId] = item
+		queue.heap = append(queue.heap, item)
+		queue.lock.Unlock()
+	}
+
+	// Use container/heap.Init to establish heap property
+	queue.lock.Lock()
+	heap.Init(&queue.heap)
+	queue.lock.Unlock()
+
+	// Verify heap maintains min-heap property (earliest time at top)
+	queue.lock.Lock()
+	if queue.heap[0].FileId != "file1" {
+		t.Errorf("Expected file1 at heap top (earliest time), got %s", queue.heap[0].FileId)
+	}
+	queue.lock.Unlock()
+
+	// Set all items to ready while preserving their relative order
+	queue.lock.Lock()
+	for _, item := range queue.itemIndex {
+		// Shift all times back by 40 seconds to make them ready, but preserve order
+		item.NextRetryAt = item.NextRetryAt.Add(-40 * time.Second)
+	}
+	heap.Init(&queue.heap) // Re-establish heap property after modification
+	queue.lock.Unlock()
+
+	// GetReadyItems should return in NextRetryAt order
+	readyItems := queue.GetReadyItems(10)
+	expectedOrder := []string{"file1", "file2", "file3"}
+
+	if len(readyItems) != 3 {
+		t.Fatalf("Expected 3 ready items, got %d", len(readyItems))
+	}
+
+	for i, item := range readyItems {
+		if item.FileId != expectedOrder[i] {
+			t.Errorf("Item %d: expected %s, got %s", i, expectedOrder[i], item.FileId)
+		}
+	}
+}
+
+func TestDeletionRetryQueue_DuplicateFileIds(t *testing.T) {
+	queue := NewDeletionRetryQueue()
+
+	// Add same file ID twice with retryable error - simulates duplicate in batch
+	queue.AddOrUpdate("file1", "timeout error")
+
+	// Verify only one item exists in queue
+	if queue.Size() != 1 {
+		t.Fatalf("Expected queue size 1 after first add, got %d", queue.Size())
+	}
+
+	// Get initial retry count
+	queue.lock.Lock()
+	item1, exists := queue.itemIndex["file1"]
+	if !exists {
+		queue.lock.Unlock()
+		t.Fatal("Item not found in queue after first add")
+	}
+	initialRetryCount := item1.RetryCount
+	queue.lock.Unlock()
+
+	// Add same file ID again - should NOT increment retry count (just update error)
+	queue.AddOrUpdate("file1", "timeout error again")
+
+	// Verify still only one item exists in queue (not duplicated)
+	if queue.Size() != 1 {
+		t.Errorf("Expected queue size 1 after duplicate add, got %d (duplicates detected)", queue.Size())
+	}
+
+	// Verify retry count did NOT increment (AddOrUpdate only updates error, not count)
+	queue.lock.Lock()
+	item2, exists := queue.itemIndex["file1"]
+	queue.lock.Unlock()
+
+	if !exists {
+		t.Fatal("Item not found in queue after second add")
+	}
+	if item2.RetryCount != initialRetryCount {
+		t.Errorf("Expected RetryCount to stay at %d after duplicate add (should not increment), got %d", initialRetryCount, item2.RetryCount)
+	}
+	if item2.LastError != "timeout error again" {
+		t.Errorf("Expected LastError to be updated to 'timeout error again', got %q", item2.LastError)
+	}
+}

weed/filer/filer_hardlink.go

@@ -0,0 +1,16 @@
+package filer
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const (
+	HARD_LINK_MARKER = '\x01'
+)
+
+type HardLinkId []byte // 16 bytes + 1 marker byte
+
+func NewHardLinkId() HardLinkId {
+	bytes := append(util.RandomBytes(16), HARD_LINK_MARKER)
+	return bytes
+}

weed/filer/filer_inode.go

@@ -0,0 +1,26 @@
+package filer
+
+import (
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// ensureEntryInode derives a stable inode the same way the FUSE mount does, so
+// the filer-stored value matches what a mount would otherwise compute and no
+// per-object reverse index is required. Hard links hash their shared
+// HardLinkId, so every link resolves to one inode; other entries hash the path
+// and creation time.
+func (f *Filer) ensureEntryInode(entry *Entry) {
+	if entry == nil || entry.Attr.Inode != 0 {
+		return
+	}
+	if entry.Attr.Crtime.IsZero() {
+		entry.Attr.Crtime = time.Now()
+	}
+	if len(entry.HardLinkId) > 0 {
+		entry.Attr.Inode = uint64(util.HashStringToLong(string(entry.HardLinkId)))
+		return
+	}
+	entry.Attr.Inode = entry.FullPath.AsInode(entry.Attr.Crtime.Unix())
+}

weed/filer/filer_inode_test.go

@@ -0,0 +1,166 @@
+package filer
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestEnsureEntryInodeMatchesFuseDerivation(t *testing.T) {
+	f := &Filer{}
+	crtime := time.Unix(1700000000, 0)
+
+	entry := &Entry{
+		FullPath: util.FullPath("/dir/file.txt"),
+		Attr:     Attr{Crtime: crtime},
+	}
+	f.ensureEntryInode(entry)
+
+	// The filer stores exactly what the FUSE mount would compute for a
+	// non-hard-linked entry, and it is deterministic across calls.
+	assert.Equal(t, entry.FullPath.AsInode(crtime.Unix()), entry.Attr.Inode)
+	again := &Entry{FullPath: entry.FullPath, Attr: Attr{Crtime: crtime}}
+	f.ensureEntryInode(again)
+	assert.Equal(t, entry.Attr.Inode, again.Attr.Inode)
+}
+
+func TestEnsureEntryInodeSharesAcrossHardLinks(t *testing.T) {
+	f := &Filer{}
+	hardLinkId := NewHardLinkId()
+
+	a := &Entry{
+		FullPath:   util.FullPath("/links/a.txt"),
+		Attr:       Attr{Crtime: time.Unix(1700000000, 0)},
+		HardLinkId: hardLinkId,
+	}
+	b := &Entry{
+		FullPath:   util.FullPath("/links/b.txt"),
+		Attr:       Attr{Crtime: time.Unix(1800000000, 0)},
+		HardLinkId: hardLinkId,
+	}
+	f.ensureEntryInode(a)
+	f.ensureEntryInode(b)
+
+	// Every link to the same target resolves to one inode, independent of path
+	// or creation time.
+	assert.Equal(t, uint64(util.HashStringToLong(string(hardLinkId))), a.Attr.Inode)
+	assert.Equal(t, a.Attr.Inode, b.Attr.Inode)
+}
+
+func newTestFilerWithStubStore() (*Filer, *stubFilerStore) {
+	store := newStubFilerStore()
+	f := NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
+	f.Store = NewFilerStoreWrapper(store)
+	return f, store
+}
+
+func TestCreateEntryAssignsInodeWhenMissing(t *testing.T) {
+	f, store := newTestFilerWithStubStore()
+
+	entry := &Entry{
+		FullPath: util.FullPath("/dir/file.txt"),
+		Attr: Attr{
+			Mode: 0o644,
+		},
+	}
+
+	err := f.CreateEntry(context.Background(), entry, nil, false, false, nil, false, f.MaxFilenameLength)
+	require.NoError(t, err)
+
+	stored, findErr := store.FindEntry(context.Background(), entry.FullPath)
+	require.NoError(t, findErr)
+	require.NotNil(t, stored)
+	assert.NotZero(t, stored.Attr.Inode)
+	assert.NotEqual(t, uint64(1), stored.Attr.Inode)
+}
+
+func TestCreateEntryAssignsInodesToAutoCreatedParents(t *testing.T) {
+	f, store := newTestFilerWithStubStore()
+
+	entry := &Entry{
+		FullPath: util.FullPath("/a/b/c.txt"),
+		Attr: Attr{
+			Mode: 0o644,
+		},
+	}
+
+	err := f.CreateEntry(context.Background(), entry, nil, false, false, nil, false, f.MaxFilenameLength)
+	require.NoError(t, err)
+
+	for _, path := range []string{"/a", "/a/b", "/a/b/c.txt"} {
+		stored, findErr := store.FindEntry(context.Background(), util.FullPath(path))
+		require.NoError(t, findErr, path)
+		require.NotNil(t, stored, path)
+		assert.NotZero(t, stored.Attr.Inode, path)
+	}
+}
+
+func TestUpdateEntryPreservesExistingInode(t *testing.T) {
+	f, store := newTestFilerWithStubStore()
+
+	original := &Entry{
+		FullPath: util.FullPath("/doc.txt"),
+		Attr: Attr{
+			Mode:  0o644,
+			Inode: 12345,
+		},
+	}
+	require.NoError(t, store.InsertEntry(context.Background(), original))
+
+	updated := &Entry{
+		FullPath: util.FullPath("/doc.txt"),
+		Attr: Attr{
+			Mode: os.ModeDir | 0o755,
+		},
+	}
+
+	err := f.UpdateEntry(context.Background(), original, updated)
+	require.Error(t, err)
+
+	updated = &Entry{
+		FullPath: util.FullPath("/doc.txt"),
+		Attr: Attr{
+			Mode: 0o600,
+		},
+	}
+	err = f.UpdateEntry(context.Background(), original, updated)
+	require.NoError(t, err)
+
+	stored, findErr := store.FindEntry(context.Background(), original.FullPath)
+	require.NoError(t, findErr)
+	require.NotNil(t, stored)
+	assert.Equal(t, uint64(12345), stored.Attr.Inode)
+}
+
+func TestUpdateEntryBackfillsMissingLegacyInode(t *testing.T) {
+	f, store := newTestFilerWithStubStore()
+
+	original := &Entry{
+		FullPath: util.FullPath("/legacy.txt"),
+		Attr: Attr{
+			Mode: 0o644,
+		},
+	}
+	require.NoError(t, store.InsertEntry(context.Background(), original))
+
+	updated := &Entry{
+		FullPath: util.FullPath("/legacy.txt"),
+		Attr: Attr{
+			Mode: 0o640,
+		},
+	}
+	err := f.UpdateEntry(context.Background(), original, updated)
+	require.NoError(t, err)
+
+	stored, findErr := store.FindEntry(context.Background(), original.FullPath)
+	require.NoError(t, findErr)
+	require.NotNil(t, stored)
+	assert.NotZero(t, stored.Attr.Inode)
+	assert.NotEqual(t, uint64(1), stored.Attr.Inode)
+}

weed/filer/filer_lazy_remote.go

@@ -0,0 +1,173 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type lazyFetchContextKey struct{}
+
+// maybeLazyFetchFromRemote is called by FindEntry when the store returns no
+// entry for p. If p is under a remote-storage mount, it stats the remote
+// object, builds a filer Entry from the result, and persists it via
+// CreateEntry with SkipCheckParentDirectory so phantom parent directories
+// under the mount are not required.
+//
+// On a CreateEntry failure after a successful StatFile the in-memory entry is
+// still returned (availability over consistency); the singleflight key is
+// forgotten so the next lookup retries the filer write.
+//
+// Returns nil without error when: p is not under a remote mount; the remote
+// reports the object does not exist; or any other remote error occurs.
+func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (*Entry, error) {
+	// Prevent recursive invocation: CreateEntry calls FindEntry, which would
+	// re-enter this function and deadlock on the singleflight key.
+	if ctx.Value(lazyFetchContextKey{}) != nil {
+		return nil, nil
+	}
+
+	if f.RemoteStorage == nil {
+		return nil, nil
+	}
+
+	mountDir, remoteLoc := f.RemoteStorage.FindMountDirectory(p)
+	if remoteLoc == nil {
+		return nil, nil
+	}
+
+	remoteConf, found := f.RemoteStorage.FindRemoteStorageConf(p)
+	if !found {
+		return nil, nil
+	}
+
+	relPath := strings.TrimPrefix(string(p), string(mountDir))
+	if relPath != "" && !strings.HasPrefix(relPath, "/") {
+		relPath = "/" + relPath
+	}
+	base := strings.TrimSuffix(remoteLoc.Path, "/")
+	remotePath := "/" + strings.TrimLeft(base+relPath, "/")
+
+	objectLoc := &remote_pb.RemoteStorageLocation{
+		Name:   remoteLoc.Name,
+		Bucket: remoteLoc.Bucket,
+		Path:   remotePath,
+	}
+
+	type lazyFetchResult struct {
+		entry *Entry
+	}
+
+	key := string(p)
+	val, err, _ := f.lazyFetchGroup.Do(key, func() (interface{}, error) {
+		buildCtx := context.WithoutCancel(ctx)
+		client, clientErr := f.buildRemoteStorageClient(buildCtx, remoteConf)
+		if clientErr != nil {
+			glog.V(1).InfofCtx(ctx, "maybeLazyFetchFromRemote: reject %s: %v", p, clientErr)
+			return lazyFetchResult{nil}, nil
+		}
+		remoteEntry, statErr := client.StatFile(objectLoc)
+		if statErr != nil {
+			if errors.Is(statErr, remote_storage.ErrRemoteObjectNotFound) {
+				glog.V(3).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s not found in remote", p)
+			} else {
+				glog.Warningf("maybeLazyFetchFromRemote: stat %s failed: %v", p, statErr)
+			}
+			return lazyFetchResult{nil}, nil
+		}
+		if remoteEntry == nil {
+			glog.V(3).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s StatFile returned nil entry", p)
+			return lazyFetchResult{nil}, nil
+		}
+
+		mtime := time.Unix(remoteEntry.RemoteMtime, 0)
+		entry := &Entry{
+			FullPath: p,
+			Attr: Attr{
+				Mtime:    mtime,
+				Crtime:   mtime,
+				Mode:     0644,
+				FileSize: uint64(remoteEntry.RemoteSize),
+			},
+			Extended: MergeRemoteContentEncoding(remoteEntry, nil),
+			Remote:   remoteEntry,
+		}
+
+		persistBaseCtx, cancelPersist := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancelPersist()
+		persistCtx := context.WithValue(persistBaseCtx, lazyFetchContextKey{}, true)
+		saveErr := f.CreateEntry(persistCtx, entry, nil, false, false, nil, true, f.MaxFilenameLength)
+		if saveErr != nil {
+			glog.Warningf("maybeLazyFetchFromRemote: failed to persist filer entry for %s: %v", p, saveErr)
+			f.lazyFetchGroup.Forget(key)
+		}
+
+		return lazyFetchResult{entry}, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	result, ok := val.(lazyFetchResult)
+	if !ok {
+		return nil, fmt.Errorf("maybeLazyFetchFromRemote: unexpected singleflight result type %T for %s", val, p)
+	}
+	return result.entry, nil
+}
+
+func (f *Filer) maybeDeleteFromRemote(ctx context.Context, entry *Entry) (bool, error) {
+	if entry == nil || f.RemoteStorage == nil {
+		return false, nil
+	}
+
+	mountDir, remoteLoc := f.RemoteStorage.FindMountDirectory(entry.FullPath)
+	if remoteLoc == nil {
+		return false, nil
+	}
+
+	if !entry.IsDirectory() && entry.Remote == nil {
+		return false, nil
+	}
+
+	remoteConf, found := f.RemoteStorage.GetRemoteStorageConf(remoteLoc.Name)
+	if !found {
+		return false, fmt.Errorf("resolve remote storage client for %s: not found", entry.FullPath)
+	}
+	client, clientErr := f.buildRemoteStorageClient(ctx, remoteConf)
+	if clientErr != nil {
+		return false, fmt.Errorf("resolve remote storage client for %s: %w", entry.FullPath, clientErr)
+	}
+	if client == nil {
+		return false, fmt.Errorf("resolve remote storage client for %s: initialization failed", entry.FullPath)
+	}
+
+	objectLoc := MapFullPathToRemoteStorageLocation(mountDir, remoteLoc, entry.FullPath)
+
+	if entry.IsDirectory() {
+		if err := client.RemoveDirectory(objectLoc); err != nil {
+			if errors.Is(err, remote_storage.ErrRemoteObjectNotFound) {
+				return true, nil
+			}
+			return false, fmt.Errorf("remove remote directory %s: %w", entry.FullPath, err)
+		}
+		glog.V(3).InfofCtx(ctx, "maybeDeleteFromRemote: deleted directory %s from remote", entry.FullPath)
+		return true, nil
+	}
+
+	if err := client.DeleteFile(objectLoc); err != nil {
+		if errors.Is(err, remote_storage.ErrRemoteObjectNotFound) {
+			return true, nil
+		}
+		return false, fmt.Errorf("delete remote file %s: %w", entry.FullPath, err)
+	}
+
+	glog.V(3).InfofCtx(ctx, "maybeDeleteFromRemote: deleted %s from remote", entry.FullPath)
+	return true, nil
+}

weed/filer/filer_lazy_remote_listing.go

@@ -0,0 +1,215 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const xattrRemoteListingSyncedAt = "remote.listing.synced_at"
+
+type lazyListContextKey struct{}
+
+// maybeLazyListFromRemote populates the local filer store with entries from the
+// remote storage backend for directory p if the following conditions hold:
+//   - p is under a remote mount with listing_cache_ttl_seconds > 0
+//   - the cached listing has expired (based on the per-mount TTL)
+//
+// When listing_cache_ttl_seconds is 0 (the default), lazy listing is disabled
+// for that mount.
+//
+// On success it updates the directory's xattrRemoteListingSyncedAt extended
+// attribute so subsequent calls within the TTL window are no-ops.
+//
+// Errors are logged and swallowed (availability over consistency).
+func (f *Filer) maybeLazyListFromRemote(ctx context.Context, p util.FullPath) {
+	// Prevent recursion: CreateEntry → FindEntry → doListDirectoryEntries → here
+	if ctx.Value(lazyListContextKey{}) != nil {
+		return
+	}
+	// Also respect the lazy-fetch guard to prevent mutual recursion
+	if ctx.Value(lazyFetchContextKey{}) != nil {
+		return
+	}
+
+	if f.RemoteStorage == nil {
+		return
+	}
+
+	// The ptrie stores mount rules with trailing "/". When p is exactly the
+	// mount directory (e.g. "/buckets/mybucket"), we must also try matching
+	// with a trailing "/" so the trie recognizes the mount root.
+	lookupPath := p
+	mountDir, remoteLoc := f.RemoteStorage.FindMountDirectory(lookupPath)
+	if remoteLoc == nil {
+		lookupPath = util.FullPath(string(p) + "/")
+		mountDir, remoteLoc = f.RemoteStorage.FindMountDirectory(lookupPath)
+		if remoteLoc == nil {
+			return
+		}
+	}
+
+	// Lazy listing is opt-in: disabled when TTL is 0
+	if remoteLoc.ListingCacheTtlSeconds <= 0 {
+		return
+	}
+	cacheTTL := time.Duration(remoteLoc.ListingCacheTtlSeconds) * time.Second
+
+	// Check staleness: read the directory entry's extended attributes.
+	// Use Store.FindEntry directly — we only need the local xattr, not lazy-fetch.
+	dirEntry, _ := f.Store.FindEntry(ctx, p)
+	if dirEntry != nil {
+		if syncedAtStr, ok := dirEntry.Extended[xattrRemoteListingSyncedAt]; ok {
+			if syncedAt, err := strconv.ParseInt(string(syncedAtStr), 10, 64); err == nil {
+				if time.Since(time.Unix(syncedAt, 0)) < cacheTTL {
+					return
+				}
+			}
+		}
+	}
+
+	remoteConf, found := f.RemoteStorage.FindRemoteStorageConf(lookupPath)
+	if !found {
+		return
+	}
+
+	key := "list:" + string(p)
+	f.lazyListGroup.Do(key, func() (interface{}, error) {
+		client, clientErr := f.buildRemoteStorageClient(context.WithoutCancel(ctx), remoteConf)
+		if clientErr != nil {
+			glog.V(1).InfofCtx(ctx, "maybeLazyListFromRemote: reject %s: %v", p, clientErr)
+			return nil, nil
+		}
+		startTime := time.Now()
+		objectLoc := MapFullPathToRemoteStorageLocation(mountDir, remoteLoc, p)
+
+		// Decouple from the caller's cancellation/deadline while preserving
+		// trace/span values for distributed tracing.
+		persistCtx := context.WithValue(context.WithoutCancel(ctx), lazyListContextKey{}, true)
+		persistCtx = context.WithValue(persistCtx, lazyFetchContextKey{}, true)
+
+		listErr := client.ListDirectory(persistCtx, objectLoc, func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error {
+			childPath := p.Child(name)
+
+			existingEntry, findErr := f.Store.FindEntry(persistCtx, childPath)
+			if findErr != nil && !errors.Is(findErr, filer_pb.ErrNotFound) {
+				glog.Warningf("maybeLazyListFromRemote: find %s: %v", childPath, findErr)
+				return nil // skip this entry on transient store error
+			}
+
+			// Skip entries that exist locally without a RemoteEntry (local-only uploads)
+			if existingEntry != nil && existingEntry.Remote == nil {
+				return nil
+			}
+
+			if existingEntry != nil {
+				// Merge: update remote metadata while preserving local state
+				// (Chunks, Extended, Uid/Gid/Mode, etc.)
+				existingEntry.Remote = remoteEntry
+				if !isDirectory && remoteEntry != nil {
+					if remoteEntry.RemoteMtime > 0 {
+						existingEntry.Attr.Mtime = time.Unix(remoteEntry.RemoteMtime, 0)
+					}
+					existingEntry.Attr.FileSize = uint64(remoteEntry.RemoteSize)
+					existingEntry.Extended = MergeRemoteContentEncoding(remoteEntry, existingEntry.Extended)
+				}
+				if saveErr := f.Store.UpdateEntry(persistCtx, existingEntry); saveErr != nil {
+					glog.Warningf("maybeLazyListFromRemote: update %s: %v", childPath, saveErr)
+				}
+			} else {
+				// New entry not yet in local store
+				var entry *Entry
+				if isDirectory {
+					now := time.Now()
+					entry = &Entry{
+						FullPath: childPath,
+						Attr: Attr{
+							Mtime:  now,
+							Crtime: now,
+							Mode:   os.ModeDir | 0755,
+							Uid:    OS_UID,
+							Gid:    OS_GID,
+						},
+					}
+				} else {
+					mtime := time.Now()
+					if remoteEntry != nil && remoteEntry.RemoteMtime > 0 {
+						mtime = time.Unix(remoteEntry.RemoteMtime, 0)
+					}
+					entry = &Entry{
+						FullPath: childPath,
+						Attr: Attr{
+							Mtime:  mtime,
+							Crtime: mtime,
+							Mode:   0644,
+							Uid:    OS_UID,
+							Gid:    OS_GID,
+						},
+						Extended: MergeRemoteContentEncoding(remoteEntry, nil),
+						Remote:   remoteEntry,
+					}
+					if remoteEntry != nil {
+						entry.Attr.FileSize = uint64(remoteEntry.RemoteSize)
+					}
+				}
+				if saveErr := f.CreateEntry(persistCtx, entry, nil, false, false, nil, true, f.MaxFilenameLength); saveErr != nil {
+					glog.Warningf("maybeLazyListFromRemote: persist %s: %v", childPath, saveErr)
+				}
+			}
+			return nil
+		})
+		if listErr != nil {
+			glog.Warningf("maybeLazyListFromRemote: list %s: %v", p, listErr)
+			return nil, nil // swallow error
+		}
+
+		// Update the synced_at timestamp on the directory entry
+		f.updateDirectoryListingSyncedAt(persistCtx, p, startTime)
+
+		return nil, nil
+	})
+}
+
+func (f *Filer) updateDirectoryListingSyncedAt(ctx context.Context, p util.FullPath, syncTime time.Time) {
+	dirEntry, findErr := f.Store.FindEntry(ctx, p)
+	if findErr != nil && !errors.Is(findErr, filer_pb.ErrNotFound) {
+		glog.Warningf("maybeLazyListFromRemote: find dir %s: %v", p, findErr)
+		return
+	}
+	if errors.Is(findErr, filer_pb.ErrNotFound) {
+		// Directory doesn't exist yet, create it
+		now := time.Now()
+		dirEntry = &Entry{
+			FullPath: p,
+			Attr: Attr{
+				Mtime:  now,
+				Crtime: now,
+				Mode:   os.ModeDir | 0755,
+				Uid:    OS_UID,
+				Gid:    OS_GID,
+			},
+		}
+		if dirEntry.Extended == nil {
+			dirEntry.Extended = make(map[string][]byte)
+		}
+		dirEntry.Extended[xattrRemoteListingSyncedAt] = []byte(fmt.Sprintf("%d", syncTime.Unix()))
+		if saveErr := f.CreateEntry(ctx, dirEntry, nil, false, false, nil, true, f.MaxFilenameLength); saveErr != nil {
+			glog.Warningf("maybeLazyListFromRemote: create dir synced_at for %s: %v", p, saveErr)
+		}
+		return
+	}
+	if dirEntry.Extended == nil {
+		dirEntry.Extended = make(map[string][]byte)
+	}
+	dirEntry.Extended[xattrRemoteListingSyncedAt] = []byte(fmt.Sprintf("%d", syncTime.Unix()))
+	if saveErr := f.Store.UpdateEntry(ctx, dirEntry); saveErr != nil {
+		glog.Warningf("maybeLazyListFromRemote: update synced_at for %s: %v", p, saveErr)
+	}
+}

weed/filer/filer_notify.go

@@ -0,0 +1,433 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	nethttp "net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/notification"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) {
+	f.notifyUpdateEvent(ctx, oldEntry, newEntry, deleteChunks, isFromOtherCluster, signatures)
+}
+
+func (f *Filer) notifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) *filer_pb.SubscribeMetadataResponse {
+	if metadataEventsSuppressed(ctx) {
+		return nil
+	}
+
+	var fullpath string
+	if oldEntry != nil {
+		fullpath = string(oldEntry.FullPath)
+	} else if newEntry != nil {
+		fullpath = string(newEntry.FullPath)
+	} else {
+		return nil
+	}
+
+	// println("fullpath:", fullpath)
+
+	if strings.HasPrefix(fullpath, SystemLogDir) {
+		return nil
+	}
+	foundSelf := false
+	for _, sig := range signatures {
+		if sig == f.Signature {
+			foundSelf = true
+		}
+	}
+	if !foundSelf {
+		signatures = append(signatures, f.Signature)
+	}
+
+	event := f.newMetadataEvent(oldEntry, newEntry, deleteChunks, isFromOtherCluster, signatures)
+	// Clear the stamp after the buffer append below - deliberately also on
+	// append failure (see the metaLogInflight comment).
+	defer f.metaLogInflight.done(event.TsNs)
+	eventNotification := event.EventNotification
+
+	if notification.Queue != nil {
+		glog.V(3).Infof("notifying entry update %v", fullpath)
+		if err := notification.Queue.SendMessage(fullpath, eventNotification); err != nil {
+			// throw message
+			glog.Error(err)
+		}
+	}
+
+	f.logMetaEvent(ctx, event)
+	if sink := metadataEventSinkFromContext(ctx); sink != nil {
+		sink.Record(event)
+	}
+
+	f.onMetadataChangeEvent(event)
+
+	return event
+}
+
+// metaLogInflight tracks events stamped but not yet appended to the local
+// log buffer - the two are separated by notification work that can block,
+// and a claim ignoring that window would assert durability or delivery for
+// timestamps still on their way in. Stamping shares the reader's lock, so an
+// event is always visible here or (bumped monotonically) in the buffer.
+//
+// An event whose append fails also clears its stamp: it is dropped from the
+// change stream entirely (loudly logged there), and a watermark waiting for
+// it would pin this filer's claims forever.
+type metaLogInflight struct {
+	sync.Mutex
+	stamped     map[int64]int
+	lastStampNs int64
+	lastClaimNs int64
+}
+
+// stamp assigns the event timestamp and registers it as in flight. Stamps
+// are monotonic against the registry's own history and every issued claim,
+// so a wall-clock step backwards cannot slip a new stamp under a floor or
+// watermark already handed out.
+func (t *metaLogInflight) stamp() int64 {
+	t.Lock()
+	defer t.Unlock()
+	floor := t.lastStampNs
+	if t.lastClaimNs > floor {
+		floor = t.lastClaimNs
+	}
+	tsNs := time.Now().UnixNano()
+	if tsNs <= floor {
+		tsNs = floor + 1
+	}
+	t.lastStampNs = tsNs
+	if t.stamped == nil {
+		t.stamped = make(map[int64]int)
+	}
+	t.stamped[tsNs]++
+	return tsNs
+}
+
+// done removes a stamp once the event has been appended to the buffer.
+func (t *metaLogInflight) done(tsNs int64) {
+	t.Lock()
+	defer t.Unlock()
+	if t.stamped[tsNs] <= 1 {
+		delete(t.stamped, tsNs)
+	} else {
+		t.stamped[tsNs]--
+	}
+}
+
+// minTsNs returns the oldest in-flight stamp, or 0 when nothing is in flight.
+func (t *metaLogInflight) minTsNs() int64 {
+	t.Lock()
+	defer t.Unlock()
+	var min int64
+	for tsNs := range t.stamped {
+		if min == 0 || tsNs < min {
+			min = tsNs
+		}
+	}
+	return min
+}
+
+// claimThrough caps a completeness claim by the oldest in-flight stamp and
+// fences it: later stamps always land above the returned claim, so a wall
+// clock stepping backwards cannot slide a new event under a watermark a
+// peer has already advanced to.
+func (t *metaLogInflight) claimThrough(nowNs int64) int64 {
+	t.Lock()
+	defer t.Unlock()
+	claim := nowNs
+	for tsNs := range t.stamped {
+		if tsNs-1 < claim {
+			claim = tsNs - 1
+		}
+	}
+	if claim > t.lastClaimNs {
+		t.lastClaimNs = claim
+	}
+	return claim
+}
+
+// LocalFlushedThroughTsNs reports the timestamp through which the local meta
+// log is durably on disk: everything at or below it is appended and flushed,
+// and nothing can land at or below it later. The registry is consulted before
+// the buffer: an event already appended is visible to the buffer claim, one
+// still in flight caps the claim, and one stamped later is fenced above it.
+func (f *Filer) LocalFlushedThroughTsNs(nowNs int64) int64 {
+	claim := f.metaLogInflight.claimThrough(nowNs)
+	if buffered := f.LocalMetaLogBuffer.FlushedThroughTsNs(nowNs); buffered < claim {
+		claim = buffered
+	}
+	return claim
+}
+
+// LocalDeliveredThroughTsNs caps a delivery-freshness claim (an idle
+// heartbeat's timestamp) by the oldest in-flight stamp: a stamped-but-
+// unappended event has not been streamed to anyone, and a peer aggregator
+// turns the claim into its delivery low-watermark.
+func (f *Filer) LocalDeliveredThroughTsNs(nowNs int64) int64 {
+	return f.metaLogInflight.claimThrough(nowNs)
+}
+
+// StampMetaLogInflightForTest and DoneMetaLogInflightForTest let tests in
+// other packages exercise the claim caps. Not for production use.
+func (f *Filer) StampMetaLogInflightForTest() int64 {
+	return f.metaLogInflight.stamp()
+}
+
+func (f *Filer) DoneMetaLogInflightForTest(tsNs int64) {
+	f.metaLogInflight.done(tsNs)
+}
+
+func (f *Filer) newMetadataEvent(oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) *filer_pb.SubscribeMetadataResponse {
+	if oldEntry == nil && newEntry == nil {
+		return nil
+	}
+	var fullpath util.FullPath
+	if oldEntry != nil {
+		fullpath = oldEntry.FullPath
+	}
+	if fullpath == "" && newEntry != nil {
+		fullpath = newEntry.FullPath
+	}
+	dir, _ := fullpath.DirAndName()
+	newParentPath := ""
+	if newEntry != nil {
+		newParentPath, _ = newEntry.FullPath.DirAndName()
+	}
+	return &filer_pb.SubscribeMetadataResponse{
+		Directory: dir,
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:           oldEntry.ToProtoEntry(),
+			NewEntry:           newEntry.ToProtoEntry(),
+			DeleteChunks:       deleteChunks,
+			NewParentPath:      newParentPath,
+			IsFromOtherCluster: isFromOtherCluster,
+			Signatures:         signatures,
+		},
+		// In flight until appended to the local log buffer (see metaLogInflight).
+		TsNs: f.metaLogInflight.stamp(),
+	}
+}
+
+func (f *Filer) logMetaEvent(ctx context.Context, event *filer_pb.SubscribeMetadataResponse) {
+	data, err := proto.Marshal(event)
+	if err != nil {
+		glog.Errorf("failed to marshal filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
+		return
+	}
+
+	if err := f.LocalMetaLogBuffer.AddDataToBuffer([]byte(event.Directory), data, event.TsNs); err != nil {
+		glog.Errorf("failed to add data to log buffer for %s: %v", event.Directory, err)
+	}
+
+}
+
+// metadataLogUploadLimit is the piece size a metadata log flush starts with. A
+// volume server refuses anything over its -fileSizeLimitMB (256 MB by default),
+// and a single oversized event — a CreateEntry carrying a large inline Content,
+// say — grows the log buffer well past that, leaving a blob that can never be
+// written and blocks every later flush behind it. BufferSize is what an
+// ordinary flush already produces, so it is a size the volume server accepts
+// under any configuration that works at all; a cluster running below it says so
+// in the rejection and volumeFileSizeLimit picks the real limit up from there.
+const metadataLogUploadLimit = log_buffer.BufferSize
+
+var fileSizeLimitPattern = regexp.MustCompile(`file over the limited (\d+) bytes`)
+
+// volumeFileSizeLimit reads the byte limit back out of a volume server's size
+// rejection, and returns 0 for any other error.
+func volumeFileSizeLimit(err error) int {
+	match := fileSizeLimitPattern.FindStringSubmatch(err.Error())
+	if match == nil {
+		return 0
+	}
+	limit, convErr := strconv.Atoi(match[1])
+	if convErr != nil {
+		return 0
+	}
+	return limit
+}
+
+func (f *Filer) logFlushFunc(logBuffer *log_buffer.LogBuffer, startTime, stopTime time.Time, buf []byte, minOffset, maxOffset int64) {
+
+	if len(buf) == 0 {
+		return
+	}
+
+	startTime, stopTime = startTime.UTC(), stopTime.UTC()
+
+	targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.%08x", SystemLogDir,
+		startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(), f.UniqueFilerId,
+		// startTime.Second(), startTime.Nanosecond(),
+	)
+
+	// One piece at a time, each retried on its own so a partial success is not
+	// replayed, and the piece size follows the limit the volume servers report.
+	limit := metadataLogUploadLimit
+	for len(buf) > 0 {
+		piece := nextLogPiece(buf, limit)
+		if err := f.appendToFile(targetFile, piece); err != nil {
+			glog.V(0).Infof("metadata log write failed %s: %v", targetFile, err)
+			if reported := volumeFileSizeLimit(err); reported > 0 && reported < limit {
+				glog.V(0).Infof("metadata log upload limit lowered to %d bytes", reported)
+				limit = reported
+				continue
+			}
+			time.Sleep(737 * time.Millisecond)
+			continue
+		}
+		buf = buf[len(piece):]
+	}
+}
+
+// nextLogPiece returns the leading piece of a flushed log buffer, at most
+// maxSize bytes and ending on a record boundary where it can so the piece still
+// decodes on its own. A record longer than maxSize is cut by size instead; the
+// readers fall back to streaming the whole file when a chunk does not decode
+// standalone, so a record may cross a chunk boundary.
+func nextLogPiece(buf []byte, maxSize int) []byte {
+	if len(buf) <= maxSize {
+		return buf
+	}
+
+	pos := 0
+	for pos+4 <= len(buf) {
+		size := int(util.BytesToUint32(buf[pos : pos+4]))
+		end := pos + 4 + size
+		if size <= 0 || end > len(buf) || end > maxSize {
+			break
+		}
+		pos = end
+	}
+	if pos == 0 {
+		// Either the leading record alone is over the limit, or buf starts
+		// mid-record because the piece before it was cut by size.
+		return buf[:maxSize]
+	}
+	return buf[:pos]
+}
+
+var (
+	volumeNotFoundPattern = regexp.MustCompile(`volume \d+? not found`)
+	chunkNotFoundPattern  = regexp.MustCompile(`(urls not found|File Not Found)`)
+	httpNotFoundPattern   = regexp.MustCompile(`404 Not Found: not found`)
+)
+
+// isChunkNotFoundError checks if the error indicates that a volume or chunk
+// has been deleted and is no longer available. These errors can be skipped
+// when reading persisted log files since the data is unrecoverable.
+func isChunkNotFoundError(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, util_http.ErrNotFound) || errors.Is(err, nethttp.ErrMissingFile) {
+		return true
+	}
+	errMsg := err.Error()
+	return volumeNotFoundPattern.MatchString(errMsg) ||
+		chunkNotFoundPattern.MatchString(errMsg) ||
+		httpNotFoundPattern.MatchString(errMsg)
+}
+
+// persistedLogReplayLimit caps concurrent legacy replays; decodes are shared
+// through the persisted-log cache, so this only bounds the listing fan-out.
+const persistedLogReplayLimit = 64
+
+var persistedLogReplaySem = make(chan struct{}, persistedLogReplayLimit)
+
+func (f *Filer) ReadPersistedLogBuffer(ctx context.Context, startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (lastTsNs int64, isDone bool, err error) {
+
+	// Cap concurrent replays; bail if the stream is already gone so cancelled
+	// clients do not park on the semaphore.
+	if err := ctx.Err(); err != nil {
+		return 0, false, err
+	}
+	select {
+	case persistedLogReplaySem <- struct{}{}:
+		defer func() { <-persistedLogReplaySem }()
+	case <-ctx.Done():
+		return 0, false, ctx.Err()
+	}
+
+	visitor, visitErr := f.collectPersistedLogBuffer(startPosition, stopTsNs)
+	if visitErr != nil {
+		if visitErr == io.EOF {
+			return
+		}
+		err = fmt.Errorf("reading from persisted logs: %w", visitErr)
+		return
+	}
+
+	// Readahead: run the visitor in a background goroutine so volume server I/O
+	// for the next log file overlaps with event processing and gRPC delivery.
+	const readaheadSize = 8192
+	type entryOrErr struct {
+		entry *filer_pb.LogEntry
+		err   error
+	}
+	ch := make(chan entryOrErr, readaheadSize)
+	stopReadahead := make(chan struct{})
+	readaheadDone := make(chan struct{})
+	go func() {
+		defer close(ch)
+		defer close(readaheadDone)
+		for {
+			entry, readErr := visitor.GetNext()
+			if readErr != nil {
+				if readErr != io.EOF {
+					select {
+					case ch <- entryOrErr{err: fmt.Errorf("read next from persisted logs: %w", readErr)}:
+					case <-stopReadahead:
+					}
+				}
+				return
+			}
+			select {
+			case ch <- entryOrErr{entry: entry}:
+			case <-stopReadahead:
+				return
+			}
+		}
+	}()
+	// Stop the readahead goroutine, wait for it to exit, then release any log
+	// file readers it left open (e.g. on early return or cancellation).
+	defer func() {
+		close(stopReadahead)
+		<-readaheadDone
+		visitor.Close()
+	}()
+
+	for item := range ch {
+		if item.err != nil {
+			err = item.err
+			return
+		}
+		var processErr error
+		isDone, processErr = eachLogEntryFn(item.entry)
+		if processErr != nil {
+			err = fmt.Errorf("process persisted log entry: %w", processErr)
+			return
+		}
+		lastTsNs = item.entry.TsNs
+		if isDone {
+			return
+		}
+	}
+
+	return
+}

weed/filer/filer_notify_append.go

@@ -0,0 +1,105 @@
+package filer
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func (f *Filer) appendToFile(targetFile string, data []byte) error {
+
+	assignResult, uploadResult, err2 := f.assignAndUpload(targetFile, data)
+	if err2 != nil {
+		return err2
+	}
+
+	// find out existing entry
+	fullpath := util.FullPath(targetFile)
+	entry, err := f.FindEntry(context.Background(), fullpath)
+	var offset int64 = 0
+	if err == filer_pb.ErrNotFound {
+		entry = &Entry{
+			FullPath: fullpath,
+			Attr: Attr{
+				Crtime: time.Now(),
+				Mtime:  time.Now(),
+				Mode:   os.FileMode(0644),
+				Uid:    OS_UID,
+				Gid:    OS_GID,
+			},
+		}
+	} else if err != nil {
+		return fmt.Errorf("find %s: %v", fullpath, err)
+	} else {
+		offset = int64(TotalSize(entry.GetChunks()))
+	}
+
+	// append to existing chunks
+	entry.Chunks = append(entry.GetChunks(), uploadResult.ToPbFileChunk(assignResult.Fid, offset, time.Now().UnixNano()))
+
+	// update the entry
+	err = f.CreateEntry(context.Background(), entry, nil, false, false, nil, false, f.MaxFilenameLength)
+
+	return err
+}
+
+// resolveMetadataLogAssignDiskType returns the disk type and matched path rule for
+// metadata log volume assigns. Disk type uses the rule when set, otherwise
+// Filer.DefaultDiskType (from -filer.disk), mirroring resolveAssignStorageOption.
+func (f *Filer) resolveMetadataLogAssignDiskType(targetFile string) (string, *filer_pb.FilerConf_PathConf) {
+	if f.FilerConf == nil {
+		return f.DefaultDiskType, &filer_pb.FilerConf_PathConf{}
+	}
+	rule := f.FilerConf.MatchStorageRule(targetFile)
+	return util.Nvl(rule.DiskType, f.DefaultDiskType), rule
+}
+
+func (f *Filer) assignAndUpload(targetFile string, data []byte) (*operation.AssignResult, *operation.UploadResult, error) {
+	// assign a volume location
+	diskType, rule := f.resolveMetadataLogAssignDiskType(targetFile)
+	assignRequest := &operation.VolumeAssignRequest{
+		Count:               1,
+		Collection:          util.Nvl(f.metaLogCollection, rule.Collection),
+		Replication:         util.Nvl(f.metaLogReplication, rule.Replication),
+		DiskType:            diskType,
+		WritableVolumeCount: rule.VolumeGrowthCount,
+		ExpectedDataSize:    uint64(len(data)),
+	}
+
+	assignResult, err := operation.Assign(context.Background(), f.GetMaster, f.GrpcDialOption, assignRequest)
+	if err != nil {
+		return nil, nil, fmt.Errorf("AssignVolume: %w", err)
+	}
+	if assignResult.Error != "" {
+		return nil, nil, fmt.Errorf("AssignVolume error: %v", assignResult.Error)
+	}
+
+	// upload data
+	targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
+	uploadOption := &operation.UploadOption{
+		UploadUrl:         targetUrl,
+		Filename:          "",
+		Cipher:            f.Cipher,
+		IsInputCompressed: false,
+		MimeType:          "",
+		PairMap:           nil,
+		Jwt:               assignResult.Auth,
+	}
+
+	uploader, err := operation.NewUploader()
+	if err != nil {
+		return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
+	}
+
+	uploadResult, err := uploader.UploadData(context.Background(), data, uploadOption)
+	if err != nil {
+		return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
+	}
+	// println("uploaded to", targetUrl)
+	return assignResult, uploadResult, nil
+}

weed/filer/filer_notify_append_test.go

@@ -0,0 +1,56 @@
+package filer
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestResolveMetadataLogAssignDiskTypeUsesPathRule(t *testing.T) {
+	fc := NewFilerConf()
+	if err := fc.SetLocationConf(&filer_pb.FilerConf_PathConf{
+		LocationPrefix: "/topics/.system/log",
+		DiskType:       "hot",
+	}); err != nil {
+		t.Fatalf("set location conf: %v", err)
+	}
+
+	f := &Filer{
+		FilerConf:       fc,
+		DefaultDiskType: "hdd",
+	}
+
+	got, rule := f.resolveMetadataLogAssignDiskType("/topics/.system/log/2026-06-23/12-00.1")
+	if got != "hot" {
+		t.Fatalf("disk type = %q, want %q", got, "hot")
+	}
+	if rule.DiskType != "hot" {
+		t.Fatalf("rule disk type = %q, want %q", rule.DiskType, "hot")
+	}
+}
+
+func TestResolveMetadataLogAssignDiskTypeFallsBackToFilerDefault(t *testing.T) {
+	f := &Filer{
+		FilerConf:       NewFilerConf(),
+		DefaultDiskType: "hot",
+	}
+
+	got, _ := f.resolveMetadataLogAssignDiskType("/topics/.system/log/2026-06-23/12-00.1")
+	if got != "hot" {
+		t.Fatalf("disk type = %q, want %q", got, "hot")
+	}
+}
+
+func TestResolveMetadataLogAssignDiskTypeNilFilerConf(t *testing.T) {
+	f := &Filer{
+		DefaultDiskType: "hot",
+	}
+
+	got, rule := f.resolveMetadataLogAssignDiskType("/topics/.system/log/2026-06-23/12-00.1")
+	if got != "hot" {
+		t.Fatalf("disk type = %q, want %q", got, "hot")
+	}
+	if rule == nil {
+		t.Fatal("expected non-nil empty rule")
+	}
+}

weed/filer/filer_notify_inflight_test.go

@@ -0,0 +1,134 @@
+package filer
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+)
+
+// TestMetaLogInflightFloor pins the in-flight stamp bookkeeping backing
+// Filer.LocalFlushedThroughTsNs: the floor is the oldest outstanding stamp,
+// duplicate stamps are reference-counted, and a cleared registry reports zero.
+func TestMetaLogInflightFloor(t *testing.T) {
+	var inflight metaLogInflight
+	if got := inflight.minTsNs(); got != 0 {
+		t.Fatalf("empty registry: floor=%d want 0", got)
+	}
+	ts1 := inflight.stamp()
+	ts2 := inflight.stamp()
+	if ts2 < ts1 {
+		t.Fatalf("stamps not monotonic: %d then %d", ts1, ts2)
+	}
+	if got := inflight.minTsNs(); got != ts1 {
+		t.Fatalf("floor=%d want oldest stamp %d", got, ts1)
+	}
+	inflight.done(ts1)
+	if got := inflight.minTsNs(); got != ts2 {
+		t.Fatalf("after done(ts1): floor=%d want %d", got, ts2)
+	}
+	inflight.done(ts2)
+	if got := inflight.minTsNs(); got != 0 {
+		t.Fatalf("cleared registry: floor=%d want 0", got)
+	}
+}
+
+// TestLocalFlushedThroughTsNsBoundsInflight pins the flush-watermark claim: a
+// drained buffer claims "now", but an event stamped and not yet appended caps
+// the claim just below its timestamp - a peer bounding disk reads by the
+// claim must not advance past an event still on its way into the buffer.
+func TestLocalFlushedThroughTsNsBoundsInflight(t *testing.T) {
+	f := &Filer{
+		LocalMetaLogBuffer: log_buffer.NewLogBuffer("inflight-test", time.Minute, nil, nil, nil),
+	}
+	defer f.LocalMetaLogBuffer.ShutdownLogBuffer()
+
+	now := time.Now().UnixNano()
+	if got := f.LocalFlushedThroughTsNs(now); got != now {
+		t.Fatalf("drained, nothing in flight: claim=%d want now=%d", got, now)
+	}
+
+	ts := f.metaLogInflight.stamp()
+	if got := f.LocalFlushedThroughTsNs(time.Now().UnixNano()); got != ts-1 {
+		t.Fatalf("with in-flight stamp %d: claim=%d want %d", ts, got, ts-1)
+	}
+
+	f.metaLogInflight.done(ts)
+	now = time.Now().UnixNano()
+	if got := f.LocalFlushedThroughTsNs(now); got != now {
+		t.Fatalf("stamp cleared: claim=%d want now=%d", got, now)
+	}
+}
+
+// TestLocalDeliveredThroughTsNsBoundsInflight pins the delivery-freshness
+// cap: an idle heartbeat must not claim delivery-completeness past an event
+// that is stamped but not yet appended - the peer aggregator turns that claim
+// into its delivery low-watermark.
+func TestLocalDeliveredThroughTsNsBoundsInflight(t *testing.T) {
+	f := &Filer{
+		LocalMetaLogBuffer: log_buffer.NewLogBuffer("delivered-test", time.Minute, nil, nil, nil),
+	}
+	defer f.LocalMetaLogBuffer.ShutdownLogBuffer()
+
+	now := time.Now().UnixNano()
+	if got := f.LocalDeliveredThroughTsNs(now); got != now {
+		t.Fatalf("nothing in flight: claim=%d want now=%d", got, now)
+	}
+	ts := f.metaLogInflight.stamp()
+	if got := f.LocalDeliveredThroughTsNs(time.Now().UnixNano()); got != ts-1 {
+		t.Fatalf("with in-flight stamp %d: claim=%d want %d", ts, got, ts-1)
+	}
+	f.metaLogInflight.done(ts)
+}
+
+// TestClaimFencesFutureStamps pins the claim fence: once a claim is issued, a
+// wall-clock step backwards must not let a later stamp land at or below it -
+// the peer has already advanced its watermark to the claim.
+func TestClaimFencesFutureStamps(t *testing.T) {
+	f := &Filer{
+		LocalMetaLogBuffer: log_buffer.NewLogBuffer("claim-fence-test", time.Minute, nil, nil, nil),
+	}
+	defer f.LocalMetaLogBuffer.ShutdownLogBuffer()
+
+	// Claim with a sampled clock one hour ahead: to a stamp taken at the real
+	// wall clock this is exactly a backward step after the claim was issued.
+	aheadNs := time.Now().Add(time.Hour).UnixNano()
+	if got := f.LocalDeliveredThroughTsNs(aheadNs); got != aheadNs {
+		t.Fatalf("nothing in flight: claim=%d want %d", got, aheadNs)
+	}
+	ts := f.metaLogInflight.stamp()
+	if ts <= aheadNs {
+		t.Fatalf("stamp %d not fenced above issued delivery claim %d", ts, aheadNs)
+	}
+	f.metaLogInflight.done(ts)
+
+	// The flush claim fences the same way.
+	aheadNs += int64(time.Hour)
+	if got := f.LocalFlushedThroughTsNs(aheadNs); got != aheadNs {
+		t.Fatalf("drained buffer: claim=%d want %d", got, aheadNs)
+	}
+	ts = f.metaLogInflight.stamp()
+	if ts <= aheadNs {
+		t.Fatalf("stamp %d not fenced above issued flush claim %d", ts, aheadNs)
+	}
+	f.metaLogInflight.done(ts)
+}
+
+// TestMetaLogInflightStampMonotonic pins the registry-local monotonicity: a
+// wall clock stepping backwards must not let a new stamp slip under an
+// already-sampled floor.
+func TestMetaLogInflightStampMonotonic(t *testing.T) {
+	var inflight metaLogInflight
+	ts1 := inflight.stamp()
+	// Simulate a wall-clock step backwards: force the registry's history
+	// ahead of the clock; the next stamp must still move forward.
+	inflight.Lock()
+	inflight.lastStampNs = ts1 + int64(time.Hour)
+	inflight.Unlock()
+	ts2 := inflight.stamp()
+	if ts2 <= ts1+int64(time.Hour) {
+		t.Fatalf("stamp regressed: %d after forcing history to %d", ts2, ts1+int64(time.Hour))
+	}
+	inflight.done(ts1)
+	inflight.done(ts2)
+}

weed/filer/filer_notify_read.go

@@ -0,0 +1,711 @@
+package filer
+
+import (
+	"container/heap"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+)
+
+type LogFileEntry struct {
+	TsNs      int64
+	FileEntry *Entry
+}
+
+// logFileMayContainAfter reports whether the log file named for fileTsNs can
+// hold any entry past startTsNs. The following file's name is not the bound: a
+// file is named for the start of the window it holds, minute-truncated, and the
+// window runs up to a flush interval longer, so "12-30" can hold 12:31:20 while
+// "12-31" exists alongside it. Comparing against the next name dropped exactly
+// the spanning file a mid-window cursor needs.
+func logFileMayContainAfter(fileTsNs, startTsNs int64) bool {
+	return fileTsNs+int64(time.Minute)+int64(LogFlushInterval) > startTsNs
+}
+
+// persistedLogScanStart backs a read position off by one flush interval before
+// choosing which log files to open. A log file is named for the start of the
+// window it holds and a window spans up to flushInterval, so a file whose name
+// sorts before the cursor's own minute can still hold entries after it -- a
+// window sealed at 12:30:59 and ending 12:31:58 lives in "12-30". Entries are
+// filtered against the exact cursor afterwards, so widening the file scan only
+// costs a little extra reading and never re-delivers.
+func persistedLogScanStart(t time.Time) time.Time {
+	return t.Add(-LogFlushInterval)
+}
+
+// LastShippedLogEntryTsNsForFiler mirrors the client's reader across one
+// filer's shipped files, in ship order: a file that fails mid-read still
+// delivers its readable prefix and later files still deliver after it, so the
+// newest file with readable content answers. answeredFileTsNs names that file
+// so the caller can roll back the sent state of everything newer - refs the
+// cursor did not reach must re-ship, or a transient probe failure leaves the
+// cursor behind them for the life of the connection.
+// complete reports whether the answering file was read through to its end: a
+// prefix-limited answer means the ref's unread suffix must re-ship too, or a
+// transient mid-file failure abandons it for the life of the connection.
+func (f *Filer) LastShippedLogEntryTsNsForFiler(refs []*filer_pb.LogFileChunkRef) (tsNs int64, answeredFileTsNs int64, ok bool, complete bool) {
+	for i := len(refs) - 1; i >= 0; i-- {
+		if tsNs, ok, complete = f.lastShippedLogEntryTsNs(refs[i].Chunks); ok {
+			return tsNs, refs[i].FileTsNs, ok, complete
+		}
+	}
+	return 0, 0, false, false
+}
+
+// lookupLogChunkFn reports whether a chunk still resolves to a live volume.
+// Swapped in tests.
+var lookupLogChunkFn = func(f *Filer, fileId string) error {
+	_, err := f.MasterClient.GetLookupFileIdFunction()(context.Background(), fileId)
+	return err
+}
+
+// lastShippedLogEntryTsNs mirrors the client's read of one shipped file: a
+// sequential scan of exactly these chunks that ends at the first unreadable
+// one, so a marker built from it never claims content the client will not
+// apply. Unreadable includes transient failures - understating the marker
+// only re-ships, overstating loses events - and no error escapes: a probe
+// failure must never block the transition the client is waiting on.
+//
+// Readability is judged where the client reads - the volumes - not by the
+// decoded-chunk cache: a chunk this server decoded an hour ago may sit on a
+// volume that has since died, and the direct-reading client stops there no
+// matter how well the cache still remembers the bytes.
+func (f *Filer) lastShippedLogEntryTsNs(chunks []*filer_pb.FileChunk) (tsNs int64, ok bool, complete bool) {
+	for _, chunk := range chunks {
+		if lookupErr := lookupLogChunkFn(f, chunk.GetFileIdString()); lookupErr != nil {
+			return tsNs, ok, false
+		}
+		entries, loadErr := f.persistedLogCache.getOrLoad(chunk.GetFileIdString(), int64(chunk.Size), func() ([]*filer_pb.LogEntry, bool, error) {
+			return loadLogFileEntriesFn(f.MasterClient, chunk)
+		})
+		if errors.Is(loadErr, errLogChunkIncomplete) {
+			// Records span chunks; the client streams such a file whole.
+			if streamTsNs, streamOk, streamComplete := f.lastStreamedLogEntryTsNs(chunks); streamOk {
+				return streamTsNs, true, streamComplete
+			}
+			return tsNs, ok, false
+		}
+		if loadErr != nil {
+			return tsNs, ok, false // prefix ends here, like the client's reader
+		}
+		if len(entries) > 0 {
+			tsNs, ok = entries[len(entries)-1].TsNs, true
+		}
+	}
+	return tsNs, ok, true
+}
+
+// lastStreamedLogEntryTsNs scans the chunk list as one byte stream, ending
+// where the client's reader ends: a clean or torn tail, a missing chunk, or
+// undecodable bytes all terminate the scan with the progress made.
+func (f *Filer) lastStreamedLogEntryTsNs(chunks []*filer_pb.FileChunk) (tsNs int64, ok bool, complete bool) {
+	r := newLogFileStreamReader(f.MasterClient, chunks)
+	if closer, isCloser := r.(io.Closer); isCloser {
+		defer closer.Close()
+	}
+	sizeBuf := make([]byte, 4)
+	for {
+		if _, readErr := io.ReadFull(r, sizeBuf); readErr != nil {
+			// A clean or torn tail is where the client's read ends too; only a
+			// mid-stream failure leaves an unread remainder worth re-shipping.
+			ended := readErr == io.EOF || readErr == io.ErrUnexpectedEOF
+			return tsNs, ok, ended
+		}
+		size := util.BytesToUint32(sizeBuf)
+		if size > maxLogEntrySize {
+			return tsNs, ok, false
+		}
+		data := make([]byte, size)
+		if _, readErr := io.ReadFull(r, data); readErr != nil {
+			return tsNs, ok, false
+		}
+		logEntry := &filer_pb.LogEntry{}
+		if unmarshalErr := logEntry.UnmarshalVT(data); unmarshalErr != nil {
+			return tsNs, ok, false
+		}
+		tsNs, ok = logEntry.TsNs, true
+	}
+}
+
+// PersistedLogScanStartTsNs is the oldest file-name timestamp a scan from t
+// can still list. File names are minute-truncated, and the collector compares
+// names at minute granularity, so the bound must truncate too: an exact-ns
+// bound sits inside the boundary file's minute and disowns a file the next
+// collection will still return.
+func PersistedLogScanStartTsNs(t time.Time) int64 {
+	return persistedLogScanStart(t).Truncate(time.Minute).UnixNano()
+}
+
+func (f *Filer) collectPersistedLogBuffer(startPosition log_buffer.MessagePosition, stopTsNs int64) (v *OrderedLogVisitor, err error) {
+
+	if stopTsNs != 0 && startPosition.Time.UnixNano() > stopTsNs {
+		return nil, io.EOF
+	}
+
+	scanFrom := persistedLogScanStart(startPosition.Time)
+	startDate := fmt.Sprintf("%04d-%02d-%02d", scanFrom.Year(), scanFrom.Month(), scanFrom.Day())
+
+	dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, math.MaxInt32, "", "", "")
+	if listDayErr != nil {
+		return nil, fmt.Errorf("fail to list log by day: %w", listDayErr)
+	}
+
+	return NewOrderedLogVisitor(f, startPosition, stopTsNs, dayEntries)
+
+}
+
+// CollectLogFileRefs lists persisted log files and returns their chunk references
+// without reading any data from volume servers. The client can use the returned
+// fids to read log file data directly from volume servers in parallel.
+func (f *Filer) CollectLogFileRefs(ctx context.Context, startPosition log_buffer.MessagePosition, stopTsNs int64) (refs []*filer_pb.LogFileChunkRef, lastTsNs int64, err error) {
+	if stopTsNs != 0 && startPosition.Time.UnixNano() > stopTsNs {
+		return nil, 0, nil
+	}
+
+	scanFrom := persistedLogScanStart(startPosition.Time)
+	startDate := fmt.Sprintf("%04d-%02d-%02d", scanFrom.Year(), scanFrom.Month(), scanFrom.Day())
+	startHourMinute := fmt.Sprintf("%02d-%02d", scanFrom.Hour(), scanFrom.Minute())
+	var stopDate, stopHourMinute string
+	if stopTsNs != 0 {
+		stopTime := time.Unix(0, stopTsNs).UTC()
+		stopDate = fmt.Sprintf("%04d-%02d-%02d", stopTime.Year(), stopTime.Month(), stopTime.Day())
+		stopHourMinute = fmt.Sprintf("%02d-%02d", stopTime.Hour(), stopTime.Minute())
+	}
+
+	dayEntries, _, listDayErr := f.ListDirectoryEntries(ctx, SystemLogDir, startDate, true, math.MaxInt32, "", "", "")
+	if listDayErr != nil {
+		return nil, 0, fmt.Errorf("fail to list log by day: %w", listDayErr)
+	}
+
+	for _, dayEntry := range dayEntries {
+		if stopDate != "" && strings.Compare(dayEntry.Name(), stopDate) > 0 {
+			break
+		}
+
+		hourMinuteEntries, _, listErr := f.ListDirectoryEntries(ctx, util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, math.MaxInt32, "", "", "")
+		if listErr != nil {
+			return nil, 0, fmt.Errorf("fail to list log %s: %w", dayEntry.Name(), listErr)
+		}
+
+		for _, hmEntry := range hourMinuteEntries {
+			hourMinute := util.FileNameBase(hmEntry.Name())
+			if dayEntry.Name() == startDate && strings.Compare(hourMinute, startHourMinute) < 0 {
+				continue
+			}
+			if dayEntry.Name() == stopDate && stopHourMinute != "" && strings.Compare(hourMinute, stopHourMinute) > 0 {
+				break
+			}
+
+			tsMinute := fmt.Sprintf("%s-%s", dayEntry.Name(), hourMinute)
+			t, parseErr := time.Parse("2006-01-02-15-04", tsMinute)
+			if parseErr != nil {
+				glog.Errorf("failed to parse %s: %v", tsMinute, parseErr)
+				continue
+			}
+			filerId := getFilerId(hmEntry.Name())
+			if filerId == "" {
+				continue
+			}
+
+			chunks := hmEntry.GetChunks()
+			if len(chunks) == 0 {
+				continue
+			}
+
+			refs = append(refs, &filer_pb.LogFileChunkRef{
+				Chunks:   chunks,
+				FileTsNs: t.UnixNano(),
+				FilerId:  filerId,
+			})
+			lastTsNs = t.UnixNano()
+		}
+	}
+	lastTsNs = clampLogRefsCursor(lastTsNs, startPosition.Time.UnixNano())
+	return
+}
+
+// clampLogRefsCursor keeps a chunk-ref read from moving the subscriber's cursor
+// backwards. Refs are named for the minute their window starts in, so the last
+// one routinely sorts before the position that was asked for -- always, now
+// that the scan reaches back a flush interval to pick up a spanning file. The
+// caller makes this value the new read position, and rewinding it would replay
+// memory from before the client's own SinceNs and re-send what the chunk reader
+// has already been handed.
+func clampLogRefsCursor(lastTsNs, startTsNs int64) int64 {
+	if lastTsNs < startTsNs {
+		return startTsNs
+	}
+	return lastTsNs
+}
+
+func (f *Filer) HasPersistedLogFiles(startPosition log_buffer.MessagePosition) (bool, error) {
+	startDate := fmt.Sprintf("%04d-%02d-%02d", startPosition.Time.Year(), startPosition.Time.Month(), startPosition.Time.Day())
+	dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 1, "", "", "")
+
+	if listDayErr != nil {
+		return false, fmt.Errorf("fail to list log by day: %w", listDayErr)
+	}
+	if len(dayEntries) == 0 {
+		return false, nil
+	}
+	return true, nil
+}
+
+// ----------
+type LogEntryItem struct {
+	Entry *filer_pb.LogEntry
+	filer string
+}
+
+// LogEntryItemPriorityQueue a priority queue for LogEntry
+type LogEntryItemPriorityQueue []*LogEntryItem
+
+func (pq LogEntryItemPriorityQueue) Len() int { return len(pq) }
+func (pq LogEntryItemPriorityQueue) Less(i, j int) bool {
+	return pq[i].Entry.TsNs < pq[j].Entry.TsNs
+}
+func (pq LogEntryItemPriorityQueue) Swap(i, j int) { pq[i], pq[j] = pq[j], pq[i] }
+func (pq *LogEntryItemPriorityQueue) Push(x any) {
+	item := x.(*LogEntryItem)
+	*pq = append(*pq, item)
+}
+func (pq *LogEntryItemPriorityQueue) Pop() any {
+	n := len(*pq)
+	item := (*pq)[n-1]
+	*pq = (*pq)[:n-1]
+	return item
+}
+
+// ----------
+
+type OrderedLogVisitor struct {
+	perFilerIteratorMap   map[string]*LogFileQueueIterator
+	pq                    *LogEntryItemPriorityQueue
+	logFileEntryCollector *LogFileEntryCollector
+}
+
+func NewOrderedLogVisitor(f *Filer, startPosition log_buffer.MessagePosition, stopTsNs int64, dayEntries []*Entry) (*OrderedLogVisitor, error) {
+
+	perFilerQueueMap := make(map[string]*LogFileQueueIterator)
+	// initialize the priority queue
+	pq := &LogEntryItemPriorityQueue{}
+	heap.Init(pq)
+
+	t := &OrderedLogVisitor{
+		perFilerIteratorMap:   perFilerQueueMap,
+		pq:                    pq,
+		logFileEntryCollector: NewLogFileEntryCollector(f, startPosition, stopTsNs, dayEntries),
+	}
+	if err := t.logFileEntryCollector.collectMore(t); err != nil && err != io.EOF {
+		return nil, err
+	}
+	return t, nil
+}
+
+func (o *OrderedLogVisitor) GetNext() (logEntry *filer_pb.LogEntry, err error) {
+	if o.pq.Len() == 0 {
+		return nil, io.EOF
+	}
+	item := heap.Pop(o.pq).(*LogEntryItem)
+	filerId := item.filer
+
+	// fill the pq with the next log entry from the same filer
+	it := o.perFilerIteratorMap[filerId]
+	next, nextErr := it.getNext(o)
+	if nextErr != nil {
+		if nextErr == io.EOF {
+			// do nothing since the filer has no more log entries
+		} else {
+			return nil, fmt.Errorf("failed to get next log entry: %w", nextErr)
+		}
+	} else {
+		heap.Push(o.pq, &LogEntryItem{
+			Entry: next,
+			filer: filerId,
+		})
+	}
+	return item.Entry, nil
+}
+
+// Close releases any log file readers still open across the per-filer
+// iterators, e.g. when a subscription stops before reaching the end. Safe to
+// call more than once.
+func (o *OrderedLogVisitor) Close() {
+	for _, it := range o.perFilerIteratorMap {
+		it.Close()
+	}
+}
+
+func getFilerId(name string) string {
+	idx := strings.LastIndex(name, ".")
+	if idx < 0 {
+		return ""
+	}
+	return name[idx+1:]
+}
+
+// ----------
+
+type LogFileEntryCollector struct {
+	f               *Filer
+	startTsNs       int64
+	stopTsNs        int64
+	dayEntryQueue   *util.Queue[*Entry]
+	startDate       string
+	startHourMinute string
+	stopDate        string
+	stopHourMinute  string
+}
+
+func NewLogFileEntryCollector(f *Filer, startPosition log_buffer.MessagePosition, stopTsNs int64, dayEntries []*Entry) *LogFileEntryCollector {
+	dayEntryQueue := util.NewQueue[*Entry]()
+	for _, dayEntry := range dayEntries {
+		dayEntryQueue.Enqueue(dayEntry)
+		// println("enqueue day entry", dayEntry.Name())
+	}
+
+	scanFrom := persistedLogScanStart(startPosition.Time)
+	startDate := fmt.Sprintf("%04d-%02d-%02d", scanFrom.Year(), scanFrom.Month(), scanFrom.Day())
+	startHourMinute := fmt.Sprintf("%02d-%02d", scanFrom.Hour(), scanFrom.Minute())
+	var stopDate, stopHourMinute string
+	if stopTsNs != 0 {
+		stopTime := time.Unix(0, stopTsNs+24*60*60*int64(time.Second)).UTC()
+		stopDate = fmt.Sprintf("%04d-%02d-%02d", stopTime.Year(), stopTime.Month(), stopTime.Day())
+		stopHourMinute = fmt.Sprintf("%02d-%02d", stopTime.Hour(), stopTime.Minute())
+	}
+
+	return &LogFileEntryCollector{
+		f:               f,
+		startTsNs:       startPosition.Time.UnixNano(),
+		stopTsNs:        stopTsNs,
+		dayEntryQueue:   dayEntryQueue,
+		startDate:       startDate,
+		startHourMinute: startHourMinute,
+		stopDate:        stopDate,
+		stopHourMinute:  stopHourMinute,
+	}
+}
+
+func (c *LogFileEntryCollector) collectMore(v *OrderedLogVisitor) (err error) {
+	dayEntry := c.dayEntryQueue.Dequeue()
+	if dayEntry == nil {
+		return io.EOF
+	}
+	// println("dequeue day entry", dayEntry.Name())
+	if c.stopDate != "" {
+		if strings.Compare(dayEntry.Name(), c.stopDate) > 0 {
+			return io.EOF
+		}
+	}
+
+	hourMinuteEntries, _, listHourMinuteErr := c.f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, math.MaxInt32, "", "", "")
+	if listHourMinuteErr != nil {
+		return fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr)
+	}
+	freshFilerIds := make(map[string]string)
+	for _, hourMinuteEntry := range hourMinuteEntries {
+		// println("checking hh-mm", hourMinuteEntry.FullPath)
+		hourMinute := util.FileNameBase(hourMinuteEntry.Name())
+		if dayEntry.Name() == c.startDate {
+			if strings.Compare(hourMinute, c.startHourMinute) < 0 {
+				continue
+			}
+		}
+		if dayEntry.Name() == c.stopDate {
+			if strings.Compare(hourMinute, c.stopHourMinute) > 0 {
+				break
+			}
+		}
+
+		tsMinute := fmt.Sprintf("%s-%s", dayEntry.Name(), hourMinute)
+		// println("  enqueue", tsMinute)
+		t, parseErr := time.Parse("2006-01-02-15-04", tsMinute)
+		if parseErr != nil {
+			glog.Errorf("failed to parse %s: %v", tsMinute, parseErr)
+			continue
+		}
+		filerId := getFilerId(hourMinuteEntry.Name())
+		if filerId == "" {
+			glog.Warningf("Invalid log file name format: %s", hourMinuteEntry.Name())
+			continue // Skip files with invalid format
+		}
+		iter, found := v.perFilerIteratorMap[filerId]
+		if !found {
+			iter = newLogFileQueueIterator(c.f.MasterClient, c.f.persistedLogCache, util.NewQueue[*LogFileEntry](), c.startTsNs, c.stopTsNs)
+			v.perFilerIteratorMap[filerId] = iter
+			freshFilerIds[filerId] = hourMinuteEntry.Name()
+		}
+		iter.q.Enqueue(&LogFileEntry{
+			TsNs:      t.UnixNano(),
+			FileEntry: hourMinuteEntry,
+		})
+	}
+
+	// fill the pq with the next log entry if it is a new filer
+	for filerId, entryName := range freshFilerIds {
+		iter, found := v.perFilerIteratorMap[filerId]
+		if !found {
+			glog.Errorf("Unexpected! failed to find iterator for filer %s", filerId)
+			continue
+		}
+		next, nextErr := iter.getNext(v)
+		if nextErr != nil {
+			if nextErr == io.EOF {
+				// do nothing since the filer has no more log entries
+			} else {
+				return fmt.Errorf("failed to get next log entry for %v: %w", entryName, nextErr)
+			}
+		} else {
+			heap.Push(v.pq, &LogEntryItem{
+				Entry: next,
+				filer: filerId,
+			})
+		}
+	}
+
+	return nil
+}
+
+// ----------
+
+type LogFileQueueIterator struct {
+	q                   *util.Queue[*LogFileEntry]
+	masterClient        *wdclient.MasterClient
+	cache               *persistedLogCache
+	startTsNs           int64
+	stopTsNs            int64
+	currentFileIterator *LogFileIterator
+}
+
+func newLogFileQueueIterator(masterClient *wdclient.MasterClient, cache *persistedLogCache, q *util.Queue[*LogFileEntry], startTsNs, stopTsNs int64) *LogFileQueueIterator {
+	return &LogFileQueueIterator{
+		q:            q,
+		masterClient: masterClient,
+		cache:        cache,
+		startTsNs:    startTsNs,
+		stopTsNs:     stopTsNs,
+	}
+}
+
+// Close releases the current log file reader, if any. Safe to call more than once.
+func (iter *LogFileQueueIterator) Close() {
+	if iter.currentFileIterator != nil {
+		if err := iter.currentFileIterator.Close(); err != nil {
+			glog.Warningf("close log file %s: %v", iter.currentFileIterator.filePath, err)
+		}
+		iter.currentFileIterator = nil
+	}
+}
+
+// getNext yields one log entry at a time from the current file, advancing to
+// the next file as each is exhausted. It returns io.EOF when done. Entries are
+// decoded one chunk at a time and shared across subscribers, so per-subscriber
+// memory stays bounded regardless of log file size.
+func (iter *LogFileQueueIterator) getNext(v *OrderedLogVisitor) (logEntry *filer_pb.LogEntry, err error) {
+	for {
+		if iter.currentFileIterator != nil {
+			logEntry, err = iter.currentFileIterator.getNext()
+			if err == nil {
+				return logEntry, nil
+			}
+			// The current file is done (io.EOF), its volume was deleted, or it is
+			// unreadable. Close it on every path so the reader is not left alive
+			// until GC; only a genuine read error is propagated.
+			readErr := err
+			switch {
+			case readErr == io.EOF:
+				readErr = nil
+			case isChunkNotFoundError(readErr):
+				// Volume or chunk was deleted, skip the rest of this log file
+				glog.Warningf("skipping rest of %s: %v", iter.currentFileIterator.filePath, readErr)
+				readErr = nil
+			}
+			if closeErr := iter.currentFileIterator.Close(); closeErr != nil {
+				glog.Warningf("close log file %s: %v", iter.currentFileIterator.filePath, closeErr)
+			}
+			iter.currentFileIterator = nil
+			if readErr != nil {
+				return nil, readErr
+			}
+		}
+
+		// advance to the next file
+		if iter.q.Len() == 0 {
+			return nil, io.EOF
+		}
+		t := iter.q.Dequeue()
+		if t == nil {
+			continue
+		}
+		// skip the file if it is after the stopTsNs
+		if iter.stopTsNs != 0 && t.TsNs > iter.stopTsNs {
+			return nil, io.EOF
+		}
+		if iter.q.Peek() == nil {
+			if collectErr := v.logFileEntryCollector.collectMore(v); collectErr != nil && collectErr != io.EOF {
+				return nil, collectErr
+			}
+		}
+		if !logFileMayContainAfter(t.TsNs, iter.startTsNs) {
+			continue
+		}
+		iter.currentFileIterator = newLogFileIterator(iter.masterClient, iter.cache, t.FileEntry, iter.startTsNs, iter.stopTsNs)
+	}
+}
+
+// ----------
+
+type LogFileIterator struct {
+	// cached mode: each immutable chunk is decoded once and shared read-only
+	// across subscribers via the persisted-log cache.
+	masterClient *wdclient.MasterClient
+	cache        *persistedLogCache
+	chunks       []*filer_pb.FileChunk
+	chunkIdx     int
+	cur          []*filer_pb.LogEntry
+	curPos       int
+	lastTsNs     int64
+	// streaming mode: the whole file as one byte stream, the fallback when a
+	// chunk does not decode standalone (records spanning chunk boundaries).
+	r         io.Reader
+	sizeBuf   []byte
+	startTsNs int64
+	stopTsNs  int64
+	filePath  string
+}
+
+// swapped in tests
+var loadLogFileEntriesFn = loadLogFileEntries
+var newLogFileStreamReader = func(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) io.Reader {
+	return NewChunkStreamReaderFromFiler(context.Background(), masterClient, chunks)
+}
+
+func newLogFileIterator(masterClient *wdclient.MasterClient, cache *persistedLogCache, fileEntry *Entry, startTsNs, stopTsNs int64) *LogFileIterator {
+	iter := &LogFileIterator{
+		masterClient: masterClient,
+		cache:        cache,
+		// sort a copy, leaving the listed entry's chunk order alone
+		chunks:    append([]*filer_pb.FileChunk(nil), fileEntry.GetChunks()...),
+		startTsNs: startTsNs,
+		stopTsNs:  stopTsNs,
+		filePath:  string(fileEntry.FullPath),
+	}
+	sort.SliceStable(iter.chunks, func(i, j int) bool {
+		return iter.chunks[i].Offset < iter.chunks[j].Offset
+	})
+	if cache == nil {
+		iter.startStreaming()
+	}
+	return iter
+}
+
+// startStreaming switches to the byte-stream fallback, resuming after the last
+// yielded entry.
+func (iter *LogFileIterator) startStreaming() {
+	if iter.lastTsNs > iter.startTsNs {
+		iter.startTsNs = iter.lastTsNs
+	}
+	iter.r = newLogFileStreamReader(iter.masterClient, iter.chunks)
+	iter.sizeBuf = make([]byte, 4)
+	iter.cur, iter.curPos = nil, 0
+}
+
+func (iter *LogFileIterator) Close() error {
+	if r, ok := iter.r.(io.Closer); ok {
+		return r.Close()
+	}
+	return nil
+}
+
+// getNext will return io.EOF when done
+func (iter *LogFileIterator) getNext() (logEntry *filer_pb.LogEntry, err error) {
+	if iter.r == nil {
+		return iter.getNextCached()
+	}
+	var n int
+	for {
+		n, err = iter.r.Read(iter.sizeBuf)
+		if err != nil {
+			return
+		}
+		if n != 4 {
+			return nil, fmt.Errorf("size %d bytes, expected 4 bytes", n)
+		}
+		size := util.BytesToUint32(iter.sizeBuf)
+		if size > maxLogEntrySize {
+			return nil, fmt.Errorf("%s entry size %d exceeds %d", iter.filePath, size, maxLogEntrySize)
+		}
+		entryData := make([]byte, size)
+		n, err = iter.r.Read(entryData)
+		if err != nil {
+			return
+		}
+		if n != int(size) {
+			return nil, fmt.Errorf("entry data %d bytes, expected %d bytes", n, size)
+		}
+		logEntry = &filer_pb.LogEntry{}
+		if err = logEntry.UnmarshalVT(entryData); err != nil {
+			return
+		}
+		if logEntry.TsNs <= iter.startTsNs {
+			continue
+		}
+		if iter.stopTsNs != 0 && logEntry.TsNs > iter.stopTsNs {
+			return nil, io.EOF
+		}
+		return
+	}
+}
+
+// getNextCached yields from shared per-chunk decoded slices, loading chunks
+// lazily and in order.
+func (iter *LogFileIterator) getNextCached() (logEntry *filer_pb.LogEntry, err error) {
+	for {
+		for iter.curPos < len(iter.cur) {
+			logEntry = iter.cur[iter.curPos]
+			iter.curPos++
+			if logEntry.TsNs <= iter.startTsNs {
+				continue
+			}
+			if iter.stopTsNs != 0 && logEntry.TsNs > iter.stopTsNs {
+				return nil, io.EOF
+			}
+			iter.lastTsNs = logEntry.TsNs
+			return logEntry, nil
+		}
+		if iter.chunkIdx >= len(iter.chunks) {
+			return nil, io.EOF
+		}
+		chunk := iter.chunks[iter.chunkIdx]
+		iter.chunkIdx++
+		// the flush upload time upper-bounds every record ts in the chunk; the
+		// margin tolerates a wall-clock retreat between stamping and upload
+		if chunk.ModifiedTsNs > 0 && chunk.ModifiedTsNs+int64(LogFlushInterval) <= iter.startTsNs {
+			continue
+		}
+		entries, loadErr := iter.cache.getOrLoad(chunk.GetFileIdString(), int64(chunk.Size), func() ([]*filer_pb.LogEntry, bool, error) {
+			return loadLogFileEntriesFn(iter.masterClient, chunk)
+		})
+		if loadErr != nil {
+			if errors.Is(loadErr, errLogChunkIncomplete) {
+				glog.V(1).Infof("log file %s chunk %s does not decode standalone, streaming the file", iter.filePath, chunk.GetFileIdString())
+				iter.startStreaming()
+				return iter.getNext()
+			}
+			return nil, loadErr
+		}
+		iter.cur = entries
+		iter.curPos = sort.Search(len(entries), func(i int) bool { return entries[i].TsNs > iter.startTsNs })
+	}
+}

weed/filer/filer_notify_read_minute_test.go

@@ -0,0 +1,270 @@
+package filer
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+)
+
+// TestPersistedLogScanStartCoversSpanningWindow pins the file-selection window.
+// A log file is named for the start of the window it holds, and a window spans
+// up to LogFlushInterval, so a cursor inside the window's later minutes must
+// still open the file named for its earlier one -- otherwise the read reports
+// "nothing on disk" for entries that are sitting in it, and the gap resolvers
+// take that miss as proof the range is empty.
+func TestPersistedLogScanStartCoversSpanningWindow(t *testing.T) {
+	// A window sealed at 12:30:59 ending 12:31:58 is written to "12-30".
+	sealed := time.Date(2026, 6, 29, 12, 30, 59, 0, time.UTC)
+	fileMinute := sealed.Format("15-04")
+
+	// A subscriber resuming mid-window must not sort past that file.
+	for _, cursor := range []time.Time{
+		sealed.Add(11 * time.Second), // 12:31:10, the reported case
+		sealed.Add(59 * time.Second), // 12:31:58, the window's last entry
+		sealed,                       // exactly the window start
+	} {
+		scanMinute := persistedLogScanStart(cursor).Format("15-04")
+		if scanMinute > fileMinute {
+			t.Fatalf("cursor %v scans from %q, which sorts past the file %q holding it",
+				cursor.Format("15:04:05"), scanMinute, fileMinute)
+		}
+	}
+}
+
+// A cursor just after midnight has to reach back into the previous day.
+func TestPersistedLogScanStartCrossesMidnight(t *testing.T) {
+	cursor := time.Date(2026, 6, 29, 0, 0, 30, 0, time.UTC)
+	scanFrom := persistedLogScanStart(cursor)
+	if got, want := scanFrom.Format("2006-01-02"), "2026-06-28"; got != want {
+		t.Fatalf("scan date = %q, want %q so the previous day's last file is listed", got, want)
+	}
+}
+
+// TestSpanningLogFileIsNotSkipped pins the other half of the minute-boundary
+// fix, against the predicate the iterator actually calls. Widening which files
+// get listed is useless if the iterator then drops the spanning file, which it
+// did by treating the following file's name as an upper bound on this file's
+// contents -- it is not.
+func TestSpanningLogFileIsNotSkipped(t *testing.T) {
+	// Window sealed at 12:30:59, ending 12:31:58, written to "12-30".
+	spanning := time.Date(2026, 6, 29, 12, 30, 0, 0, time.UTC).UnixNano()
+	// The next window starts at 12:31:59 and is written to "12-31".
+	following := time.Date(2026, 6, 29, 12, 31, 0, 0, time.UTC).UnixNano()
+
+	cursor := time.Date(2026, 6, 29, 12, 31, 10, 0, time.UTC).UnixNano()
+	if following > cursor {
+		t.Fatal("precondition: the following file's name sorts at or before the cursor")
+	}
+	if !logFileMayContainAfter(spanning, cursor) {
+		t.Fatal("the spanning file holds entries past the cursor and must be read")
+	}
+
+	// A file that genuinely cannot reach the cursor is still skipped, so the
+	// widening does not turn into reading the whole day.
+	old := time.Date(2026, 6, 29, 12, 0, 0, 0, time.UTC).UnixNano()
+	if logFileMayContainAfter(old, cursor) {
+		t.Fatal("a file a full interval behind the cursor should still be skipped")
+	}
+}
+
+// TestClampLogRefsCursorNeverRewinds pins that a chunk-ref read cannot move the
+// subscriber backwards. The refs carry minute-level names, so the last one
+// normally sorts before the requested position, and the caller assigns that
+// value straight to the read cursor.
+func TestClampLogRefsCursorNeverRewinds(t *testing.T) {
+	cursor := time.Date(2026, 6, 29, 12, 31, 10, 0, time.UTC).UnixNano()
+	spanningFile := time.Date(2026, 6, 29, 12, 30, 0, 0, time.UTC).UnixNano()
+
+	if got := clampLogRefsCursor(spanningFile, cursor); got != cursor {
+		t.Fatalf("cursor moved to %v, want it held at %v", time.Unix(0, got), time.Unix(0, cursor))
+	}
+	// A ref genuinely ahead of the cursor still advances it.
+	ahead := time.Date(2026, 6, 29, 12, 32, 0, 0, time.UTC).UnixNano()
+	if got := clampLogRefsCursor(ahead, cursor); got != ahead {
+		t.Fatalf("cursor = %v, want it to advance to %v", time.Unix(0, got), time.Unix(0, ahead))
+	}
+}
+
+// TestPersistedLogScanStartTsNsMinuteAligned pins the prune bound against the
+// collector's minute-granular file comparison: a cursor at 12:31:20 still
+// collects the 12-30 file, so the bound must not sit past 12:30:00 - an
+// exact-ns bound disowns the file's sent state and the next pass reships it
+// whole, re-creating the duplicate-refs class the state exists to prevent.
+func TestPersistedLogScanStartTsNsMinuteAligned(t *testing.T) {
+	cursor := time.Date(2026, 6, 29, 12, 31, 20, 0, time.UTC)
+	fileTsNs := time.Date(2026, 6, 29, 12, 30, 0, 0, time.UTC).UnixNano()
+
+	bound := PersistedLogScanStartTsNs(cursor)
+	if fileTsNs < bound {
+		t.Fatalf("bound %v disowns the 12-30 file the collector still lists", time.Unix(0, bound).UTC())
+	}
+	// A file a full interval plus a minute behind is genuinely out of scan range.
+	old := time.Date(2026, 6, 29, 12, 28, 0, 0, time.UTC).UnixNano()
+	if old >= bound {
+		t.Fatalf("bound %v keeps state for files the scan can no longer list", time.Unix(0, bound).UTC())
+	}
+}
+
+// TestLastShippedLogEntryTsNs pins the chunk-mode cursor source against the
+// client's reader, which it must mirror: per file the readable prefix - the
+// client stops at the first unreadable chunk and never resumes within the file
+// - and per filer the newest file with content, because the client keeps
+// reading later files after skipping a bad one. A marker past what the client
+// applies loses events; one short only re-ships.
+func TestLastShippedLogEntryTsNs(t *testing.T) {
+	f := &Filer{persistedLogCache: newPersistedLogCache(1 << 20)}
+
+	entry := func(ts int64) *filer_pb.LogEntry { return &filer_pb.LogEntry{TsNs: ts} }
+	chunk := func(id string) *filer_pb.FileChunk { return &filer_pb.FileChunk{FileId: id, Size: 10} }
+	ref := func(fileTsNs int64, chunks ...*filer_pb.FileChunk) *filer_pb.LogFileChunkRef {
+		return &filer_pb.LogFileChunkRef{FilerId: "a", FileTsNs: fileTsNs, Chunks: chunks}
+	}
+
+	byId := map[string]struct {
+		entries []*filer_pb.LogEntry
+		err     error
+	}{
+		"good-early": {entries: []*filer_pb.LogEntry{entry(100), entry(200)}},
+		"good-late":  {entries: []*filer_pb.LogEntry{entry(300), entry(400)}},
+		"missing":    {err: fmt.Errorf("volume 7 not found")},
+		"missing2":   {err: fmt.Errorf("volume 8 not found")},
+		"empty":      {},
+	}
+	origLoad := loadLogFileEntriesFn
+	loadLogFileEntriesFn = func(masterClient *wdclient.MasterClient, c *filer_pb.FileChunk) ([]*filer_pb.LogEntry, bool, error) {
+		r := byId[c.FileId]
+		return r.entries, true, r.err
+	}
+	origLookup := lookupLogChunkFn
+	deadVolumes := map[string]bool{}
+	lookupLogChunkFn = func(f *Filer, fileId string) error {
+		if deadVolumes[fileId] {
+			return fmt.Errorf("volume 9 not found")
+		}
+		return nil
+	}
+	defer func() { loadLogFileEntriesFn = origLoad; lookupLogChunkFn = origLookup }()
+
+	t.Run("a fully readable file answers with its tail, complete", func(t *testing.T) {
+		ts, ok, complete := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("good-early"), chunk("good-late")})
+		if !ok || ts != 400 || !complete {
+			t.Fatalf("ts=%d ok=%v complete=%v, want 400 and complete", ts, ok, complete)
+		}
+	})
+
+	t.Run("the prefix ends at the first missing chunk", func(t *testing.T) {
+		// The client's reader stops there and never resumes within the file:
+		// answering from the readable suffix would put the marker past entries
+		// the client never applied, losing them permanently.
+		ts, ok, complete := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("good-early"), chunk("missing"), chunk("good-late")})
+		if !ok || ts != 200 {
+			t.Fatalf("ts=%d ok=%v, want the prefix end 200, never the suffix 400", ts, ok)
+		}
+		if complete {
+			t.Fatal("a prefix-limited read must report incomplete, or the unread suffix is never re-shipped")
+		}
+	})
+
+	t.Run("nothing readable is a clean no-answer", func(t *testing.T) {
+		if _, ok, _ := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("missing"), chunk("empty")}); ok {
+			t.Fatal("want no answer")
+		}
+	})
+
+	t.Run("an entirely missing last file falls back to earlier files", func(t *testing.T) {
+		// The client skips the bad file but has applied the earlier ones;
+		// discarding their progress would rewind the marker to the start
+		// cursor and stall or replay.
+		ts, answeredFileTsNs, ok, _ := f.LastShippedLogEntryTsNsForFiler([]*filer_pb.LogFileChunkRef{
+			ref(1000, chunk("good-late")),
+			ref(2000, chunk("missing"), chunk("missing2")),
+		})
+		if !ok || ts != 400 {
+			t.Fatalf("ts=%d ok=%v, want the previous file's 400", ts, ok)
+		}
+		if answeredFileTsNs != 1000 {
+			t.Fatalf("answered file %d, want 1000: the caller rolls back sent state above it", answeredFileTsNs)
+		}
+	})
+
+	t.Run("a cached chunk on a dead volume ends the prefix", func(t *testing.T) {
+		// Warm the cache, then kill the middle chunk's volume. The client
+		// reads directly and stops there; a probe trusting its own decoded
+		// cache would sail past to the tail and put the marker beyond entries
+		// the client never applied.
+		if ts, ok, _ := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("good-early"), chunk("good-late")}); !ok || ts != 400 {
+			t.Fatalf("warmup: ts=%d ok=%v, want 400", ts, ok)
+		}
+		deadVolumes["good-early"] = true
+		defer delete(deadVolumes, "good-early")
+		ts, ok, complete := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("good-early"), chunk("good-late")})
+		if ok || ts != 0 {
+			t.Fatalf("ts=%d ok=%v, want no answer: the first chunk's volume is gone however warm the cache", ts, ok)
+		}
+		if complete {
+			t.Fatal("a lookup-limited read must report incomplete so the ref re-ships")
+		}
+	})
+
+	t.Run("a dead needle behind a warm cache is the accepted residual", func(t *testing.T) {
+		// Production liveness is a volume lookup: it cannot see a needle 404
+		// or a stale location inside a resolvable volume, so a warm cache can
+		// answer past a chunk the client fails to read. Accepted because
+		// metadata log chunks die volume-at-a-time (TTL'd log volumes) and the
+		// alternative is a real read per probe, which is what the probe exists
+		// to avoid; the client logs the skip at V(0). This test pins the
+		// boundary so a change to it is a decision, not an accident.
+		if ts, ok, _ := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("good-early"), chunk("good-late")}); !ok || ts != 400 {
+			t.Fatalf("warmup: ts=%d ok=%v, want 400", ts, ok)
+		}
+		// The needle dies: loads fail, but the volume still resolves.
+		prev := byId["good-early"]
+		byId["good-early"] = struct {
+			entries []*filer_pb.LogEntry
+			err     error
+		}{err: fmt.Errorf("read: 404 Not Found: not found")}
+		defer func() { byId["good-early"] = prev }()
+		ts, ok, complete := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("good-early"), chunk("good-late")})
+		if !ok || ts != 400 || !complete {
+			t.Fatalf("ts=%d ok=%v complete=%v: the warm cache masks a dead needle - if this now fails, the residual was closed and this test should assert the new behavior", ts, ok, complete)
+		}
+	})
+
+	t.Run("spanning records stream the shipped list", func(t *testing.T) {
+		origLoad2 := loadLogFileEntriesFn
+		loadLogFileEntriesFn = func(masterClient *wdclient.MasterClient, c *filer_pb.FileChunk) ([]*filer_pb.LogEntry, bool, error) {
+			return nil, false, errLogChunkIncomplete
+		}
+		origStream := newLogFileStreamReader
+		newLogFileStreamReader = func(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) io.Reader {
+			var buf bytes.Buffer
+			for _, ts := range []int64{500, 600} {
+				data, _ := (&filer_pb.LogEntry{TsNs: ts}).MarshalVT()
+				var sizeBuf [4]byte
+				util.Uint32toBytes(sizeBuf[:], uint32(len(data)))
+				buf.Write(sizeBuf[:])
+				buf.Write(data)
+			}
+			// A torn trailing size prefix, as a crashed writer leaves it: the
+			// client reads it as a clean end, so the probe must too - failing
+			// here would block the marker forever on data the client accepts.
+			buf.Write([]byte{0x01, 0x02})
+			return &buf
+		}
+		defer func() { loadLogFileEntriesFn = origLoad2; newLogFileStreamReader = origStream }()
+
+		ts, ok, complete := f.lastShippedLogEntryTsNs([]*filer_pb.FileChunk{chunk("incomplete")})
+		if !ok || ts != 600 {
+			t.Fatalf("ts=%d ok=%v, want the streamed tail 600 past the torn prefix", ts, ok)
+		}
+		if !complete {
+			t.Fatal("a torn tail is where the client's read ends too: complete, nothing to re-ship")
+		}
+	})
+}

weed/filer/filer_notify_read_testing.go

@@ -0,0 +1,33 @@
+package filer
+
+import (
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+)
+
+// SetLogReadHooksForTesting swaps the volume-touching pieces of persisted log
+// reading - chunk decode, byte streaming, volume liveness - for fakes, so loop
+// tests can run the real subscribe machinery against an in-memory volume
+// layer. Returns a restore func. Test support only.
+func SetLogReadHooksForTesting(
+	load func(chunk *filer_pb.FileChunk) ([]*filer_pb.LogEntry, error),
+	stream func(chunks []*filer_pb.FileChunk) io.Reader,
+	lookup func(fileId string) error,
+) (restore func()) {
+	prevLoad, prevStream, prevLookup := loadLogFileEntriesFn, newLogFileStreamReader, lookupLogChunkFn
+	loadLogFileEntriesFn = func(masterClient *wdclient.MasterClient, chunk *filer_pb.FileChunk) ([]*filer_pb.LogEntry, bool, error) {
+		entries, err := load(chunk)
+		return entries, true, err
+	}
+	newLogFileStreamReader = func(masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) io.Reader {
+		return stream(chunks)
+	}
+	lookupLogChunkFn = func(f *Filer, fileId string) error {
+		return lookup(fileId)
+	}
+	return func() {
+		loadLogFileEntriesFn, newLogFileStreamReader, lookupLogChunkFn = prevLoad, prevStream, prevLookup
+	}
+}

weed/filer/filer_notify_test.go

@@ -0,0 +1,215 @@
+package filer
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
+	"google.golang.org/protobuf/proto"
+)
+
+func TestProtoMarshal(t *testing.T) {
+
+	oldEntry := &Entry{
+		FullPath: util.FullPath("/this/path/to"),
+		Attr: Attr{
+			Mtime:  time.Now(),
+			Mode:   0644,
+			Uid:    1,
+			Mime:   "text/json",
+			TtlSec: 25,
+		},
+		Chunks: []*filer_pb.FileChunk{
+			{
+				FileId:       "234,2423423422",
+				Offset:       234234,
+				Size:         234,
+				ModifiedTsNs: 12312423,
+				ETag:         "2342342354",
+				SourceFileId: "23234,2342342342",
+			},
+		},
+	}
+
+	notification := &filer_pb.EventNotification{
+		OldEntry:     oldEntry.ToProtoEntry(),
+		NewEntry:     nil,
+		DeleteChunks: true,
+	}
+
+	text, _ := proto.Marshal(notification)
+
+	notification2 := &filer_pb.EventNotification{}
+	proto.Unmarshal(text, notification2)
+
+	if notification2.OldEntry.GetChunks()[0].SourceFileId != notification.OldEntry.GetChunks()[0].SourceFileId {
+		t.Fatalf("marshal/unmarshal error: %s", text)
+	}
+
+	println(string(text))
+
+}
+
+// buildLogBuffer lays out records exactly as LogBuffer.AddDataToBuffer does:
+// a 4-byte size prefix in front of each marshaled LogEntry.
+func buildLogBuffer(t *testing.T, payloadSizes []int) (buf []byte, count int) {
+	t.Helper()
+	sizeBuf := make([]byte, 4)
+	for i, payloadSize := range payloadSizes {
+		data, err := proto.Marshal(&filer_pb.LogEntry{
+			TsNs: int64(i + 1),
+			Data: make([]byte, payloadSize),
+		})
+		if err != nil {
+			t.Fatalf("marshal log entry: %v", err)
+		}
+		util.Uint32toBytes(sizeBuf, uint32(len(data)))
+		buf = append(buf, sizeBuf...)
+		buf = append(buf, data...)
+	}
+	return buf, len(payloadSizes)
+}
+
+// splitLogBuffer drains a buffer the way logFlushFunc does, one piece at a
+// time, so the tests exercise the same walk.
+func splitLogBuffer(buf []byte, maxSize int) [][]byte {
+	var pieces [][]byte
+	for len(buf) > 0 {
+		piece := nextLogPiece(buf, maxSize)
+		pieces = append(pieces, piece)
+		buf = buf[len(piece):]
+	}
+	return pieces
+}
+
+func TestNextLogPiece(t *testing.T) {
+	const maxSize = 1024
+
+	testCases := []struct {
+		name         string
+		payloadSizes []int
+	}{
+		{"fits in one piece", []int{10, 20, 30}},
+		{"many small records", []int{300, 300, 300, 300, 300, 300, 300}},
+		{"one record far over the limit", []int{5000}},
+		{"oversized record between small ones", []int{100, 5000, 100}},
+		{"record exactly at the limit", []int{maxSize - 4 - 6}},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			buf, count := buildLogBuffer(t, tc.payloadSizes)
+
+			var rejoined []byte
+			for i, piece := range splitLogBuffer(buf, maxSize) {
+				if len(piece) > maxSize {
+					t.Errorf("piece %d is %d bytes, over the %d limit", i, len(piece), maxSize)
+				}
+				if len(piece) == 0 {
+					t.Errorf("piece %d is empty", i)
+				}
+				rejoined = append(rejoined, piece...)
+			}
+			if !bytes.Equal(rejoined, buf) {
+				t.Errorf("rejoined pieces differ from the original buffer")
+			}
+
+			// The pieces still stream back as the same records in order.
+			decoded, _, err := decodeLogRecords(rejoined)
+			if err != nil {
+				t.Fatalf("decode rejoined buffer: %v", err)
+			}
+			if len(decoded) != count {
+				t.Errorf("decoded %d records, want %d", len(decoded), count)
+			}
+		})
+	}
+}
+
+// A buffer of ordinary records splits on record boundaries, so every piece
+// decodes on its own and the readers keep the per-chunk cache path.
+func TestNextLogPieceKeepsRecordBoundaries(t *testing.T) {
+	buf, count := buildLogBuffer(t, []int{300, 300, 300, 300, 300, 300, 300})
+
+	var decodedCount int
+	for i, piece := range splitLogBuffer(buf, 1024) {
+		entries, cacheable, err := decodeLogRecords(piece)
+		if err != nil {
+			t.Fatalf("piece %d does not decode standalone: %v", i, err)
+		}
+		if !cacheable {
+			t.Errorf("piece %d is not cacheable", i)
+		}
+		decodedCount += len(entries)
+	}
+	if decodedCount != count {
+		t.Errorf("decoded %d records across pieces, want %d", decodedCount, count)
+	}
+}
+
+// A truncated tail must not be dropped or duplicated, only cut by size.
+func TestNextLogPieceTruncatedTail(t *testing.T) {
+	buf, _ := buildLogBuffer(t, []int{300, 300})
+	buf = append(buf, 0xff, 0xff, 0xff)
+
+	var rejoined []byte
+	for i, piece := range splitLogBuffer(buf, 320) {
+		if len(piece) > 320 {
+			t.Errorf("piece %d is %d bytes, over the 320 limit", i, len(piece))
+		}
+		rejoined = append(rejoined, piece...)
+	}
+	if !bytes.Equal(rejoined, buf) {
+		t.Errorf("rejoined pieces differ from the original buffer")
+	}
+}
+
+// A cluster whose fileSizeLimitMB is under metadataLogUploadLimit says so in
+// the rejection, and the flush has to take that limit up rather than retry an
+// unwritable piece forever.
+func TestVolumeFileSizeLimit(t *testing.T) {
+	testCases := []struct {
+		name string
+		err  error
+		want int
+	}{
+		{
+			"volume server size rejection",
+			fmt.Errorf("upload data http://127.0.0.1:8180/1,16ef8dca8a: unmarshalled error http://127.0.0.1:8180/1,16ef8dca8a: file over the limited 16777216 bytes"),
+			16777216,
+		},
+		{"default limit", errors.New("file over the limited 268435456 bytes"), 268435456},
+		{"unrelated failure", errors.New("connection refused"), 0},
+		{"no byte count", errors.New("file over the limited bytes"), 0},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := volumeFileSizeLimit(tc.err); got != tc.want {
+				t.Errorf("volumeFileSizeLimit = %d, want %d", got, tc.want)
+			}
+		})
+	}
+}
+
+// Once the reported limit is adopted, every piece fits it.
+func TestNextLogPieceHonorsLoweredLimit(t *testing.T) {
+	buf, _ := buildLogBuffer(t, []int{20 << 20})
+
+	const lowered = 1 << 20
+	var rejoined []byte
+	for i, piece := range splitLogBuffer(buf, lowered) {
+		if len(piece) > lowered {
+			t.Errorf("piece %d is %d bytes, over the lowered %d limit", i, len(piece), lowered)
+		}
+		rejoined = append(rejoined, piece...)
+	}
+	if !bytes.Equal(rejoined, buf) {
+		t.Errorf("rejoined pieces differ from the original buffer")
+	}
+}

weed/filer/filer_on_meta_event.go

@@ -0,0 +1,146 @@
+package filer
+
+import (
+	"bytes"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// onMetadataChangeEvent is triggered after filer processed change events from local or remote filers
+func (f *Filer) onMetadataChangeEvent(event *filer_pb.SubscribeMetadataResponse) {
+	f.maybeReloadFilerConfiguration(event)
+	f.maybeReloadRemoteStorageConfigurationAndMapping(event)
+	f.onBucketEvents(event)
+	f.onEmptyFolderCleanupEvents(event)
+}
+
+func (f *Filer) onBucketEvents(event *filer_pb.SubscribeMetadataResponse) {
+	message := event.EventNotification
+	oldDir := event.Directory
+	newDir := filer_pb.MetadataEventTargetDirectory(event)
+
+	if filer_pb.IsCreate(event) {
+		if newDir == f.DirBucketsPath && message.NewEntry.IsDirectory {
+			f.Store.OnBucketCreation(message.NewEntry.Name)
+		}
+	}
+	if filer_pb.IsDelete(event) {
+		if oldDir == f.DirBucketsPath && message.OldEntry.IsDirectory {
+			f.Store.OnBucketDeletion(message.OldEntry.Name)
+		}
+	}
+	if filer_pb.IsRename(event) {
+		if oldDir == f.DirBucketsPath && message.OldEntry.IsDirectory {
+			f.Store.OnBucketDeletion(message.OldEntry.Name)
+		}
+		if newDir == f.DirBucketsPath && message.NewEntry.IsDirectory {
+			f.Store.OnBucketCreation(message.NewEntry.Name)
+		}
+	}
+}
+
+// onEmptyFolderCleanupEvents handles create/delete events for empty folder cleanup
+func (f *Filer) onEmptyFolderCleanupEvents(event *filer_pb.SubscribeMetadataResponse) {
+	if f.EmptyFolderCleaner == nil || !f.EmptyFolderCleaner.IsEnabled() {
+		return
+	}
+
+	message := event.EventNotification
+	directory := event.Directory
+	eventTime := time.Unix(0, event.TsNs)
+
+	// Handle delete events - trigger folder cleanup check
+	if filer_pb.IsDelete(event) && message.OldEntry != nil {
+		f.EmptyFolderCleaner.OnDeleteEvent(directory, message.OldEntry.Name, message.OldEntry.IsDirectory, eventTime)
+	}
+
+	// Handle create events - cancel pending cleanup for the folder
+	if filer_pb.IsCreate(event) && message.NewEntry != nil {
+		f.EmptyFolderCleaner.OnCreateEvent(directory, message.NewEntry.Name, message.NewEntry.IsDirectory)
+	}
+
+	// Handle rename/move events
+	if filer_pb.IsRename(event) {
+		// Treat the old location as a delete
+		if message.OldEntry != nil {
+			f.EmptyFolderCleaner.OnDeleteEvent(directory, message.OldEntry.Name, message.OldEntry.IsDirectory, eventTime)
+		}
+		// Treat the new location as a create
+		if message.NewEntry != nil {
+			newDir := message.NewParentPath
+			if newDir == "" {
+				newDir = directory
+			}
+			f.EmptyFolderCleaner.OnCreateEvent(newDir, message.NewEntry.Name, message.NewEntry.IsDirectory)
+		}
+	}
+}
+
+func (f *Filer) maybeReloadFilerConfiguration(event *filer_pb.SubscribeMetadataResponse) {
+	if !filer_pb.MetadataEventTouchesDirectory(event, DirectoryEtcSeaweedFS) {
+		return
+	}
+
+	entry := event.EventNotification.NewEntry
+	if entry == nil {
+		return
+	}
+
+	glog.V(0).Infof("processing %v", event)
+	if entry.Name == FilerConfName {
+		f.reloadFilerConfiguration(entry)
+	}
+}
+
+func (f *Filer) readEntry(chunks []*filer_pb.FileChunk, size uint64) ([]byte, error) {
+	var buf bytes.Buffer
+	err := StreamContent(f.MasterClient, &buf, chunks, 0, int64(size))
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func (f *Filer) reloadFilerConfiguration(entry *filer_pb.Entry) {
+	fc := NewFilerConf()
+	err := fc.loadFromChunks(f, entry.Content, entry.GetChunks(), FileSize(entry))
+	if err != nil {
+		glog.Errorf("read filer conf chunks: %v", err)
+		return
+	}
+	f.FilerConf = fc
+}
+
+func (f *Filer) LoadFilerConf() {
+	fc := NewFilerConf()
+	err := util.Retry("loadFilerConf", func() error {
+		return fc.loadFromFiler(f)
+	})
+	if err != nil {
+		glog.Errorf("read filer conf: %v", err)
+		return
+	}
+	f.FilerConf = fc
+}
+
+// //////////////////////////////////
+// load and maintain remote storages
+// //////////////////////////////////
+func (f *Filer) LoadRemoteStorageConfAndMapping() {
+	if err := f.RemoteStorage.LoadRemoteStorageConfigurationsAndMapping(f); err != nil {
+		glog.Errorf("read remote conf and mapping: %v", err)
+		return
+	}
+}
+func (f *Filer) maybeReloadRemoteStorageConfigurationAndMapping(event *filer_pb.SubscribeMetadataResponse) {
+	if !filer_pb.MetadataEventTouchesDirectory(event, DirectoryEtcRemote) {
+		return
+	}
+
+	// a mount left behind after remote.unmount would still send deletes to a
+	// remote the filer no longer owns
+	f.LoadRemoteStorageConfAndMapping()
+}

weed/filer/filer_on_meta_event_test.go

@@ -0,0 +1,70 @@
+package filer
+
+import (
+	"context"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type bucketTrackingStore struct {
+	created []string
+	deleted []string
+}
+
+func (s *bucketTrackingStore) GetName() string { return "bucket-tracking" }
+func (s *bucketTrackingStore) Initialize(configuration util.Configuration, prefix string) error {
+	return nil
+}
+func (s *bucketTrackingStore) InsertEntry(context.Context, *Entry) error { return nil }
+func (s *bucketTrackingStore) UpdateEntry(context.Context, *Entry) error { return nil }
+func (s *bucketTrackingStore) FindEntry(context.Context, util.FullPath) (*Entry, error) {
+	return nil, filer_pb.ErrNotFound
+}
+func (s *bucketTrackingStore) DeleteEntry(context.Context, util.FullPath) error          { return nil }
+func (s *bucketTrackingStore) DeleteFolderChildren(context.Context, util.FullPath) error { return nil }
+func (s *bucketTrackingStore) ListDirectoryEntries(context.Context, util.FullPath, string, bool, int64, ListEachEntryFunc) (string, error) {
+	return "", nil
+}
+func (s *bucketTrackingStore) ListDirectoryPrefixedEntries(context.Context, util.FullPath, string, bool, int64, string, ListEachEntryFunc) (string, error) {
+	return "", nil
+}
+func (s *bucketTrackingStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (s *bucketTrackingStore) CommitTransaction(context.Context) error     { return nil }
+func (s *bucketTrackingStore) RollbackTransaction(context.Context) error   { return nil }
+func (s *bucketTrackingStore) KvPut(context.Context, []byte, []byte) error { return nil }
+func (s *bucketTrackingStore) KvGet(context.Context, []byte) ([]byte, error) {
+	return nil, ErrKvNotFound
+}
+func (s *bucketTrackingStore) KvDelete(context.Context, []byte) error { return nil }
+func (s *bucketTrackingStore) Shutdown()                              {}
+func (s *bucketTrackingStore) OnBucketCreation(bucket string)         { s.created = append(s.created, bucket) }
+func (s *bucketTrackingStore) OnBucketDeletion(bucket string)         { s.deleted = append(s.deleted, bucket) }
+func (s *bucketTrackingStore) CanDropWholeBucket() bool               { return false }
+
+func TestOnBucketEventsRenameIntoBucketsRootCreatesBucket(t *testing.T) {
+	store := &bucketTrackingStore{}
+	f := &Filer{
+		DirBucketsPath: "/buckets",
+		Store:          NewFilerStoreWrapper(store),
+	}
+
+	f.onBucketEvents(&filer_pb.SubscribeMetadataResponse{
+		Directory: "/tmp",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: "migrated", IsDirectory: true},
+			NewEntry:      &filer_pb.Entry{Name: "migrated", IsDirectory: true},
+			NewParentPath: "/buckets",
+		},
+	})
+
+	if len(store.created) != 1 || store.created[0] != "migrated" {
+		t.Fatalf("created buckets = %v, want [migrated]", store.created)
+	}
+	if len(store.deleted) != 0 {
+		t.Fatalf("deleted buckets = %v, want []", store.deleted)
+	}
+}

weed/filer/filer_prefix_object_test.go

@@ -0,0 +1,106 @@
+package filer
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestPromoteToPrefixObject covers a key written before the keys nested under it:
+// the file becomes the directory they live in, and has to stay an object of its own.
+func TestPromoteToPrefixObject(t *testing.T) {
+	f, store := newTestFilerWithStubStore()
+	ctx := context.Background()
+
+	object := &Entry{
+		FullPath: util.FullPath("/buckets/bkt/a/foo"),
+		Attr: Attr{
+			Mode:   0o644,
+			Mime:   "text/plain",
+			TtlSec: 3600,
+		},
+		Chunks:   []*filer_pb.FileChunk{{FileId: "1,01", Size: 4}},
+		Extended: map[string][]byte{s3_constants.SeaweedFSExpiresS3: []byte("true")},
+	}
+	require.NoError(t, f.CreateEntry(ctx, object, nil, false, false, nil, false, f.MaxFilenameLength))
+
+	nested := &Entry{FullPath: util.FullPath("/buckets/bkt/a/foo/bar"), Attr: Attr{Mode: 0o644}}
+	require.NoError(t, f.CreateEntry(ctx, nested, nil, false, false, nil, false, f.MaxFilenameLength))
+
+	promoted, err := store.FindEntry(ctx, object.FullPath)
+	require.NoError(t, err)
+	require.NotNil(t, promoted)
+
+	assert.True(t, promoted.IsDirectory(), "the nested key needs a directory here")
+	assert.Equal(t, object.Chunks, promoted.Chunks, "the object's data stays on it")
+	assert.Contains(t, promoted.Extended, s3_constants.SeaweedFSPrefixObject)
+	// Expiring the entry deletes the directory row on its own and strands the keys
+	// under it, so the promotion gives up the lazy TTL.
+	assert.Zero(t, promoted.Attr.TtlSec)
+}
+
+// TestExpiredDirectoryIsNotDeletedOnRead pins the other half: a TTL an older build
+// left on a promoted directory must not take the keys under it with it.
+func TestExpiredDirectoryIsNotDeletedOnRead(t *testing.T) {
+	f, store := newTestFilerWithStubStore()
+	ctx := context.Background()
+
+	dirPath := util.FullPath("/buckets/bkt/a/foo")
+	expired := time.Now().Add(-2 * time.Hour)
+	require.NoError(t, store.InsertEntry(ctx, &Entry{
+		FullPath: dirPath,
+		Attr: Attr{
+			Mode:   os.ModeDir | 0o755,
+			Crtime: expired,
+			Mtime:  expired,
+			TtlSec: 60,
+		},
+		Extended: map[string][]byte{s3_constants.SeaweedFSExpiresS3: []byte("true")},
+	}))
+	nested := &Entry{FullPath: dirPath + "/bar", Attr: Attr{Mode: 0o644}}
+	require.NoError(t, store.InsertEntry(ctx, nested))
+
+	found, err := f.FindEntry(ctx, dirPath)
+	require.NoError(t, err)
+	require.NotNil(t, found, "deleting it here would leave the nested key unreachable")
+
+	stillThere, err := store.FindEntry(ctx, nested.FullPath)
+	require.NoError(t, err)
+	require.NotNil(t, stillThere)
+}
+
+// TestExpiredFileIsDeletedOnRead pins the native cache-TTL path: regular
+// entries become invisible and their metadata row is removed without a
+// cache-specific transaction or reverse Volume index.
+func TestExpiredFileIsDeletedOnRead(t *testing.T) {
+	f, store := newTestFilerWithStubStore()
+	ctx := context.Background()
+
+	filePath := util.FullPath("/buckets/lmcache/expired")
+	expired := time.Now().Add(-2 * time.Hour)
+	require.NoError(t, store.InsertEntry(ctx, &Entry{
+		FullPath: filePath,
+		Attr: Attr{
+			Mode:   0o644,
+			Crtime: expired,
+			Mtime:  expired,
+			TtlSec: 60,
+		},
+		Chunks: []*filer_pb.FileChunk{{FileId: "7,01", Size: 4}},
+	}))
+
+	found, err := f.FindEntry(ctx, filePath)
+	require.ErrorIs(t, err, filer_pb.ErrNotFound)
+	require.Nil(t, found)
+
+	_, err = store.FindEntry(ctx, filePath)
+	require.ErrorIs(t, err, filer_pb.ErrNotFound,
+		"native TTL lookup should remove the expired metadata row")
+}

weed/filer/filer_rename.go

@@ -0,0 +1,48 @@
+package filer
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func (f *Filer) CanRename(ctx context.Context, source, target util.FullPath, oldName string) error {
+	sourcePath := source.Child(oldName)
+	if strings.HasPrefix(string(target), string(sourcePath)) {
+		return fmt.Errorf("mv: can not move directory to a subdirectory of itself")
+	}
+
+	// Check if attempting to rename a bucket itself
+	// Need to load the entry to check if it's a bucket
+	entry, err := f.FindEntry(ctx, sourcePath)
+	if err != nil {
+		return err
+	}
+	if f.IsBucket(entry) {
+		return fmt.Errorf("bucket renaming is not allowed")
+	}
+
+	sourceBucket := f.DetectBucket(source)
+	targetBucket := f.DetectBucket(target)
+	if sourceBucket != targetBucket {
+		return fmt.Errorf("can not move across collection %s => %s", sourceBucket, targetBucket)
+	}
+
+	return nil
+}
+
+func (f *Filer) DetectBucket(source util.FullPath) (bucket string) {
+	if strings.HasPrefix(string(source), f.DirBucketsPath+"/") {
+		bucketAndObjectKey := string(source)[len(f.DirBucketsPath)+1:]
+		t := strings.Index(bucketAndObjectKey, "/")
+		if t < 0 {
+			bucket = bucketAndObjectKey
+		}
+		if t > 0 {
+			bucket = bucketAndObjectKey[:t]
+		}
+	}
+	return bucket
+}

weed/filer/filer_rename_test.go

@@ -0,0 +1,31 @@
+package filer
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestDetectBucket(t *testing.T) {
+	tests := []struct {
+		name        string
+		fullPath    string
+		bucketsPath string
+		want        string
+	}{
+		{"object in bucket", "/buckets/mybucket/a/b.txt", "/buckets", "mybucket"},
+		{"bucket root", "/buckets/mybucket", "/buckets", "mybucket"},
+		{"not under buckets path", "/other/path/x", "/buckets", ""},
+		{"buckets path itself", "/buckets", "/buckets", ""},
+		{"custom buckets path", "/data/buckets/mybucket/x", "/data/buckets", "mybucket"},
+		{"nested object", "/buckets/b/deep/nested/key", "/buckets", "b"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := &Filer{DirBucketsPath: tt.bucketsPath}
+			if got := f.DetectBucket(util.FullPath(tt.fullPath)); got != tt.want {
+				t.Errorf("DetectBucket(%q) with buckets path %q = %q, want %q", tt.fullPath, tt.bucketsPath, got, tt.want)
+			}
+		})
+	}
+}

weed/filer/filer_search.go

@@ -0,0 +1,125 @@
+package filer
+
+import (
+	"context"
+	"math"
+	"path/filepath"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func splitPattern(pattern string) (prefix string, restPattern string) {
+	position := strings.Index(pattern, "*")
+	if position >= 0 {
+		return pattern[:position], pattern[position:]
+	}
+	position = strings.Index(pattern, "?")
+	if position >= 0 {
+		return pattern[:position], pattern[position:]
+	}
+	return "", restPattern
+}
+
+// For now, prefix and namePattern are mutually exclusive
+func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, namePatternExclude string) (entries []*Entry, hasMore bool, err error) {
+
+	if limit > math.MaxInt32-1 {
+		limit = math.MaxInt32 - 1
+	}
+
+	_, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, limit+1, prefix, namePattern, namePatternExclude, func(entry *Entry) (bool, error) {
+		entries = append(entries, entry)
+		return true, nil
+	})
+
+	hasMore = int64(len(entries)) >= limit+1
+	if hasMore {
+		entries = entries[:limit]
+	}
+
+	return entries, hasMore, err
+}
+
+// CountDirectoryEntries counts entries in a directory up to limit
+func (f *Filer) CountDirectoryEntries(ctx context.Context, p util.FullPath, limit int) (count int, err error) {
+	entries, hasMore, err := f.ListDirectoryEntries(ctx, p, "", false, int64(limit), "", "", "")
+	if err != nil {
+		return 0, err
+	}
+	count = len(entries)
+	if hasMore {
+		count = limit // At least this many
+	}
+	return count, nil
+}
+
+// For now, prefix and namePattern are mutually exclusive
+func (f *Filer) StreamListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
+	if strings.HasSuffix(string(p), "/") && len(p) > 1 {
+		p = p[0 : len(p)-1]
+	}
+
+	prefixInNamePattern, restNamePattern := splitPattern(namePattern)
+	if prefixInNamePattern != "" {
+		prefix = prefixInNamePattern
+	}
+	var missedCount int64
+
+	missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, startFileName, inclusive, limit, prefix, restNamePattern, namePatternExclude, eachEntryFunc)
+
+	for missedCount > 0 && err == nil {
+		missedCount, lastFileName, err = f.doListPatternMatchedEntries(ctx, p, lastFileName, false, missedCount, prefix, restNamePattern, namePatternExclude, eachEntryFunc)
+	}
+
+	return
+}
+
+func (f *Filer) doListPatternMatchedEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix, restNamePattern string, namePatternExclude string, eachEntryFunc ListEachEntryFunc) (missedCount int64, lastFileName string, err error) {
+
+	if len(restNamePattern) == 0 && len(namePatternExclude) == 0 {
+		lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix, eachEntryFunc)
+		return 0, lastFileName, err
+	}
+
+	lastFileName, err = f.doListValidEntries(ctx, p, startFileName, inclusive, limit, prefix, func(entry *Entry) (bool, error) {
+		nameToTest := entry.Name()
+		if len(namePatternExclude) > 0 {
+			if matched, matchErr := filepath.Match(namePatternExclude, nameToTest); matchErr == nil && matched {
+				missedCount++
+				return true, nil
+			}
+		}
+		if len(restNamePattern) > 0 {
+			if matched, matchErr := filepath.Match(restNamePattern, nameToTest[len(prefix):]); matchErr == nil && !matched {
+				missedCount++
+				return true, nil
+			}
+		}
+
+		res, resErr := eachEntryFunc(entry)
+
+		if resErr != nil {
+			return false, resErr
+		}
+
+		if !res {
+			return false, nil
+		}
+
+		return true, nil
+	})
+	if err != nil {
+		return
+	}
+	return
+}
+
+func (f *Filer) doListValidEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
+	var expiredCount int64
+	expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, startFileName, inclusive, limit, prefix, eachEntryFunc)
+	for expiredCount > 0 && err == nil {
+		expiredCount, lastFileName, err = f.doListDirectoryEntries(ctx, p, lastFileName, false, expiredCount, prefix, eachEntryFunc)
+	}
+	return
+}

weed/filer/filerstore.go

@@ -0,0 +1,55 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const CountEntryChunksForGzip = 50
+
+var (
+	ErrUnsupportedListDirectoryPrefixed      = errors.New("unsupported directory prefix listing")
+	ErrUnsupportedSuperLargeDirectoryListing = errors.New("unsupported super large directory listing")
+	ErrKvNotImplemented                      = errors.New("kv not implemented yet")
+	ErrKvNotFound                            = errors.New("kv: not found")
+)
+
+type ListEachEntryFunc func(entry *Entry) (bool, error)
+
+type FilerStore interface {
+	// GetName gets the name to locate the configuration in filer.toml file
+	GetName() string
+	// Initialize initializes the file store
+	Initialize(configuration util.Configuration, prefix string) error
+	InsertEntry(context.Context, *Entry) error
+	UpdateEntry(context.Context, *Entry) (err error)
+	// err == filer_pb.ErrNotFound if not found
+	FindEntry(context.Context, util.FullPath) (entry *Entry, err error)
+	DeleteEntry(context.Context, util.FullPath) (err error)
+	DeleteFolderChildren(context.Context, util.FullPath) (err error)
+	ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)
+	ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error)
+
+	BeginTransaction(ctx context.Context) (context.Context, error)
+	CommitTransaction(ctx context.Context) error
+	RollbackTransaction(ctx context.Context) error
+
+	KvPut(ctx context.Context, key []byte, value []byte) (err error)
+	KvGet(ctx context.Context, key []byte) (value []byte, err error)
+	KvDelete(ctx context.Context, key []byte) (err error)
+
+	Shutdown()
+}
+
+type BucketAware interface {
+	OnBucketCreation(bucket string)
+	OnBucketDeletion(bucket string)
+	CanDropWholeBucket() bool
+}
+
+type Debuggable interface {
+	Debug(writer io.Writer)
+}

weed/filer/filerstore_hardlink.go

@@ -0,0 +1,129 @@
+package filer
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func (fsw *FilerStoreWrapper) handleUpdateToHardLinks(ctx context.Context, entry *Entry) error {
+
+	if entry.IsDirectory() {
+		return nil
+	}
+
+	if len(entry.HardLinkId) > 0 {
+		// handle hard links
+		glog.V(4).InfofCtx(ctx, "handleUpdateToHardLinks %s HardLinkId %x counter=%d chunks=%d",
+			entry.FullPath, entry.HardLinkId, entry.HardLinkCounter, len(entry.GetChunks()))
+		if err := fsw.setHardLink(ctx, entry); err != nil {
+			return fmt.Errorf("setHardLink %d: %v", entry.HardLinkId, err)
+		}
+	}
+
+	// check what is existing entry
+	// glog.V(4).Infof("handleUpdateToHardLinks FindEntry %s", entry.FullPath)
+	actualStore := fsw.getActualStore(entry.FullPath)
+	existingEntry, err := actualStore.FindEntry(ctx, entry.FullPath)
+	if err != nil && err != filer_pb.ErrNotFound {
+		return fmt.Errorf("update existing entry %s: %v", entry.FullPath, err)
+	}
+
+	// remove old hard link
+	if err == nil && len(existingEntry.HardLinkId) != 0 && bytes.Compare(existingEntry.HardLinkId, entry.HardLinkId) != 0 {
+		glog.V(4).InfofCtx(ctx, "handleUpdateToHardLinks DeleteHardLink %s", entry.FullPath)
+		if err = fsw.DeleteHardLink(ctx, existingEntry.HardLinkId); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (fsw *FilerStoreWrapper) setHardLink(ctx context.Context, entry *Entry) error {
+	if len(entry.HardLinkId) == 0 {
+		return nil
+	}
+	key := entry.HardLinkId
+
+	newBlob, encodeErr := entry.EncodeAttributesAndChunks()
+	if encodeErr != nil {
+		return encodeErr
+	}
+
+	glog.V(4).InfofCtx(ctx, "setHardLink KvPut %s HardLinkId %x nlink:%d blobLen=%d",
+		entry.FullPath, entry.HardLinkId, entry.HardLinkCounter, len(newBlob))
+
+	return fsw.KvPut(ctx, key, newBlob)
+}
+
+func (fsw *FilerStoreWrapper) maybeReadHardLink(ctx context.Context, entry *Entry) error {
+	if len(entry.HardLinkId) == 0 {
+		return nil
+	}
+	key := entry.HardLinkId
+
+	glog.V(1).InfofCtx(ctx, "maybeReadHardLink %s HardLinkId %x", entry.FullPath, entry.HardLinkId)
+
+	value, err := fsw.KvGet(ctx, key)
+	if err != nil {
+		if errors.Is(err, ErrKvNotFound) {
+			glog.V(4).InfofCtx(ctx, "maybeReadHardLink %s HardLinkId %x: not found", entry.FullPath, entry.HardLinkId)
+		} else {
+			glog.ErrorfCtx(ctx, "read %s HardLinkId %x: %v", entry.FullPath, entry.HardLinkId, err)
+		}
+		return err
+	}
+
+	if err = entry.DecodeAttributesAndChunks(value); err != nil {
+		glog.ErrorfCtx(ctx, "decode %s HardLinkId %x: %v", entry.FullPath, entry.HardLinkId, err)
+		return err
+	}
+
+	glog.V(1).InfofCtx(ctx, "maybeReadHardLink %s HardLinkId %x nlink:%d chunks=%d",
+		entry.FullPath, entry.HardLinkId, entry.HardLinkCounter, len(entry.GetChunks()))
+
+	return nil
+}
+
+func (fsw *FilerStoreWrapper) DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error {
+	key := hardLinkId
+	glog.V(4).InfofCtx(ctx, "DeleteHardLink HardLinkId %x", key)
+	value, err := fsw.KvGet(ctx, key)
+	if err == ErrKvNotFound {
+		glog.V(4).InfofCtx(ctx, "DeleteHardLink HardLinkId %x: already gone", key)
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	entry := &Entry{}
+	if err = entry.DecodeAttributesAndChunks(value); err != nil {
+		return err
+	}
+
+	entry.HardLinkCounter--
+	if entry.HardLinkCounter <= 0 {
+		glog.V(4).InfofCtx(ctx, "DeleteHardLink KvDelete HardLinkId %x counter reached %d",
+			key, entry.HardLinkCounter)
+		return fsw.KvDelete(ctx, key)
+	}
+
+	// POSIX: update ctime when nlink changes (a hard link was removed).
+	entry.Attr.Ctime = time.Now().UTC()
+
+	newBlob, encodeErr := entry.EncodeAttributesAndChunks()
+	if encodeErr != nil {
+		return encodeErr
+	}
+
+	glog.V(4).InfofCtx(ctx, "DeleteHardLink KvPut HardLinkId %x counter decremented to %d",
+		key, entry.HardLinkCounter)
+	return fsw.KvPut(ctx, key, newBlob)
+
+}

weed/filer/filerstore_translate_path.go

@@ -0,0 +1,159 @@
+package filer
+
+import (
+	"context"
+	"math"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var (
+	_ = FilerStore(&FilerStorePathTranslator{})
+)
+
+type FilerStorePathTranslator struct {
+	actualStore FilerStore
+	storeRoot   string
+}
+
+func NewFilerStorePathTranslator(storeRoot string, store FilerStore) *FilerStorePathTranslator {
+	if innerStore, ok := store.(*FilerStorePathTranslator); ok {
+		return innerStore
+	}
+
+	if !strings.HasSuffix(storeRoot, "/") {
+		storeRoot += "/"
+	}
+
+	return &FilerStorePathTranslator{
+		actualStore: store,
+		storeRoot:   storeRoot,
+	}
+}
+
+func (t *FilerStorePathTranslator) translatePath(fp util.FullPath) (newPath util.FullPath) {
+	newPath = fp
+	if t.storeRoot == "/" {
+		return
+	}
+	newPath = fp[len(t.storeRoot)-1:]
+	if newPath == "" {
+		newPath = "/"
+	}
+	return
+}
+func (t *FilerStorePathTranslator) changeEntryPath(entry *Entry) (previousPath util.FullPath) {
+	previousPath = entry.FullPath
+	if t.storeRoot == "/" {
+		return
+	}
+	entry.FullPath = t.translatePath(previousPath)
+	return
+}
+func (t *FilerStorePathTranslator) recoverEntryPath(entry *Entry, previousPath util.FullPath) {
+	entry.FullPath = previousPath
+}
+
+func (t *FilerStorePathTranslator) GetName() string {
+	return t.actualStore.GetName()
+}
+
+func (t *FilerStorePathTranslator) Initialize(configuration util.Configuration, prefix string) error {
+	return t.actualStore.Initialize(configuration, prefix)
+}
+
+func (t *FilerStorePathTranslator) InsertEntry(ctx context.Context, entry *Entry) error {
+	previousPath := t.changeEntryPath(entry)
+	defer t.recoverEntryPath(entry, previousPath)
+
+	return t.actualStore.InsertEntry(ctx, entry)
+}
+
+func (t *FilerStorePathTranslator) UpdateEntry(ctx context.Context, entry *Entry) error {
+	previousPath := t.changeEntryPath(entry)
+	defer t.recoverEntryPath(entry, previousPath)
+
+	return t.actualStore.UpdateEntry(ctx, entry)
+}
+
+func (t *FilerStorePathTranslator) FindEntry(ctx context.Context, fp util.FullPath) (entry *Entry, err error) {
+	if t.storeRoot == "/" {
+		return t.actualStore.FindEntry(ctx, fp)
+	}
+	newFullPath := t.translatePath(fp)
+	entry, err = t.actualStore.FindEntry(ctx, newFullPath)
+	if err == nil {
+		entry.FullPath = fp[:len(t.storeRoot)-1] + entry.FullPath
+	}
+	return
+}
+
+func (t *FilerStorePathTranslator) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) {
+	newFullPath := t.translatePath(fp)
+	return t.actualStore.DeleteEntry(ctx, newFullPath)
+}
+
+func (t *FilerStorePathTranslator) DeleteOneEntry(ctx context.Context, existingEntry *Entry) (err error) {
+
+	previousPath := t.changeEntryPath(existingEntry)
+	defer t.recoverEntryPath(existingEntry, previousPath)
+
+	return t.actualStore.DeleteEntry(ctx, existingEntry.FullPath)
+}
+
+func (t *FilerStorePathTranslator) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
+	newFullPath := t.translatePath(fp)
+
+	return t.actualStore.DeleteFolderChildren(ctx, newFullPath)
+}
+
+func (t *FilerStorePathTranslator) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) {
+
+	newFullPath := t.translatePath(dirPath)
+
+	return t.actualStore.ListDirectoryEntries(ctx, newFullPath, startFileName, includeStartFile, limit, func(entry *Entry) (bool, error) {
+		entry.FullPath = dirPath[:len(t.storeRoot)-1] + entry.FullPath
+		return eachEntryFunc(entry)
+	})
+}
+
+func (t *FilerStorePathTranslator) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (string, error) {
+
+	newFullPath := t.translatePath(dirPath)
+
+	if limit > math.MaxInt32-1 {
+		limit = math.MaxInt32 - 1
+	}
+
+	return t.actualStore.ListDirectoryPrefixedEntries(ctx, newFullPath, startFileName, includeStartFile, limit, prefix, func(entry *Entry) (bool, error) {
+		entry.FullPath = dirPath[:len(t.storeRoot)-1] + entry.FullPath
+		return eachEntryFunc(entry)
+	})
+}
+
+func (t *FilerStorePathTranslator) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return t.actualStore.BeginTransaction(ctx)
+}
+
+func (t *FilerStorePathTranslator) CommitTransaction(ctx context.Context) error {
+	return t.actualStore.CommitTransaction(ctx)
+}
+
+func (t *FilerStorePathTranslator) RollbackTransaction(ctx context.Context) error {
+	return t.actualStore.RollbackTransaction(ctx)
+}
+
+func (t *FilerStorePathTranslator) Shutdown() {
+	t.actualStore.Shutdown()
+}
+
+func (t *FilerStorePathTranslator) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+	return t.actualStore.KvPut(ctx, key, value)
+}
+func (t *FilerStorePathTranslator) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+	return t.actualStore.KvGet(ctx, key)
+}
+func (t *FilerStorePathTranslator) KvDelete(ctx context.Context, key []byte) (err error) {
+	return t.actualStore.KvDelete(ctx, key)
+}

weed/filer/filerstore_wrapper.go

@@ -0,0 +1,464 @@
+package filer
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/viant/ptrie"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var (
+	_ = VirtualFilerStore(&FilerStoreWrapper{})
+	_ = Debuggable(&FilerStoreWrapper{})
+)
+
+type VirtualFilerStore interface {
+	FilerStore
+	DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error
+	DeleteOneEntry(ctx context.Context, entry *Entry) error
+	InsertEntryKnownAbsent(ctx context.Context, entry *Entry) error
+	AddPathSpecificStore(path string, storeId string, store FilerStore)
+	OnBucketCreation(bucket string)
+	OnBucketDeletion(bucket string)
+	CanDropWholeBucket() bool
+	SameActualStore(a, b util.FullPath) bool
+}
+
+type FilerStoreWrapper struct {
+	defaultStore         FilerStore
+	pathToStore          ptrie.Trie[string]
+	storeIdToStore       map[string]FilerStore
+	hasPathSpecificStore bool // fast check to skip MatchPrefix when no path-specific stores
+}
+
+func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper {
+	if innerStore, ok := store.(*FilerStoreWrapper); ok {
+		return innerStore
+	}
+	return &FilerStoreWrapper{
+		defaultStore:   store,
+		pathToStore:    ptrie.New[string](),
+		storeIdToStore: make(map[string]FilerStore),
+	}
+}
+
+func (fsw *FilerStoreWrapper) CanDropWholeBucket() bool {
+	if ba, ok := fsw.defaultStore.(BucketAware); ok {
+		return ba.CanDropWholeBucket()
+	}
+	return false
+}
+
+func (fsw *FilerStoreWrapper) OnBucketCreation(bucket string) {
+	for _, store := range fsw.storeIdToStore {
+		if ba, ok := store.(BucketAware); ok {
+			ba.OnBucketCreation(bucket)
+		}
+	}
+	if ba, ok := fsw.defaultStore.(BucketAware); ok {
+		ba.OnBucketCreation(bucket)
+	}
+}
+func (fsw *FilerStoreWrapper) OnBucketDeletion(bucket string) {
+	for _, store := range fsw.storeIdToStore {
+		if ba, ok := store.(BucketAware); ok {
+			ba.OnBucketDeletion(bucket)
+		}
+	}
+	if ba, ok := fsw.defaultStore.(BucketAware); ok {
+		ba.OnBucketDeletion(bucket)
+	}
+}
+
+func (fsw *FilerStoreWrapper) AddPathSpecificStore(path string, storeId string, store FilerStore) {
+	fsw.storeIdToStore[storeId] = NewFilerStorePathTranslator(path, store)
+	err := fsw.pathToStore.Put([]byte(path), storeId)
+	if err != nil {
+		glog.Fatalf("put path specific store: %v", err)
+	}
+	fsw.hasPathSpecificStore = true
+}
+
+func (fsw *FilerStoreWrapper) getActualStore(path util.FullPath) (store FilerStore) {
+	store = fsw.defaultStore
+	// Fast path: skip MatchPrefix if no path-specific stores are configured (common case)
+	if !fsw.hasPathSpecificStore {
+		return
+	}
+	if path == "/" || path == "//" {
+		return
+	}
+	var storeId string
+	fsw.pathToStore.MatchPrefix([]byte(path), func(key []byte, value string) bool {
+		storeId = value
+		return false
+	})
+	if storeId != "" {
+		store = fsw.storeIdToStore[storeId]
+	}
+	return
+}
+
+// SameActualStore reports whether two paths resolve to the same underlying
+// store. When path-specific stores are configured, different subtrees may
+// be served by different backends.
+func (fsw *FilerStoreWrapper) SameActualStore(a, b util.FullPath) bool {
+	return fsw.getActualStore(a) == fsw.getActualStore(b)
+}
+
+func (fsw *FilerStoreWrapper) getDefaultStore() (store FilerStore) {
+	return fsw.defaultStore
+}
+
+func (fsw *FilerStoreWrapper) GetName() string {
+	return fsw.getDefaultStore().GetName()
+}
+
+func (fsw *FilerStoreWrapper) Initialize(configuration util.Configuration, prefix string) error {
+	return fsw.getDefaultStore().Initialize(configuration, prefix)
+}
+
+func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	ctx = context.WithoutCancel(ctx)
+	actualStore := fsw.getActualStore(entry.FullPath)
+	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "insert").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "insert").Observe(time.Since(start).Seconds())
+	}()
+
+	filer_pb.BeforeEntrySerialization(entry.GetChunks())
+	normalizeEntryMimeForStore(entry)
+
+	if len(entry.HardLinkId) > 0 {
+		glog.V(4).InfofCtx(ctx, "InsertEntry %s has HardLinkId %x counter=%d",
+			entry.FullPath, entry.HardLinkId, entry.HardLinkCounter)
+	}
+
+	if err := fsw.handleUpdateToHardLinks(ctx, entry); err != nil {
+		return err
+	}
+
+	return actualStore.InsertEntry(ctx, entry)
+}
+
+// InsertEntryKnownAbsent skips the pre-insert FindEntry path when the caller has
+// already established that the target path does not exist.
+func (fsw *FilerStoreWrapper) InsertEntryKnownAbsent(ctx context.Context, entry *Entry) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	ctx = context.WithoutCancel(ctx)
+	actualStore := fsw.getActualStore(entry.FullPath)
+	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "insert").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "insert").Observe(time.Since(start).Seconds())
+	}()
+
+	filer_pb.BeforeEntrySerialization(entry.GetChunks())
+	normalizeEntryMimeForStore(entry)
+
+	if len(entry.HardLinkId) > 0 {
+		glog.V(4).InfofCtx(ctx, "InsertEntryKnownAbsent %s has HardLinkId %x counter=%d",
+			entry.FullPath, entry.HardLinkId, entry.HardLinkCounter)
+		if err := fsw.setHardLink(ctx, entry); err != nil {
+			return fmt.Errorf("setHardLink %x: %v", entry.HardLinkId, err)
+		}
+	}
+
+	return actualStore.InsertEntry(ctx, entry)
+}
+
+func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	ctx = context.WithoutCancel(ctx)
+	actualStore := fsw.getActualStore(entry.FullPath)
+	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "update").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "update").Observe(time.Since(start).Seconds())
+	}()
+
+	filer_pb.BeforeEntrySerialization(entry.GetChunks())
+	normalizeEntryMimeForStore(entry)
+
+	if len(entry.HardLinkId) > 0 {
+		glog.V(4).InfofCtx(ctx, "UpdateEntry %s has HardLinkId %x counter=%d",
+			entry.FullPath, entry.HardLinkId, entry.HardLinkCounter)
+	}
+
+	if err := fsw.handleUpdateToHardLinks(ctx, entry); err != nil {
+		return err
+	}
+
+	return actualStore.UpdateEntry(ctx, entry)
+}
+
+func normalizeEntryMimeForStore(entry *Entry) {
+	if entry == nil {
+		return
+	}
+	if !entry.IsDirectory() && entry.Mime == "application/octet-stream" {
+		entry.Mime = ""
+	}
+}
+
+func (fsw *FilerStoreWrapper) FindEntry(ctx context.Context, fp util.FullPath) (entry *Entry, err error) {
+	ctx = context.WithoutCancel(ctx)
+	actualStore := fsw.getActualStore(fp)
+	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "find").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "find").Observe(time.Since(start).Seconds())
+	}()
+
+	entry, err = actualStore.FindEntry(ctx, fp)
+	// glog.V(4).Infof("FindEntry %s: %v", fp, err)
+	if err != nil {
+		if fsw.CanDropWholeBucket() && strings.Contains(err.Error(), "Table") && strings.Contains(err.Error(), "doesn't exist") {
+			err = filer_pb.ErrNotFound
+		}
+		return nil, err
+	}
+
+	fsw.maybeReadHardLink(ctx, entry)
+
+	filer_pb.AfterEntryDeserialization(entry.GetChunks())
+	return
+}
+
+func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp util.FullPath) (err error) {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	ctx = context.WithoutCancel(ctx)
+	actualStore := fsw.getActualStore(fp)
+	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "delete").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "delete").Observe(time.Since(start).Seconds())
+	}()
+
+	existingEntry, findErr := fsw.FindEntry(ctx, fp)
+	if findErr == filer_pb.ErrNotFound || existingEntry == nil {
+		return nil
+	}
+	if len(existingEntry.HardLinkId) != 0 {
+		// remove hard link
+		op := ctx.Value("OP")
+		if op != "MV" {
+			glog.V(4).InfofCtx(ctx, "DeleteHardLink %s", existingEntry.FullPath)
+			if hlErr := fsw.DeleteHardLink(ctx, existingEntry.HardLinkId); hlErr != nil {
+				// Log but continue: the directory entry must be removed
+				// even if hard link counter cleanup fails, otherwise the
+				// parent directory cannot be removed (rmdir ENOTEMPTY).
+				glog.Warningf("DeleteHardLink %s (id %x): %v", existingEntry.FullPath, existingEntry.HardLinkId, hlErr)
+			}
+		}
+	}
+
+	return actualStore.DeleteEntry(ctx, fp)
+}
+
+func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry *Entry) (err error) {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	ctx = context.WithoutCancel(ctx)
+	actualStore := fsw.getActualStore(existingEntry.FullPath)
+	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "delete").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "delete").Observe(time.Since(start).Seconds())
+	}()
+
+	if len(existingEntry.HardLinkId) != 0 {
+		// remove hard link
+		op := ctx.Value("OP")
+		if op != "MV" {
+			glog.V(4).InfofCtx(ctx, "DeleteHardLink %s", existingEntry.FullPath)
+			if hlErr := fsw.DeleteHardLink(ctx, existingEntry.HardLinkId); hlErr != nil {
+				// Log the hard link cleanup error but continue to delete
+				// the directory entry. If we return early here, the entry
+				// remains in the store and the parent directory cannot be
+				// removed (rmdir returns ENOTEMPTY).
+				glog.Warningf("DeleteHardLink %s (id %x): %v", existingEntry.FullPath, existingEntry.HardLinkId, hlErr)
+			}
+		}
+	}
+
+	return actualStore.DeleteEntry(ctx, existingEntry.FullPath)
+}
+
+func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	ctx = context.WithoutCancel(ctx)
+	actualStore := fsw.getActualStore(fp + "/")
+	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "deleteFolderChildren").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "deleteFolderChildren").Observe(time.Since(start).Seconds())
+	}()
+
+	return actualStore.DeleteFolderChildren(ctx, fp)
+}
+
+func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) {
+	ctx = context.WithoutCancel(ctx)
+	actualStore := fsw.getActualStore(dirPath + "/")
+	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "list").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "list").Observe(time.Since(start).Seconds())
+	}()
+
+	// glog.V(4).Infof("ListDirectoryEntries %s from %s limit %d", dirPath, startFileName, limit)
+	return actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *Entry) (bool, error) {
+		fsw.maybeReadHardLink(ctx, entry)
+		filer_pb.AfterEntryDeserialization(entry.GetChunks())
+		return eachEntryFunc(entry)
+	})
+}
+
+func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
+	ctx = context.WithoutCancel(ctx)
+	actualStore := fsw.getActualStore(dirPath + "/")
+	stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "prefixList").Inc()
+	start := time.Now()
+	defer func() {
+		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "prefixList").Observe(time.Since(start).Seconds())
+	}()
+	if limit > math.MaxInt32-1 {
+		limit = math.MaxInt32 - 1
+	}
+	// glog.V(4).Infof("ListDirectoryPrefixedEntries %s from %s prefix %s limit %d", dirPath, startFileName, prefix, limit)
+	adjustedEntryFunc := func(entry *Entry) (bool, error) {
+		fsw.maybeReadHardLink(ctx, entry)
+		filer_pb.AfterEntryDeserialization(entry.GetChunks())
+		return eachEntryFunc(entry)
+	}
+	lastFileName, err = actualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, adjustedEntryFunc)
+	if err == ErrUnsupportedListDirectoryPrefixed {
+		lastFileName, err = fsw.prefixFilterEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, adjustedEntryFunc)
+	}
+	return lastFileName, err
+}
+
+func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (lastFileName string, err error) {
+	actualStore := fsw.getActualStore(dirPath + "/")
+
+	if prefix == "" {
+		return actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, eachEntryFunc)
+	}
+
+	var notPrefixed []*Entry
+	lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *Entry) (bool, error) {
+		notPrefixed = append(notPrefixed, entry)
+		return true, nil
+	})
+	if err != nil {
+		return
+	}
+
+	count := int64(0)
+	for count < limit && len(notPrefixed) > 0 {
+		for _, entry := range notPrefixed {
+			if strings.HasPrefix(entry.Name(), prefix) {
+				count++
+				res, resErr := eachEntryFunc(entry)
+
+				if resErr != nil {
+					err = fmt.Errorf("failed to process eachEntryFunc for entry %q: %w", entry.Name(), resErr)
+					return
+				}
+
+				if !res {
+					return
+				}
+				if count >= limit {
+					break
+				}
+			}
+		}
+		if count < limit && lastFileName < prefix {
+			notPrefixed = notPrefixed[:0]
+			lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) (bool, error) {
+				notPrefixed = append(notPrefixed, entry)
+				return true, nil
+			})
+			if err != nil {
+				return
+			}
+		} else {
+			break
+		}
+	}
+	return
+}
+
+func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	ctx = context.WithoutCancel(ctx)
+	return fsw.getDefaultStore().BeginTransaction(ctx)
+}
+
+func (fsw *FilerStoreWrapper) CommitTransaction(ctx context.Context) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	ctx = context.WithoutCancel(ctx)
+	return fsw.getDefaultStore().CommitTransaction(ctx)
+}
+
+func (fsw *FilerStoreWrapper) RollbackTransaction(ctx context.Context) error {
+	ctx = context.WithoutCancel(ctx)
+	return fsw.getDefaultStore().RollbackTransaction(ctx)
+}
+
+func (fsw *FilerStoreWrapper) Shutdown() {
+	fsw.getDefaultStore().Shutdown()
+}
+
+func (fsw *FilerStoreWrapper) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	ctx = context.WithoutCancel(ctx)
+	return fsw.getDefaultStore().KvPut(ctx, key, value)
+}
+func (fsw *FilerStoreWrapper) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+	ctx = context.WithoutCancel(ctx)
+	return fsw.getDefaultStore().KvGet(ctx, key)
+}
+func (fsw *FilerStoreWrapper) KvDelete(ctx context.Context, key []byte) (err error) {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	ctx = context.WithoutCancel(ctx)
+	return fsw.getDefaultStore().KvDelete(ctx, key)
+}
+
+func (fsw *FilerStoreWrapper) Debug(writer io.Writer) {
+	if debuggable, ok := fsw.getDefaultStore().(Debuggable); ok {
+		debuggable.Debug(writer)
+	}
+}

weed/filer/filerstore_wrapper_test.go

@@ -0,0 +1,207 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFilerStoreWrapperMimeNormalization(t *testing.T) {
+	tests := []struct {
+		name     string
+		mode     os.FileMode
+		wantMime string
+	}{
+		{
+			name:     "files strip octet-stream",
+			mode:     0o660,
+			wantMime: "",
+		},
+		{
+			name:     "directories keep octet-stream",
+			mode:     os.ModeDir | 0o770,
+			wantMime: "application/octet-stream",
+		},
+	}
+
+	operations := []struct {
+		name string
+		run  func(*FilerStoreWrapper, context.Context, *Entry) error
+	}{
+		{
+			name: "insert",
+			run: func(fsw *FilerStoreWrapper, ctx context.Context, entry *Entry) error {
+				return fsw.InsertEntry(ctx, entry)
+			},
+		},
+		{
+			name: "update",
+			run: func(fsw *FilerStoreWrapper, ctx context.Context, entry *Entry) error {
+				return fsw.UpdateEntry(ctx, entry)
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		for _, op := range operations {
+			t.Run(tt.name+"/"+op.name, func(t *testing.T) {
+				store := newStubFilerStore()
+				wrapper := NewFilerStoreWrapper(store)
+				entry := &Entry{
+					FullPath: util.FullPath("/buckets/test/object"),
+					Attr: Attr{
+						Mode: tt.mode,
+						Mime: "application/octet-stream",
+					},
+				}
+
+				err := op.run(wrapper, context.Background(), entry)
+				require.NoError(t, err)
+
+				storedEntry, findErr := store.FindEntry(context.Background(), entry.FullPath)
+				require.NoError(t, findErr)
+				assert.Equal(t, tt.wantMime, storedEntry.Mime)
+			})
+		}
+	}
+}
+
+// cancelledCtx returns a context that is already cancelled.
+func cancelledCtx() context.Context {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	return ctx
+}
+
+// expiredCtx returns a context whose deadline has already passed.
+func expiredCtx() context.Context {
+	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
+	cancel() // release resources immediately as it's already expired
+	return ctx
+}
+
+func TestFilerStoreWrapperWriteOpsRejectCancelledContext(t *testing.T) {
+	newEntry := func(path string) *Entry {
+		return &Entry{
+			FullPath: util.FullPath(path),
+			Attr:     Attr{Mode: 0o660, Mime: "application/octet-stream"},
+		}
+	}
+
+	// Each write operation that should be guarded.
+	writeOps := []struct {
+		name string
+		run  func(*FilerStoreWrapper, context.Context) error
+	}{
+		{"InsertEntry", func(fsw *FilerStoreWrapper, ctx context.Context) error {
+			return fsw.InsertEntry(ctx, newEntry("/test/a"))
+		}},
+		{"InsertEntryKnownAbsent", func(fsw *FilerStoreWrapper, ctx context.Context) error {
+			return fsw.InsertEntryKnownAbsent(ctx, newEntry("/test/b"))
+		}},
+		{"UpdateEntry", func(fsw *FilerStoreWrapper, ctx context.Context) error {
+			_ = fsw.InsertEntry(context.Background(), newEntry("/test/c"))
+			return fsw.UpdateEntry(ctx, newEntry("/test/c"))
+		}},
+		{"DeleteEntry", func(fsw *FilerStoreWrapper, ctx context.Context) error {
+			_ = fsw.InsertEntry(context.Background(), newEntry("/test/d"))
+			return fsw.DeleteEntry(ctx, "/test/d")
+		}},
+		{"DeleteOneEntry", func(fsw *FilerStoreWrapper, ctx context.Context) error {
+			e := newEntry("/test/e")
+			_ = fsw.InsertEntry(context.Background(), e)
+			return fsw.DeleteOneEntry(ctx, e)
+		}},
+		{"DeleteFolderChildren", func(fsw *FilerStoreWrapper, ctx context.Context) error {
+			_ = fsw.InsertEntry(context.Background(), newEntry("/test/folder/child"))
+			return fsw.DeleteFolderChildren(ctx, "/test/folder")
+		}},
+		{"BeginTransaction", func(fsw *FilerStoreWrapper, ctx context.Context) error {
+			_, err := fsw.BeginTransaction(ctx)
+			return err
+		}},
+		{"CommitTransaction", func(fsw *FilerStoreWrapper, ctx context.Context) error {
+			return fsw.CommitTransaction(ctx)
+		}},
+		{"KvPut", func(fsw *FilerStoreWrapper, ctx context.Context) error {
+			return fsw.KvPut(ctx, []byte("k"), []byte("v"))
+		}},
+		{"KvDelete", func(fsw *FilerStoreWrapper, ctx context.Context) error {
+			_ = fsw.KvPut(context.Background(), []byte("k"), []byte("v"))
+			return fsw.KvDelete(ctx, []byte("k"))
+		}},
+	}
+
+	badContexts := []struct {
+		name      string
+		ctx       context.Context
+		wantError error
+	}{
+		{"cancelled", cancelledCtx(), context.Canceled},
+		{"deadline exceeded", expiredCtx(), context.DeadlineExceeded},
+	}
+
+	for _, op := range writeOps {
+		for _, bc := range badContexts {
+			t.Run(op.name+"/"+bc.name, func(t *testing.T) {
+				wrapper := NewFilerStoreWrapper(newStubFilerStore())
+				err := op.run(wrapper, bc.ctx)
+				require.Error(t, err)
+				assert.True(t, errors.Is(err, bc.wantError), "got %v, want %v", err, bc.wantError)
+			})
+		}
+	}
+}
+
+func TestFilerStoreWrapperWriteOpsSucceedWithActiveContext(t *testing.T) {
+	wrapper := NewFilerStoreWrapper(newStubFilerStore())
+	ctx := context.Background()
+	entry := &Entry{
+		FullPath: util.FullPath("/test/obj"),
+		Attr:     Attr{Mode: 0o660},
+	}
+
+	require.NoError(t, wrapper.InsertEntry(ctx, entry))
+	require.NoError(t, wrapper.UpdateEntry(ctx, entry))
+	require.NoError(t, wrapper.DeleteOneEntry(ctx, entry))
+	require.NoError(t, wrapper.InsertEntryKnownAbsent(ctx, entry))
+	require.NoError(t, wrapper.DeleteEntry(ctx, entry.FullPath))
+	require.NoError(t, wrapper.KvPut(ctx, []byte("k"), []byte("v")))
+	require.NoError(t, wrapper.KvDelete(ctx, []byte("k")))
+
+	txCtx, err := wrapper.BeginTransaction(ctx)
+	require.NoError(t, err)
+	require.NoError(t, wrapper.CommitTransaction(txCtx))
+}
+
+func TestFilerStoreWrapperReadOpsSucceedWithCancelledContext(t *testing.T) {
+	wrapper := NewFilerStoreWrapper(newStubFilerStore())
+	entry := &Entry{
+		FullPath: util.FullPath("/test/readable"),
+		Attr:     Attr{Mode: 0o660},
+	}
+	require.NoError(t, wrapper.InsertEntry(context.Background(), entry))
+	require.NoError(t, wrapper.KvPut(context.Background(), []byte("rk"), []byte("rv")))
+
+	ctx := cancelledCtx()
+
+	_, err := wrapper.FindEntry(ctx, entry.FullPath)
+	assert.NoError(t, err)
+
+	_, err = wrapper.KvGet(ctx, []byte("rk"))
+	assert.NoError(t, err)
+}
+
+// RollbackTransaction must succeed even when the context is cancelled or
+// expired, because it is a cleanup operation called after failures.
+func TestFilerStoreWrapperRollbackSucceedsWithCancelledContext(t *testing.T) {
+	wrapper := NewFilerStoreWrapper(newStubFilerStore())
+	assert.NoError(t, wrapper.RollbackTransaction(cancelledCtx()))
+	assert.NoError(t, wrapper.RollbackTransaction(expiredCtx()))
+}

weed/filer/foundationdb/CONFIGURATION.md

@@ -0,0 +1,423 @@
+# FoundationDB Filer Store Configuration Reference
+
+This document provides comprehensive configuration options for the FoundationDB filer store.
+
+## Configuration Methods
+
+### 1. Configuration File (filer.toml)
+
+```toml
+[foundationdb]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+api_version = 740
+timeout = "5s"
+max_retry_delay = "1s"
+directory_prefix = "seaweedfs"
+```
+
+### 2. Environment Variables
+
+All configuration options can be set via environment variables with the `WEED_FOUNDATIONDB_` prefix:
+
+```bash
+export WEED_FOUNDATIONDB_ENABLED=true
+export WEED_FOUNDATIONDB_CLUSTER_FILE=/etc/foundationdb/fdb.cluster
+export WEED_FOUNDATIONDB_API_VERSION=740
+export WEED_FOUNDATIONDB_TIMEOUT=5s
+export WEED_FOUNDATIONDB_MAX_RETRY_DELAY=1s
+export WEED_FOUNDATIONDB_DIRECTORY_PREFIX=seaweedfs
+# Write batching (disabled by default)
+export WEED_FOUNDATIONDB_BATCH_ENABLED=false
+export WEED_FOUNDATIONDB_BATCH_SIZE=100
+export WEED_FOUNDATIONDB_BATCH_INTERVAL=1ms
+```
+
+### 3. Command Line Arguments
+
+While not directly supported, configuration can be specified via config files passed to the `weed` command.
+
+## Configuration Options
+
+### Basic Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `enabled` | boolean | `false` | Enable the FoundationDB filer store |
+| `cluster_file` | string | `/etc/foundationdb/fdb.cluster` | Path to FoundationDB cluster file |
+| `api_version` | integer | `740` | FoundationDB API version to use |
+
+### Connection Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `timeout` | duration | `5s` | Transaction timeout duration |
+| `max_retry_delay` | duration | `1s` | Maximum delay between retries |
+
+### Storage Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `directory_prefix` | string | `seaweedfs` | Directory prefix for key organization |
+
+### Write Batching Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `batch_enabled` | boolean | `false` | Enable write batching. Disabled by default for optimal S3 PUT latency. |
+| `batch_size` | integer | `100` | Maximum number of operations per batch (when batching is enabled) |
+| `batch_interval` | duration | `1ms` | Maximum time to wait before flushing a batch (when batching is enabled) |
+
+**Note:** Write batching is **disabled by default**. Each write commits immediately in its own 
+transaction, providing optimal latency for S3 PUT operations. Enable batching only for 
+high-throughput bulk ingestion workloads where you can tolerate slightly higher per-operation latency.
+
+## Configuration Examples
+
+### Development Environment
+
+```toml
+[foundationdb]
+enabled = true
+cluster_file = "/var/fdb/config/fdb.cluster"
+api_version = 740
+timeout = "10s"
+max_retry_delay = "2s"
+directory_prefix = "seaweedfs_dev"
+```
+
+### Production Environment
+
+```toml
+[foundationdb]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+api_version = 740
+timeout = "30s"
+max_retry_delay = "5s"  
+directory_prefix = "seaweedfs_prod"
+```
+
+### High-Performance Setup (Low Latency)
+
+For S3 workloads requiring low latency per operation:
+
+```toml
+[foundationdb]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+api_version = 740
+timeout = "5s"
+max_retry_delay = "1s"
+directory_prefix = "sw"  # Shorter prefix for efficiency
+# Batching disabled (default) for optimal per-operation latency
+batch_enabled = false
+```
+
+### High-Throughput Bulk Ingestion
+
+For bulk data loading where throughput matters more than per-operation latency:
+
+```toml
+[foundationdb]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+api_version = 740
+timeout = "60s"
+max_retry_delay = "10s"
+directory_prefix = "sw"
+# Enable batching for higher throughput
+batch_enabled = true
+batch_size = 100
+batch_interval = "1ms"
+```
+
+### Path-Specific Configuration
+
+Configure different FoundationDB settings for different paths:
+
+```toml
+# Default configuration
+[foundationdb]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+directory_prefix = "seaweedfs_main"
+
+# Backup path with different prefix
+[foundationdb.backup]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+directory_prefix = "seaweedfs_backup"
+location = "/backup"
+timeout = "120s"
+
+# Archive path with extended timeouts
+[foundationdb.archive]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+directory_prefix = "seaweedfs_archive"
+location = "/archive"
+timeout = "300s"
+max_retry_delay = "30s"
+```
+
+## Configuration Validation
+
+### Required Settings
+
+The following settings are required for FoundationDB to function:
+
+1. `enabled = true`
+2. `cluster_file` must point to a valid FoundationDB cluster file
+3. `api_version` must match your FoundationDB installation
+
+### Validation Rules
+
+- `api_version` must be between 600 and 740
+- `timeout` must be a valid duration string (e.g., "5s", "30s", "2m")
+- `max_retry_delay` must be a valid duration string
+- `cluster_file` must exist and be readable
+- `directory_prefix` must not be empty
+
+### Error Handling
+
+Invalid configurations will result in startup errors:
+
+```
+FATAL: Failed to initialize store for foundationdb: invalid timeout duration
+FATAL: Failed to initialize store for foundationdb: failed to open FoundationDB database
+FATAL: Failed to initialize store for foundationdb: cluster file not found
+```
+
+## Performance Tuning
+
+### Timeout Configuration
+
+| Use Case | Timeout | Max Retry Delay | Notes |
+|----------|---------|-----------------|-------|
+| Interactive workloads | 5s | 1s | Fast response times |
+| Batch processing | 60s | 10s | Handle large operations |
+| Archive operations | 300s | 30s | Very large data sets |
+
+### Connection Pool Settings
+
+FoundationDB automatically manages connection pooling. No additional configuration needed.
+
+### Directory Organization
+
+Use meaningful directory prefixes to organize data:
+
+```toml
+# Separate environments
+directory_prefix = "prod_seaweedfs"      # Production
+directory_prefix = "staging_seaweedfs"   # Staging
+directory_prefix = "dev_seaweedfs"       # Development
+
+# Separate applications
+directory_prefix = "app1_seaweedfs"      # Application 1
+directory_prefix = "app2_seaweedfs"      # Application 2
+```
+
+## Security Configuration
+
+### Cluster File Security
+
+Protect the FoundationDB cluster file:
+
+```bash
+# Set proper permissions
+sudo chown root:seaweedfs /etc/foundationdb/fdb.cluster
+sudo chmod 640 /etc/foundationdb/fdb.cluster
+```
+
+### Network Security
+
+FoundationDB supports TLS encryption. Configure in the cluster file:
+
+```
+description:cluster_id@tls(server1:4500,server2:4500,server3:4500)
+```
+
+### Access Control
+
+Use FoundationDB's built-in access control mechanisms when available.
+
+## Monitoring Configuration
+
+### Health Check Settings
+
+Configure health check timeouts appropriately:
+
+```toml
+[foundationdb]
+enabled = true
+timeout = "10s"  # Reasonable timeout for health checks
+```
+
+### Logging Configuration
+
+Enable verbose logging for troubleshooting:
+
+```bash
+# Start SeaweedFS with debug logs
+WEED_FOUNDATIONDB_ENABLED=true weed -v=2 server -filer
+```
+
+## Migration Configuration
+
+### From Other Filer Stores
+
+When migrating from other filer stores:
+
+1. Configure both stores temporarily
+2. Use path-specific configuration for gradual migration
+3. Migrate data using SeaweedFS tools
+
+```toml
+# During migration - keep old store for reads
+[leveldb2]
+enabled = true
+dir = "/old/filer/data"
+
+# New writes go to FoundationDB
+[foundationdb.migration]
+enabled = true
+location = "/new"
+cluster_file = "/etc/foundationdb/fdb.cluster"
+```
+
+## Backup Configuration
+
+### Metadata Backup Strategy
+
+```toml
+# Main storage
+[foundationdb]
+enabled = true
+directory_prefix = "seaweedfs_main"
+
+# Backup storage (different cluster recommended)
+[foundationdb.backup]
+enabled = true
+cluster_file = "/etc/foundationdb/backup_fdb.cluster"
+directory_prefix = "seaweedfs_backup"
+location = "/backup"
+```
+
+## Container Configuration
+
+### Docker Environment Variables
+
+```bash
+# Docker environment
+WEED_FOUNDATIONDB_ENABLED=true
+WEED_FOUNDATIONDB_CLUSTER_FILE=/var/fdb/config/fdb.cluster
+WEED_FOUNDATIONDB_API_VERSION=740
+```
+
+### Kubernetes ConfigMap
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: seaweedfs-config
+data:
+  filer.toml: |
+    [foundationdb]
+    enabled = true
+    cluster_file = "/var/fdb/config/cluster_file"
+    api_version = 740
+    timeout = "30s"
+    max_retry_delay = "5s"
+    directory_prefix = "k8s_seaweedfs"
+```
+
+## Troubleshooting Configuration
+
+### Debug Configuration
+
+```toml
+[foundationdb]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+timeout = "60s"        # Longer timeouts for debugging
+max_retry_delay = "10s"
+directory_prefix = "debug_seaweedfs"
+```
+
+### Test Configuration
+
+```toml
+[foundationdb]
+enabled = true
+cluster_file = "/tmp/fdb.cluster"  # Test cluster
+timeout = "5s"
+directory_prefix = "test_seaweedfs"
+```
+
+## Configuration Best Practices
+
+### 1. Environment Separation
+
+Use different directory prefixes for different environments:
+- Production: `prod_seaweedfs`
+- Staging: `staging_seaweedfs` 
+- Development: `dev_seaweedfs`
+
+### 2. Timeout Settings
+
+- Interactive: 5-10 seconds
+- Batch: 30-60 seconds
+- Archive: 120-300 seconds
+
+### 3. Cluster File Management
+
+- Use absolute paths for cluster files
+- Ensure proper file permissions
+- Keep backup copies of cluster files
+
+### 4. Directory Naming
+
+- Use descriptive prefixes
+- Include environment/application identifiers
+- Keep prefixes reasonably short for efficiency
+
+### 5. Error Handling
+
+- Configure appropriate timeouts
+- Monitor retry patterns
+- Set up alerting for configuration errors
+
+## Configuration Testing
+
+### Validation Script
+
+```bash
+#!/bin/bash
+# Test FoundationDB configuration
+
+# Check cluster file
+if [ ! -f "$WEED_FOUNDATIONDB_CLUSTER_FILE" ]; then
+    echo "ERROR: Cluster file not found: $WEED_FOUNDATIONDB_CLUSTER_FILE"
+    exit 1
+fi
+
+# Test connection
+fdbcli -C "$WEED_FOUNDATIONDB_CLUSTER_FILE" --exec 'status' > /dev/null
+if [ $? -ne 0 ]; then
+    echo "ERROR: Cannot connect to FoundationDB cluster"
+    exit 1
+fi
+
+echo "Configuration validation passed"
+```
+
+### Integration Testing
+
+```bash
+# Test configuration with SeaweedFS
+cd test/foundationdb
+make check-env
+make test-unit
+```

weed/filer/foundationdb/INSTALL.md

@@ -0,0 +1,435 @@
+# FoundationDB Filer Store Installation Guide
+
+This guide covers the installation and setup of the FoundationDB filer store for SeaweedFS.
+
+## Prerequisites
+
+### FoundationDB Server
+
+1. **Install FoundationDB Server**
+   
+   **Ubuntu/Debian:**
+   ```bash
+   # Add FoundationDB repository
+   curl -L https://github.com/apple/foundationdb/releases/download/7.4.5/foundationdb-clients_7.4.5-1_amd64.deb -o foundationdb-clients.deb
+   curl -L https://github.com/apple/foundationdb/releases/download/7.4.5/foundationdb-server_7.4.5-1_amd64.deb -o foundationdb-server.deb
+   
+   sudo dpkg -i foundationdb-clients.deb foundationdb-server.deb
+   ```
+   
+   **CentOS/RHEL:**
+   ```bash
+   # Install RPM packages
+   wget https://github.com/apple/foundationdb/releases/download/7.4.5/foundationdb-clients-7.4.5-1.el7.x86_64.rpm
+   wget https://github.com/apple/foundationdb/releases/download/7.4.5/foundationdb-server-7.4.5-1.el7.x86_64.rpm
+   
+   sudo rpm -Uvh foundationdb-clients-7.4.5-1.el7.x86_64.rpm foundationdb-server-7.4.5-1.el7.x86_64.rpm
+   ```
+   
+   **macOS:**
+   ```bash
+   # Using Homebrew (if available)
+   brew install foundationdb
+   
+   # Or download from GitHub releases
+   # https://github.com/apple/foundationdb/releases
+   ```
+
+2. **Initialize FoundationDB Cluster**
+   
+   **Single Node (Development):**
+   ```bash
+   # Start FoundationDB service
+   sudo systemctl start foundationdb
+   sudo systemctl enable foundationdb
+   
+   # Initialize database
+   fdbcli --exec 'configure new single ssd'
+   ```
+   
+   **Multi-Node Cluster (Production):**
+   ```bash
+   # On each node, edit /etc/foundationdb/fdb.cluster
+   # Example: testing:testing@node1:4500,node2:4500,node3:4500
+   
+   # On one node, initialize cluster
+   fdbcli --exec 'configure new double ssd'
+   ```
+
+3. **Verify Installation**
+   ```bash
+   fdbcli --exec 'status'
+   ```
+
+### FoundationDB Client Libraries
+
+The SeaweedFS FoundationDB integration requires the FoundationDB client libraries.
+
+**Ubuntu/Debian:**
+```bash
+sudo apt-get install libfdb-dev
+```
+
+**CentOS/RHEL:**
+```bash
+sudo yum install foundationdb-devel
+```
+
+**macOS:**
+```bash
+# Client libraries are included with the server installation
+export LIBRARY_PATH=/usr/local/lib
+export CPATH=/usr/local/include
+```
+
+## Building SeaweedFS with FoundationDB Support
+
+### Download FoundationDB Go Bindings
+
+```bash
+go mod init seaweedfs-foundationdb
+go get github.com/apple/foundationdb/bindings/go/src/fdb
+```
+
+### Build SeaweedFS
+
+```bash
+# Clone SeaweedFS repository
+git clone https://github.com/seaweedfs/seaweedfs.git
+cd seaweedfs
+
+# Build with FoundationDB support
+go build -tags foundationdb -o weed
+```
+
+### Verify Build
+
+```bash
+./weed version
+# Should show version information
+
+./weed help
+# Should list available commands
+```
+
+## Configuration
+
+### Basic Configuration
+
+Create or edit `filer.toml`:
+
+```toml
+[foundationdb]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+api_version = 740
+timeout = "5s"
+max_retry_delay = "1s"
+directory_prefix = "seaweedfs"
+```
+
+### Environment Variables
+
+Alternative configuration via environment variables:
+
+```bash
+export WEED_FOUNDATIONDB_ENABLED=true
+export WEED_FOUNDATIONDB_CLUSTER_FILE=/etc/foundationdb/fdb.cluster
+export WEED_FOUNDATIONDB_API_VERSION=740
+export WEED_FOUNDATIONDB_TIMEOUT=5s
+export WEED_FOUNDATIONDB_MAX_RETRY_DELAY=1s
+export WEED_FOUNDATIONDB_DIRECTORY_PREFIX=seaweedfs
+```
+
+### Advanced Configuration
+
+For production deployments:
+
+```toml
+[foundationdb]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+api_version = 740
+timeout = "30s"
+max_retry_delay = "5s"
+directory_prefix = "seaweedfs_prod"
+
+# Path-specific configuration for backups
+[foundationdb.backup]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+directory_prefix = "seaweedfs_backup"
+location = "/backup"
+timeout = "60s"
+```
+
+## Deployment
+
+### Single Node Deployment
+
+```bash
+# Start SeaweedFS with FoundationDB filer
+./weed server -filer \
+  -master.port=9333 \
+  -volume.port=8080 \
+  -filer.port=8888 \
+  -s3.port=8333
+```
+
+### Distributed Deployment
+
+**Master Servers:**
+```bash
+# Node 1
+./weed master -port=9333 -peers=master1:9333,master2:9333,master3:9333
+
+# Node 2  
+./weed master -port=9333 -peers=master1:9333,master2:9333,master3:9333 -ip=master2
+
+# Node 3
+./weed master -port=9333 -peers=master1:9333,master2:9333,master3:9333 -ip=master3
+```
+
+**Filer Servers with FoundationDB:**
+```bash
+# Filer nodes
+./weed filer -master=master1:9333,master2:9333,master3:9333 -port=8888
+```
+
+**Volume Servers:**
+```bash
+./weed volume -master=master1:9333,master2:9333,master3:9333 -port=8080
+```
+
+### Docker Deployment
+
+**docker-compose.yml:**
+```yaml
+version: '3.9'
+services:
+  foundationdb:
+    image: foundationdb/foundationdb:7.4.5
+    ports:
+      - "4500:4500"
+    volumes:
+      - fdb_data:/var/fdb/data
+      - fdb_config:/var/fdb/config
+
+  seaweedfs:
+    image: chrislusf/seaweedfs:latest
+    command: "server -filer -ip=seaweedfs"
+    ports:
+      - "9333:9333"
+      - "8888:8888"
+      - "8333:8333"
+    environment:
+      WEED_FOUNDATIONDB_ENABLED: "true"
+      WEED_FOUNDATIONDB_CLUSTER_FILE: "/var/fdb/config/fdb.cluster"
+    volumes:
+      - fdb_config:/var/fdb/config
+    depends_on:
+      - foundationdb
+
+volumes:
+  fdb_data:
+  fdb_config:
+```
+
+### Kubernetes Deployment
+
+**FoundationDB Operator:**
+```bash
+# Install FoundationDB operator
+kubectl apply -f https://raw.githubusercontent.com/FoundationDB/fdb-kubernetes-operator/main/config/samples/deployment.yaml
+```
+
+**SeaweedFS with FoundationDB:**
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: seaweedfs-filer
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: seaweedfs-filer
+  template:
+    metadata:
+      labels:
+        app: seaweedfs-filer
+    spec:
+      containers:
+      - name: seaweedfs
+        image: chrislusf/seaweedfs:latest
+        command: ["weed", "filer"]
+        env:
+        - name: WEED_FOUNDATIONDB_ENABLED
+          value: "true"
+        - name: WEED_FOUNDATIONDB_CLUSTER_FILE
+          value: "/var/fdb/config/cluster_file"
+        ports:
+        - containerPort: 8888
+        volumeMounts:
+        - name: fdb-config
+          mountPath: /var/fdb/config
+      volumes:
+      - name: fdb-config
+        configMap:
+          name: fdb-cluster-config
+```
+
+## Testing Installation
+
+### Quick Test
+
+```bash
+# Start SeaweedFS with FoundationDB
+./weed server -filer &
+
+# Test file operations
+echo "Hello FoundationDB" > test.txt
+curl -F file=@test.txt "http://localhost:8888/test/"
+curl "http://localhost:8888/test/test.txt"
+
+# Test S3 API
+curl -X PUT "http://localhost:8333/testbucket"
+curl -T test.txt "http://localhost:8333/testbucket/test.txt"
+```
+
+### Integration Test Suite
+
+```bash
+# Run the provided test suite
+cd test/foundationdb
+make setup
+make test
+```
+
+## Performance Tuning
+
+### FoundationDB Tuning
+
+```bash
+# Configure for high performance
+fdbcli --exec 'configure triple ssd'
+fdbcli --exec 'configure storage_engine=ssd-redwood-1-experimental'
+```
+
+### SeaweedFS Configuration
+
+```toml
+[foundationdb]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+timeout = "10s"           # Longer timeout for large operations
+max_retry_delay = "2s"    # Adjust retry behavior
+directory_prefix = "sw"   # Shorter prefix for efficiency
+```
+
+### OS-Level Tuning
+
+```bash
+# Increase file descriptor limits
+echo "* soft nofile 65536" >> /etc/security/limits.conf
+echo "* hard nofile 65536" >> /etc/security/limits.conf
+
+# Adjust network parameters
+echo "net.core.rmem_max = 134217728" >> /etc/sysctl.conf
+echo "net.core.wmem_max = 134217728" >> /etc/sysctl.conf
+sysctl -p
+```
+
+## Monitoring and Maintenance
+
+### Health Checks
+
+```bash
+# FoundationDB cluster health
+fdbcli --exec 'status'
+fdbcli --exec 'status details'
+
+# SeaweedFS health
+curl http://localhost:9333/cluster/status
+curl http://localhost:8888/statistics/health
+```
+
+### Log Monitoring
+
+**FoundationDB Logs:**
+- `/var/log/foundationdb/` (default location)
+- Monitor for errors, warnings, and performance issues
+
+**SeaweedFS Logs:**
+```bash
+# Start with verbose logging
+./weed -v=2 server -filer
+```
+
+### Backup and Recovery
+
+**FoundationDB Backup:**
+```bash
+# Start backup
+fdbbackup start -d file:///path/to/backup -t backup_tag
+
+# Monitor backup
+fdbbackup status -t backup_tag
+
+# Restore from backup
+fdbrestore start -r file:///path/to/backup -t backup_tag --wait
+```
+
+**SeaweedFS Metadata Backup:**
+```bash
+# Export filer metadata
+./weed shell
+> fs.meta.save /path/to/metadata/backup.gz
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Connection Refused**
+   - Check FoundationDB service status: `sudo systemctl status foundationdb`
+   - Verify cluster file: `cat /etc/foundationdb/fdb.cluster`
+   - Check network connectivity: `telnet localhost 4500`
+
+2. **API Version Mismatch**
+   - Update API version in configuration
+   - Rebuild SeaweedFS with matching FDB client library
+
+3. **Transaction Conflicts**
+   - Reduce transaction scope
+   - Implement appropriate retry logic
+   - Check for concurrent access patterns
+
+4. **Performance Issues**
+   - Monitor cluster status: `fdbcli --exec 'status details'`
+   - Check data distribution: `fdbcli --exec 'status json'`
+   - Verify storage configuration
+
+### Debug Mode
+
+```bash
+# Enable FoundationDB client tracing
+export FDB_TRACE_ENABLE=1
+export FDB_TRACE_PATH=/tmp/fdb_trace
+
+# Start SeaweedFS with debug logging
+./weed -v=3 server -filer
+```
+
+### Getting Help
+
+1. **FoundationDB Documentation**: https://apple.github.io/foundationdb/
+2. **SeaweedFS Community**: https://github.com/seaweedfs/seaweedfs/discussions
+3. **Issue Reporting**: https://github.com/seaweedfs/seaweedfs/issues
+
+For specific FoundationDB filer store issues, include:
+- FoundationDB version and cluster configuration
+- SeaweedFS version and build tags
+- Configuration files (filer.toml)
+- Error messages and logs
+- Steps to reproduce the issue

weed/filer/foundationdb/README.md

@@ -0,0 +1,250 @@
+# FoundationDB Filer Store
+
+This package provides a FoundationDB-based filer store for SeaweedFS, offering ACID transactions and horizontal scalability.
+
+## Features
+
+- **ACID Transactions**: Strong consistency guarantees with full ACID properties
+- **Horizontal Scalability**: Automatic data distribution across multiple nodes
+- **High Availability**: Built-in fault tolerance and automatic failover
+- **Efficient Directory Operations**: Optimized for large directory listings
+- **Key-Value Support**: Full KV operations for metadata storage
+- **Compression**: Automatic compression for large entry chunks
+
+## Installation
+
+### Prerequisites
+
+1. **FoundationDB Server**: Install and configure a FoundationDB cluster
+2. **FoundationDB Client Libraries**: Install libfdb_c client libraries
+3. **Go Build Tags**: Use the `foundationdb` build tag when compiling
+
+### Building SeaweedFS with FoundationDB Support
+
+```bash
+go build -tags foundationdb -o weed
+```
+
+## Configuration
+
+### Basic Configuration
+
+Add the following to your `filer.toml`:
+
+```toml
+[foundationdb]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+api_version = 740
+timeout = "5s"
+max_retry_delay = "1s"
+directory_prefix = "seaweedfs"
+```
+
+### Configuration Options
+
+| Option | Description | Default | Required |
+|--------|-------------|---------|----------|
+| `enabled` | Enable FoundationDB filer store | `false` | Yes |
+| `cluster_file` | Path to FDB cluster file | `/etc/foundationdb/fdb.cluster` | Yes |
+| `api_version` | FoundationDB API version | `740` | No |
+| `timeout` | Operation timeout duration | `5s` | No |
+| `max_retry_delay` | Maximum retry delay | `1s` | No |
+| `directory_prefix` | Directory prefix for organization | `seaweedfs` | No |
+| `batch_enabled` | Enable write batching (see Performance section) | `false` | No |
+| `batch_size` | Max operations per batch | `100` | No |
+| `batch_interval` | Max time before batch flush | `1ms` | No |
+
+### Path-Specific Configuration
+
+For path-specific filer stores:
+
+```toml
+[foundationdb.backup]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+directory_prefix = "seaweedfs_backup"
+location = "/backup"
+```
+
+## Environment Variables
+
+Configure via environment variables:
+
+```bash
+export WEED_FOUNDATIONDB_ENABLED=true
+export WEED_FOUNDATIONDB_CLUSTER_FILE=/etc/foundationdb/fdb.cluster
+export WEED_FOUNDATIONDB_API_VERSION=740
+export WEED_FOUNDATIONDB_TIMEOUT=5s
+export WEED_FOUNDATIONDB_MAX_RETRY_DELAY=1s
+export WEED_FOUNDATIONDB_DIRECTORY_PREFIX=seaweedfs
+```
+
+## FoundationDB Cluster Setup
+
+### Single Node (Development)
+
+```bash
+# Start FoundationDB server
+foundationdb start
+
+# Initialize database
+fdbcli --exec 'configure new single ssd'
+```
+
+### Multi-Node Cluster (Production)
+
+1. **Install FoundationDB** on all nodes
+2. **Configure cluster file** (`/etc/foundationdb/fdb.cluster`)
+3. **Initialize cluster**:
+   ```bash
+   fdbcli --exec 'configure new double ssd'
+   ```
+
+### Docker Setup
+
+Use the provided docker-compose.yml in `test/foundationdb/`:
+
+```bash
+cd test/foundationdb
+make setup
+```
+
+## Performance Considerations
+
+### Write Batching Configuration
+
+By default, write batching is **disabled** (`batch_enabled = false`). Each write commits 
+immediately in its own transaction. This provides optimal latency for S3 PUT operations.
+
+**When to enable batching:**
+- High-throughput bulk ingestion workloads
+- Scenarios where you can tolerate slightly higher per-operation latency
+- Workloads with many concurrent small writes
+
+**Batching configuration options:**
+
+```toml
+[foundationdb]
+# Enable write batching (disabled by default for optimal S3 latency)
+batch_enabled = true
+# Maximum operations per batch
+batch_size = 100
+# Maximum time to wait before flushing a batch
+batch_interval = "1ms"
+```
+
+**Performance comparison:**
+- **Batching disabled**: Each S3 PUT commits immediately (~1-5ms per op depending on FDB latency)
+- **Batching enabled**: Operations are grouped, reducing total commits but adding batch interval latency
+
+### Optimal Configuration
+
+- **API Version**: Use the latest stable API version (720+)
+- **Directory Structure**: Use logical directory prefixes to isolate different SeaweedFS instances
+- **Transaction Size**: Keep transactions under 10MB (FDB limit)
+- **Concurrency**: Use multiple client connections for parallel operations
+
+### Monitoring
+
+Monitor FoundationDB cluster status:
+
+```bash
+fdbcli --exec 'status'
+fdbcli --exec 'status details'
+```
+
+### Scaling
+
+FoundationDB automatically handles:
+- Data distribution across nodes
+- Load balancing
+- Automatic failover
+- Storage node addition/removal
+
+## Testing
+
+### Unit Tests
+
+```bash
+cd weed/filer/foundationdb
+go test -tags foundationdb -v
+```
+
+### Integration Tests
+
+```bash
+cd test/foundationdb
+make test
+```
+
+### End-to-End Tests
+
+```bash
+cd test/foundationdb
+make test-e2e
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Connection Failures**:
+   - Verify cluster file path
+   - Check FoundationDB server status
+   - Validate network connectivity
+
+2. **Transaction Conflicts**:
+   - Reduce transaction scope
+   - Implement retry logic
+   - Check for concurrent operations
+
+3. **Performance Issues**:
+   - Monitor cluster health
+   - Check data distribution
+   - Optimize directory structure
+
+### Debug Information
+
+Enable verbose logging:
+
+```bash
+weed -v=2 server -filer
+```
+
+Check FoundationDB status:
+
+```bash
+fdbcli --exec 'status details'
+```
+
+## Security
+
+### Network Security
+
+- Configure TLS for FoundationDB connections
+- Use firewall rules to restrict access
+- Monitor connection attempts
+
+### Data Encryption
+
+- Enable encryption at rest in FoundationDB
+- Use encrypted connections
+- Implement proper key management
+
+## Limitations
+
+- Maximum transaction size: 10MB
+- Single transaction timeout: configurable (default 5s)
+- API version compatibility required
+- Requires FoundationDB cluster setup
+
+## Support
+
+For issues specific to the FoundationDB filer store:
+1. Check FoundationDB cluster status
+2. Verify configuration settings
+3. Review SeaweedFS logs with verbose output
+4. Test with minimal reproduction case
+
+For FoundationDB-specific issues, consult the [FoundationDB documentation](https://apple.github.io/foundationdb/).

weed/filer/foundationdb/doc.go

@@ -0,0 +1,13 @@
+/*
+Package foundationdb provides a FoundationDB-based filer store for SeaweedFS.
+
+FoundationDB is a distributed ACID database with strong consistency guarantees
+and excellent scalability characteristics. This filer store leverages FDB's
+directory layer for organizing file metadata and its key-value interface for
+efficient storage and retrieval.
+
+The referenced "github.com/apple/foundationdb/bindings/go/src/fdb" library
+requires FoundationDB client libraries to be installed.
+So this is only compiled with "go build -tags foundationdb".
+*/
+package foundationdb

weed/filer/foundationdb/foundationdb_store.go

@@ -0,0 +1,808 @@
+//go:build foundationdb
+
+// Package foundationdb provides a filer store implementation using FoundationDB as the backend.
+//
+// IMPORTANT DESIGN NOTE - DeleteFolderChildren and Transaction Limits:
+//
+// FoundationDB imposes strict transaction limits:
+//   - Maximum transaction size: 10MB
+//   - Maximum transaction duration: 5 seconds
+//
+// The DeleteFolderChildren operation always uses batched deletion with multiple small transactions
+// to safely handle directories of any size. Even if called within an existing transaction context,
+// it will create its own batch transactions to avoid exceeding FDB limits.
+//
+// This means DeleteFolderChildren is NOT atomic with respect to an outer transaction - it manages
+// its own transaction boundaries for safety and reliability.
+
+package foundationdb
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/directory"
+	"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const (
+	// Safe limit for batch size (leave margin for FDB overhead)
+	FDB_BATCH_SIZE_LIMIT = 8 * 1024 * 1024
+	// Maximum number of entries to return in a single directory listing
+	// Large batches can cause transaction timeouts and increase memory pressure
+	MAX_DIRECTORY_LIST_LIMIT = 1000
+
+	// Write batching defaults
+	// Note: Batching is disabled by default because S3 semantics require waiting
+	// for durability, and the batch timer adds latency to each operation.
+	// Enable batching only for workloads that can tolerate potential latency.
+	DEFAULT_BATCH_SIZE     = 100
+	DEFAULT_BATCH_INTERVAL = 1 * time.Millisecond
+	DEFAULT_BATCH_ENABLED  = false
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &FoundationDBStore{})
+}
+
+// writeOp represents a pending write operation
+type writeOp struct {
+	key   fdb.Key
+	value []byte // nil for delete
+	done  chan error
+}
+
+// opSize returns the approximate size of an operation in bytes
+func (op *writeOp) size() int {
+	return len(op.key) + len(op.value)
+}
+
+// writeBatcher batches multiple writes into single transactions
+type writeBatcher struct {
+	store    *FoundationDBStore
+	ops      chan *writeOp
+	stop     chan struct{}
+	wg       sync.WaitGroup
+	size     int
+	interval time.Duration
+}
+
+func newWriteBatcher(store *FoundationDBStore, size int, interval time.Duration) *writeBatcher {
+	b := &writeBatcher{
+		store:    store,
+		ops:      make(chan *writeOp, size*10),
+		stop:     make(chan struct{}),
+		size:     size,
+		interval: interval,
+	}
+	b.wg.Add(1)
+	go b.run()
+	return b
+}
+
+func (b *writeBatcher) run() {
+	defer b.wg.Done()
+	batch := make([]*writeOp, 0, b.size)
+	batchBytes := 0 // Track cumulative size of batch
+	timer := time.NewTimer(b.interval)
+	defer timer.Stop()
+
+	flush := func() {
+		if len(batch) == 0 {
+			return
+		}
+		_, err := b.store.database.Transact(func(tr fdb.Transaction) (interface{}, error) {
+			for _, op := range batch {
+				if op.value != nil {
+					tr.Set(op.key, op.value)
+				} else {
+					tr.Clear(op.key)
+				}
+			}
+			return nil, nil
+		})
+		for _, op := range batch {
+			if op.done != nil {
+				op.done <- err
+				close(op.done)
+			}
+		}
+		batch = batch[:0]
+		batchBytes = 0
+	}
+
+	resetTimer := func() {
+		if !timer.Stop() {
+			select {
+			case <-timer.C:
+			default:
+			}
+		}
+		timer.Reset(b.interval)
+	}
+
+	// Collect available ops without blocking
+	collectAvailable := func() {
+		for {
+			select {
+			case op := <-b.ops:
+				batch = append(batch, op)
+				batchBytes += op.size()
+				if len(batch) >= b.size || batchBytes >= FDB_BATCH_SIZE_LIMIT {
+					return
+				}
+			default:
+				return
+			}
+		}
+	}
+
+	for {
+		select {
+		case op := <-b.ops:
+			batch = append(batch, op)
+			batchBytes += op.size()
+
+			// Optimization: When an operation arrives, try to collect more
+			// available operations without blocking. This improves throughput
+			// when multiple goroutines are submitting concurrently.
+			collectAvailable()
+
+			// Flush when batch count or size limit is reached
+			if len(batch) >= b.size || batchBytes >= FDB_BATCH_SIZE_LIMIT {
+				flush()
+				resetTimer()
+			}
+		case <-timer.C:
+			flush()
+			// Timer already fired, safe to reset directly
+			timer.Reset(b.interval)
+		case <-b.stop:
+			for {
+				select {
+				case op := <-b.ops:
+					batch = append(batch, op)
+				default:
+					flush()
+					return
+				}
+			}
+		}
+	}
+}
+
+func (b *writeBatcher) submit(key fdb.Key, value []byte, wait bool) error {
+	op := &writeOp{key: key, value: value}
+	if wait {
+		op.done = make(chan error, 1)
+	}
+	select {
+	case b.ops <- op:
+		if wait {
+			return <-op.done
+		}
+		return nil
+	case <-b.stop:
+		return fmt.Errorf("batcher stopped")
+	}
+}
+
+func (b *writeBatcher) shutdown() {
+	close(b.stop)
+	b.wg.Wait()
+}
+
+type FoundationDBStore struct {
+	database        fdb.Database
+	seaweedfsDir    directory.DirectorySubspace
+	kvDir           directory.DirectorySubspace
+	directoryPrefix string
+	timeout         time.Duration
+	maxRetryDelay   time.Duration
+	// Write batching - disabled by default for optimal S3 latency
+	// Enable for high-throughput bulk ingestion workloads
+	batcher       *writeBatcher
+	batchEnabled  bool
+	batchSize     int
+	batchInterval time.Duration
+}
+
+// Context key type for storing transactions
+type contextKey string
+
+const transactionKey contextKey = "fdb_transaction"
+
+// Helper functions for context-scoped transactions
+func (store *FoundationDBStore) getTransactionFromContext(ctx context.Context) (fdb.Transaction, bool) {
+	val := ctx.Value(transactionKey)
+	if val == nil {
+		var emptyTx fdb.Transaction
+		return emptyTx, false
+	}
+	if tx, ok := val.(fdb.Transaction); ok {
+		return tx, true
+	}
+	var emptyTx fdb.Transaction
+	return emptyTx, false
+}
+
+func (store *FoundationDBStore) setTransactionInContext(ctx context.Context, tx fdb.Transaction) context.Context {
+	return context.WithValue(ctx, transactionKey, tx)
+}
+
+func (store *FoundationDBStore) GetName() string {
+	return "foundationdb"
+}
+
+func (store *FoundationDBStore) Initialize(configuration util.Configuration, prefix string) error {
+	// Set default configuration values
+	configuration.SetDefault(prefix+"cluster_file", "/etc/foundationdb/fdb.cluster")
+	configuration.SetDefault(prefix+"api_version", 740)
+	configuration.SetDefault(prefix+"timeout", "5s")
+	configuration.SetDefault(prefix+"max_retry_delay", "1s")
+	configuration.SetDefault(prefix+"directory_prefix", "seaweedfs")
+	// Batching is disabled by default - each write commits immediately.
+	// This provides optimal latency for S3 PUT operations.
+	// Enable batching for high-throughput bulk ingestion workloads.
+	configuration.SetDefault(prefix+"batch_enabled", DEFAULT_BATCH_ENABLED)
+	configuration.SetDefault(prefix+"batch_size", DEFAULT_BATCH_SIZE)
+	configuration.SetDefault(prefix+"batch_interval", DEFAULT_BATCH_INTERVAL.String())
+
+	clusterFile := configuration.GetString(prefix + "cluster_file")
+	apiVersion := configuration.GetInt(prefix + "api_version")
+	timeoutStr := configuration.GetString(prefix + "timeout")
+	maxRetryDelayStr := configuration.GetString(prefix + "max_retry_delay")
+	store.directoryPrefix = configuration.GetString(prefix + "directory_prefix")
+
+	// Parse timeout values
+	var err error
+	store.timeout, err = time.ParseDuration(timeoutStr)
+	if err != nil {
+		return fmt.Errorf("invalid timeout duration %s: %w", timeoutStr, err)
+	}
+
+	store.maxRetryDelay, err = time.ParseDuration(maxRetryDelayStr)
+	if err != nil {
+		return fmt.Errorf("invalid max_retry_delay duration %s: %w", maxRetryDelayStr, err)
+	}
+
+	// Parse batch configuration
+	store.batchEnabled = configuration.GetBool(prefix + "batch_enabled")
+	store.batchSize = configuration.GetInt(prefix + "batch_size")
+	if store.batchSize <= 0 {
+		store.batchSize = DEFAULT_BATCH_SIZE
+	}
+	batchIntervalStr := configuration.GetString(prefix + "batch_interval")
+	store.batchInterval, err = time.ParseDuration(batchIntervalStr)
+	if err != nil {
+		glog.Warningf("invalid %sbatch_interval duration %q, using default %v: %v", prefix, batchIntervalStr, DEFAULT_BATCH_INTERVAL, err)
+		store.batchInterval = DEFAULT_BATCH_INTERVAL
+	}
+
+	return store.initialize(clusterFile, apiVersion)
+}
+
+func (store *FoundationDBStore) initialize(clusterFile string, apiVersion int) error {
+	glog.V(0).Infof("FoundationDB: connecting to cluster file: %s, API version: %d", clusterFile, apiVersion)
+
+	// Set FDB API version
+	if err := fdb.APIVersion(apiVersion); err != nil {
+		return fmt.Errorf("failed to set FoundationDB API version %d: %w", apiVersion, err)
+	}
+
+	// Open database
+	var err error
+	store.database, err = fdb.OpenDatabase(clusterFile)
+	if err != nil {
+		return fmt.Errorf("failed to open FoundationDB database: %w", err)
+	}
+
+	// Create/open seaweedfs directory
+	store.seaweedfsDir, err = directory.CreateOrOpen(store.database, []string{store.directoryPrefix}, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create/open seaweedfs directory: %w", err)
+	}
+
+	// Create/open kv subdirectory for key-value operations
+	store.kvDir, err = directory.CreateOrOpen(store.database, []string{store.directoryPrefix, "kv"}, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create/open kv directory: %w", err)
+	}
+
+	// Conditionally start write batcher
+	// Batching is disabled by default for optimal S3 latency.
+	// When disabled, each write commits immediately in its own transaction.
+	if store.batchEnabled {
+		store.batcher = newWriteBatcher(store, store.batchSize, store.batchInterval)
+		glog.V(0).Infof("FoundationDB: write batching enabled (batch_size=%d, batch_interval=%v)",
+			store.batchSize, store.batchInterval)
+	} else {
+		glog.V(0).Infof("FoundationDB: write batching disabled (direct commit mode for optimal latency)")
+	}
+
+	glog.V(0).Infof("FoundationDB store initialized successfully with directory prefix: %s", store.directoryPrefix)
+	return nil
+}
+
+func (store *FoundationDBStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	// Check if there's already a transaction in this context
+	if _, exists := store.getTransactionFromContext(ctx); exists {
+		return ctx, fmt.Errorf("transaction already in progress for this context")
+	}
+
+	// Create a new transaction
+	tx, err := store.database.CreateTransaction()
+	if err != nil {
+		return ctx, fmt.Errorf("failed to create transaction: %w", err)
+	}
+
+	// Store the transaction in context and return the new context
+	newCtx := store.setTransactionInContext(ctx, tx)
+	return newCtx, nil
+}
+
+func (store *FoundationDBStore) CommitTransaction(ctx context.Context) error {
+	// Get transaction from context
+	tx, exists := store.getTransactionFromContext(ctx)
+	if !exists {
+		return fmt.Errorf("no transaction in progress for this context")
+	}
+
+	// Commit the transaction
+	err := tx.Commit().Get()
+	if err != nil {
+		return fmt.Errorf("failed to commit transaction: %w", err)
+	}
+
+	return nil
+}
+
+func (store *FoundationDBStore) RollbackTransaction(ctx context.Context) error {
+	// Get transaction from context
+	tx, exists := store.getTransactionFromContext(ctx)
+	if !exists {
+		return fmt.Errorf("no transaction in progress for this context")
+	}
+
+	// Cancel the transaction
+	tx.Cancel()
+	return nil
+}
+
+func (store *FoundationDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) error {
+	return store.UpdateEntry(ctx, entry)
+}
+
+func (store *FoundationDBStore) UpdateEntry(ctx context.Context, entry *filer.Entry) error {
+	key := store.genKey(entry.DirAndName())
+
+	value, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encoding %s %+v: %w", entry.FullPath, entry.Attr, err)
+	}
+
+	if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+		value = util.MaybeGzipData(value)
+	}
+
+	if err := errIfValueTooLarge(string(entry.FullPath), value); err != nil {
+		return err
+	}
+
+	// Check if there's a transaction in context
+	if tx, exists := store.getTransactionFromContext(ctx); exists {
+		tx.Set(key, value)
+		return nil
+	}
+
+	// Use write batcher for better throughput
+	if store.batcher != nil {
+		return store.batcher.submit(key, value, true)
+	}
+
+	// Fallback: execute in a new transaction
+	_, err = store.database.Transact(func(tr fdb.Transaction) (interface{}, error) {
+		tr.Set(key, value)
+		return nil, nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("persisting %s: %w", entry.FullPath, err)
+	}
+
+	return nil
+}
+
+func (store *FoundationDBStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+	key := store.genKey(util.FullPath(fullpath).DirAndName())
+
+	var data []byte
+	// Check if there's a transaction in context
+	if tx, exists := store.getTransactionFromContext(ctx); exists {
+		data, err = tx.Get(key).Get()
+	} else {
+		var result interface{}
+		result, err = store.database.ReadTransact(func(rtr fdb.ReadTransaction) (interface{}, error) {
+			return rtr.Get(key).Get()
+		})
+		if err == nil {
+			if resultBytes, ok := result.([]byte); ok {
+				data = resultBytes
+			}
+		}
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("find entry %s: %w", fullpath, err)
+	}
+
+	if data == nil {
+		return nil, filer_pb.ErrNotFound
+	}
+
+	entry = &filer.Entry{
+		FullPath: fullpath,
+	}
+
+	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %w", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *FoundationDBStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error {
+	key := store.genKey(util.FullPath(fullpath).DirAndName())
+
+	// Check if there's a transaction in context
+	if tx, exists := store.getTransactionFromContext(ctx); exists {
+		tx.Clear(key)
+		return nil
+	}
+
+	// Use write batcher for better throughput (nil value = delete)
+	if store.batcher != nil {
+		return store.batcher.submit(key, nil, true)
+	}
+
+	// Fallback: execute in a new transaction
+	_, err := store.database.Transact(func(tr fdb.Transaction) (interface{}, error) {
+		tr.Clear(key)
+		return nil, nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("deleting %s: %w", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *FoundationDBStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error {
+	// Recursively delete all entries in this directory and its subdirectories
+	// We need recursion because our key structure is tuple{dirPath, fileName}
+	// not tuple{dirPath, ...pathComponents}, so a simple prefix range won't catch subdirectories
+
+	// ALWAYS use batched deletion to safely handle directories of any size.
+	// This avoids FoundationDB's 10MB transaction size and 5s timeout limits.
+	//
+	// Note: Even if called within an existing transaction, we create our own batch transactions.
+	// This means DeleteFolderChildren is NOT atomic with an outer transaction, but it ensures
+	// reliability and prevents transaction limit violations.
+	return store.deleteFolderChildrenInBatches(ctx, fullpath)
+}
+
+// deleteFolderChildrenInBatches deletes directory contents in multiple transactions
+// to avoid hitting FoundationDB's transaction size (10MB) and time (5s) limits
+func (store *FoundationDBStore) deleteFolderChildrenInBatches(ctx context.Context, fullpath util.FullPath) error {
+	const BATCH_SIZE = 100 // Delete up to 100 entries per transaction
+
+	// Ensure listing and recursion run outside of any ambient transaction
+	// Store a sentinel nil value so getTransactionFromContext returns false
+	ctxNoTxn := context.WithValue(ctx, transactionKey, (*struct{})(nil))
+
+	for {
+		// Collect one batch of entries
+		var entriesToDelete []util.FullPath
+		var subDirectories []util.FullPath
+
+		// List entries - we'll process BATCH_SIZE at a time
+		_, err := store.ListDirectoryEntries(ctxNoTxn, fullpath, "", true, int64(BATCH_SIZE), func(entry *filer.Entry) (bool, error) {
+			entriesToDelete = append(entriesToDelete, entry.FullPath)
+			if entry.IsDirectory() {
+				subDirectories = append(subDirectories, entry.FullPath)
+			}
+			return true, nil
+		})
+
+		if err != nil {
+			return fmt.Errorf("listing children of %s: %w", fullpath, err)
+		}
+
+		// If no entries found, we're done
+		if len(entriesToDelete) == 0 {
+			break
+		}
+
+		// Recursively delete subdirectories first (also in batches)
+		for _, subDir := range subDirectories {
+			if err := store.deleteFolderChildrenInBatches(ctxNoTxn, subDir); err != nil {
+				return err
+			}
+		}
+
+		// Delete this batch of entries in a single transaction
+		_, err = store.database.Transact(func(tr fdb.Transaction) (interface{}, error) {
+			txCtx := store.setTransactionInContext(context.Background(), tr)
+			for _, entryPath := range entriesToDelete {
+				if delErr := store.DeleteEntry(txCtx, entryPath); delErr != nil {
+					return nil, fmt.Errorf("deleting entry %s: %w", entryPath, delErr)
+				}
+			}
+			return nil, nil
+		})
+
+		if err != nil {
+			return err
+		}
+
+		// If we got fewer entries than BATCH_SIZE, we're done with this directory
+		if len(entriesToDelete) < BATCH_SIZE {
+			break
+		}
+	}
+
+	return nil
+}
+
+func (store *FoundationDBStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc)
+}
+
+func (store *FoundationDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	// Cap limit for optimal FoundationDB performance
+	// Large batches can cause transaction timeouts and increase memory pressure
+	if limit > MAX_DIRECTORY_LIST_LIMIT || limit <= 0 {
+		limit = MAX_DIRECTORY_LIST_LIMIT
+	}
+
+	// Get the range for the entire directory first
+	dirTuple := tuple.Tuple{string(dirPath)}
+	dirRange, err := fdb.PrefixRange(store.seaweedfsDir.Pack(dirTuple))
+	if err != nil {
+		return "", fmt.Errorf("creating prefix range for %s: %w", dirPath, err)
+	}
+
+	// Determine the key range for the scan
+	// Use FDB's range capabilities to only fetch keys matching the prefix
+	var beginKey, endKey fdb.Key
+	dirBeginConv, dirEndConv := dirRange.FDBRangeKeys()
+	dirBegin := dirBeginConv.FDBKey()
+	dirEnd := dirEndConv.FDBKey()
+
+	if prefix != "" {
+		// Build range by bracketing the filename component
+		// Start at Pack(dirPath, prefix) and end at Pack(dirPath, nextPrefix)
+		// where nextPrefix is the next lexicographic string
+		beginKey = store.seaweedfsDir.Pack(tuple.Tuple{string(dirPath), prefix})
+		endKey = dirEnd
+
+		// Use Strinc to get the next string for proper prefix range
+		if nextPrefix, strincErr := fdb.Strinc([]byte(prefix)); strincErr == nil {
+			endKey = store.seaweedfsDir.Pack(tuple.Tuple{string(dirPath), string(nextPrefix)})
+		}
+	} else {
+		// Use entire directory range
+		beginKey = dirBegin
+		endKey = dirEnd
+	}
+
+	// Determine start key and selector based on startFileName
+	var beginSelector fdb.KeySelector
+	if startFileName != "" {
+		// Start from the specified file
+		startKey := store.seaweedfsDir.Pack(tuple.Tuple{string(dirPath), startFileName})
+		if includeStartFile {
+			beginSelector = fdb.FirstGreaterOrEqual(startKey)
+		} else {
+			beginSelector = fdb.FirstGreaterThan(startKey)
+		}
+		// Ensure beginSelector is within our desired range
+		if bytes.Compare(beginSelector.Key.FDBKey(), beginKey.FDBKey()) < 0 {
+			beginSelector = fdb.FirstGreaterOrEqual(beginKey)
+		}
+	} else {
+		// Start from beginning of the range
+		beginSelector = fdb.FirstGreaterOrEqual(beginKey)
+	}
+
+	// End selector is the end of our calculated range
+	endSelector := fdb.FirstGreaterOrEqual(endKey)
+
+	var kvs []fdb.KeyValue
+	var rangeErr error
+	// Check if there's a transaction in context
+	if tx, exists := store.getTransactionFromContext(ctx); exists {
+		sr := fdb.SelectorRange{Begin: beginSelector, End: endSelector}
+		kvs, rangeErr = tx.GetRange(sr, fdb.RangeOptions{Limit: int(limit)}).GetSliceWithError()
+		if rangeErr != nil {
+			return "", fmt.Errorf("scanning %s: %w", dirPath, rangeErr)
+		}
+	} else {
+		result, err := store.database.ReadTransact(func(rtr fdb.ReadTransaction) (interface{}, error) {
+			sr := fdb.SelectorRange{Begin: beginSelector, End: endSelector}
+			kvSlice, err := rtr.GetRange(sr, fdb.RangeOptions{Limit: int(limit)}).GetSliceWithError()
+			if err != nil {
+				return nil, err
+			}
+			return kvSlice, nil
+		})
+		if err != nil {
+			return "", fmt.Errorf("scanning %s: %w", dirPath, err)
+		}
+		var ok bool
+		kvs, ok = result.([]fdb.KeyValue)
+		if !ok {
+			return "", fmt.Errorf("unexpected type from ReadTransact: %T, expected []fdb.KeyValue", result)
+		}
+	}
+
+	for _, kv := range kvs {
+		fileName, extractErr := store.extractFileName(kv.Key)
+		if extractErr != nil {
+			glog.Warningf("list %s: failed to extract fileName from key %v: %v", dirPath, kv.Key, extractErr)
+			continue
+		}
+
+		entry := &filer.Entry{
+			FullPath: util.NewFullPath(string(dirPath), fileName),
+		}
+
+		if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(kv.Value)); decodeErr != nil {
+			glog.V(0).Infof("list %s : %v", entry.FullPath, decodeErr)
+			continue
+		}
+
+		resEachEntryFunc, resEachEntryFuncErr := eachEntryFunc(entry)
+		if resEachEntryFuncErr != nil {
+			glog.ErrorfCtx(ctx, "failed to process eachEntryFunc for entry %q: %v", fileName, resEachEntryFuncErr)
+			return lastFileName, fmt.Errorf("failed to process eachEntryFunc for entry %q: %w", fileName, resEachEntryFuncErr)
+		}
+		if !resEachEntryFunc {
+			break
+		}
+
+		lastFileName = fileName
+	}
+
+	return lastFileName, nil
+}
+
+// KV operations
+func (store *FoundationDBStore) KvPut(ctx context.Context, key []byte, value []byte) error {
+	fdbKey := store.kvDir.Pack(tuple.Tuple{key})
+
+	if err := errIfValueTooLarge(string(key), value); err != nil {
+		return err
+	}
+
+	// Check if there's a transaction in context
+	if tx, exists := store.getTransactionFromContext(ctx); exists {
+		tx.Set(fdbKey, value)
+		return nil
+	}
+
+	_, err := store.database.Transact(func(tr fdb.Transaction) (interface{}, error) {
+		tr.Set(fdbKey, value)
+		return nil, nil
+	})
+
+	return err
+}
+
+func (store *FoundationDBStore) KvGet(ctx context.Context, key []byte) ([]byte, error) {
+	fdbKey := store.kvDir.Pack(tuple.Tuple{key})
+
+	var data []byte
+	var err error
+
+	// Check if there's a transaction in context
+	if tx, exists := store.getTransactionFromContext(ctx); exists {
+		data, err = tx.Get(fdbKey).Get()
+	} else {
+		var result interface{}
+		result, err = store.database.ReadTransact(func(rtr fdb.ReadTransaction) (interface{}, error) {
+			return rtr.Get(fdbKey).Get()
+		})
+		if err == nil {
+			if resultBytes, ok := result.([]byte); ok {
+				data = resultBytes
+			}
+		}
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("kv get %s: %w", string(key), err)
+	}
+	if data == nil {
+		return nil, filer.ErrKvNotFound
+	}
+
+	return data, nil
+}
+
+func (store *FoundationDBStore) KvDelete(ctx context.Context, key []byte) error {
+	fdbKey := store.kvDir.Pack(tuple.Tuple{key})
+
+	// Check if there's a transaction in context
+	if tx, exists := store.getTransactionFromContext(ctx); exists {
+		tx.Clear(fdbKey)
+		return nil
+	}
+
+	_, err := store.database.Transact(func(tr fdb.Transaction) (interface{}, error) {
+		tr.Clear(fdbKey)
+		return nil, nil
+	})
+
+	return err
+}
+
+func (store *FoundationDBStore) Shutdown() {
+	// Stop write batcher
+	if store.batcher != nil {
+		store.batcher.shutdown()
+		store.batcher = nil
+	}
+	// FoundationDB doesn't have an explicit close method for Database
+	glog.V(0).Infof("FoundationDB store shutdown")
+}
+
+// tuplePool reduces allocations in genKey which is called on every FDB operation
+var tuplePool = sync.Pool{
+	New: func() interface{} {
+		// Pre-allocate slice with capacity 2 for (dirPath, fileName)
+		t := make(tuple.Tuple, 2)
+		return &t
+	},
+}
+
+// Helper functions
+func (store *FoundationDBStore) genKey(dirPath, fileName string) fdb.Key {
+	// Get a tuple from pool to avoid slice allocation
+	tp := tuplePool.Get().(*tuple.Tuple)
+	defer func() {
+		// Clear references before returning to pool to avoid memory leaks
+		(*tp)[0] = nil
+		(*tp)[1] = nil
+		tuplePool.Put(tp)
+	}()
+	(*tp)[0] = dirPath
+	(*tp)[1] = fileName
+	return store.seaweedfsDir.Pack(*tp)
+}
+
+func (store *FoundationDBStore) extractFileName(key fdb.Key) (string, error) {
+	t, err := store.seaweedfsDir.Unpack(key)
+	if err != nil {
+		return "", fmt.Errorf("unpack key %v: %w", key, err)
+	}
+	if len(t) != 2 {
+		return "", fmt.Errorf("tuple unexpected length (len=%d, expected 2) for key %v", len(t), key)
+	}
+
+	if fileName, ok := t[1].(string); ok {
+		return fileName, nil
+	}
+	return "", fmt.Errorf("second element not a string (type=%T) for key %v", t[1], key)
+}

weed/filer/foundationdb/foundationdb_store_test.go

@@ -0,0 +1,685 @@
+//go:build foundationdb
+
+package foundationdb
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestFoundationDBStore_Initialize(t *testing.T) {
+	// Test with default configuration
+	config := util.GetViper()
+	config.Set("foundationdb.cluster_file", getTestClusterFile())
+	config.Set("foundationdb.api_version", 740)
+
+	store := &FoundationDBStore{}
+	err := store.Initialize(config, "foundationdb.")
+	if err != nil {
+		t.Skip("FoundationDB not available for testing, skipping")
+	}
+
+	defer store.Shutdown()
+
+	if store.GetName() != "foundationdb" {
+		t.Errorf("Expected store name 'foundationdb', got '%s'", store.GetName())
+	}
+
+	if store.directoryPrefix != "seaweedfs" {
+		t.Errorf("Expected default directory prefix 'seaweedfs', got '%s'", store.directoryPrefix)
+	}
+}
+
+func TestFoundationDBStore_InitializeWithCustomConfig(t *testing.T) {
+	config := util.GetViper()
+	config.Set("foundationdb.cluster_file", getTestClusterFile())
+	config.Set("foundationdb.api_version", 740)
+	config.Set("foundationdb.timeout", "10s")
+	config.Set("foundationdb.max_retry_delay", "2s")
+	config.Set("foundationdb.directory_prefix", "custom_prefix")
+
+	store := &FoundationDBStore{}
+	err := store.Initialize(config, "foundationdb.")
+	if err != nil {
+		t.Skip("FoundationDB not available for testing, skipping")
+	}
+
+	defer store.Shutdown()
+
+	if store.directoryPrefix != "custom_prefix" {
+		t.Errorf("Expected custom directory prefix 'custom_prefix', got '%s'", store.directoryPrefix)
+	}
+
+	if store.timeout != 10*time.Second {
+		t.Errorf("Expected timeout 10s, got %v", store.timeout)
+	}
+
+	if store.maxRetryDelay != 2*time.Second {
+		t.Errorf("Expected max retry delay 2s, got %v", store.maxRetryDelay)
+	}
+}
+
+func TestFoundationDBStore_InitializeInvalidConfig(t *testing.T) {
+	tests := []struct {
+		name     string
+		config   map[string]interface{}
+		errorMsg string
+	}{
+		{
+			name: "invalid timeout",
+			config: map[string]interface{}{
+				"foundationdb.cluster_file":     getTestClusterFile(),
+				"foundationdb.api_version":      740,
+				"foundationdb.timeout":          "invalid",
+				"foundationdb.directory_prefix": "test",
+			},
+			errorMsg: "invalid timeout duration",
+		},
+		{
+			name: "invalid max_retry_delay",
+			config: map[string]interface{}{
+				"foundationdb.cluster_file":     getTestClusterFile(),
+				"foundationdb.api_version":      740,
+				"foundationdb.timeout":          "5s",
+				"foundationdb.max_retry_delay":  "invalid",
+				"foundationdb.directory_prefix": "test",
+			},
+			errorMsg: "invalid max_retry_delay duration",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			config := util.GetViper()
+			for key, value := range tt.config {
+				config.Set(key, value)
+			}
+
+			store := &FoundationDBStore{}
+			err := store.Initialize(config, "foundationdb.")
+			if err == nil {
+				store.Shutdown()
+				t.Errorf("Expected initialization to fail, but it succeeded")
+			} else if !containsString(err.Error(), tt.errorMsg) {
+				t.Errorf("Expected error message to contain '%s', got '%s'", tt.errorMsg, err.Error())
+			}
+		})
+	}
+}
+
+func TestFoundationDBStore_KeyGeneration(t *testing.T) {
+	store := &FoundationDBStore{}
+	err := store.initialize(getTestClusterFile(), 740)
+	if err != nil {
+		t.Skip("FoundationDB not available for testing, skipping")
+	}
+	defer store.Shutdown()
+
+	// Test key generation for different paths
+	testCases := []struct {
+		dirPath  string
+		fileName string
+		desc     string
+	}{
+		{"/", "file.txt", "root directory file"},
+		{"/dir", "file.txt", "subdirectory file"},
+		{"/deep/nested/dir", "file.txt", "deep nested file"},
+		{"/dir with spaces", "file with spaces.txt", "paths with spaces"},
+		{"/unicode/测试", "文件.txt", "unicode paths"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.desc, func(t *testing.T) {
+			key := store.genKey(tc.dirPath, tc.fileName)
+			if len(key) == 0 {
+				t.Error("Generated key should not be empty")
+			}
+
+			// Test that we can extract filename back
+			// Note: This tests internal consistency
+			if tc.fileName != "" {
+				extractedName, err := store.extractFileName(key)
+				if err != nil {
+					t.Errorf("extractFileName failed: %v", err)
+				}
+				if extractedName != tc.fileName {
+					t.Errorf("Expected extracted filename '%s', got '%s'", tc.fileName, extractedName)
+				}
+			}
+		})
+	}
+}
+
+func TestFoundationDBStore_ErrorHandling(t *testing.T) {
+	store := &FoundationDBStore{}
+	err := store.initialize(getTestClusterFile(), 740)
+	if err != nil {
+		t.Skip("FoundationDB not available for testing, skipping")
+	}
+	defer store.Shutdown()
+
+	ctx := context.Background()
+
+	// Test FindEntry with non-existent path
+	_, err = store.FindEntry(ctx, "/non/existent/file.txt")
+	if err == nil {
+		t.Error("Expected error for non-existent file")
+	}
+	if !errors.Is(err, filer_pb.ErrNotFound) {
+		t.Errorf("Expected ErrNotFound, got %v", err)
+	}
+
+	// Test KvGet with non-existent key
+	_, err = store.KvGet(ctx, []byte("non_existent_key"))
+	if err == nil {
+		t.Error("Expected error for non-existent key")
+	}
+	if !errors.Is(err, filer.ErrKvNotFound) {
+		t.Errorf("Expected ErrKvNotFound, got %v", err)
+	}
+
+	// Test transaction state errors
+	err = store.CommitTransaction(ctx)
+	if err == nil {
+		t.Error("Expected error when committing without active transaction")
+	}
+
+	err = store.RollbackTransaction(ctx)
+	if err == nil {
+		t.Error("Expected error when rolling back without active transaction")
+	}
+}
+
+func TestFoundationDBStore_TransactionState(t *testing.T) {
+	store := &FoundationDBStore{}
+	err := store.initialize(getTestClusterFile(), 740)
+	if err != nil {
+		t.Skip("FoundationDB not available for testing, skipping")
+	}
+	defer store.Shutdown()
+
+	ctx := context.Background()
+
+	// Test double transaction begin
+	txCtx, err := store.BeginTransaction(ctx)
+	if err != nil {
+		t.Fatalf("BeginTransaction failed: %v", err)
+	}
+
+	// Try to begin another transaction on the same context
+	_, err = store.BeginTransaction(txCtx)
+	if err == nil {
+		t.Error("Expected error when beginning transaction while one is active")
+	}
+
+	// Commit the transaction
+	err = store.CommitTransaction(txCtx)
+	if err != nil {
+		t.Fatalf("CommitTransaction failed: %v", err)
+	}
+
+	// Now should be able to begin a new transaction
+	txCtx2, err := store.BeginTransaction(ctx)
+	if err != nil {
+		t.Fatalf("BeginTransaction after commit failed: %v", err)
+	}
+
+	// Rollback this time
+	err = store.RollbackTransaction(txCtx2)
+	if err != nil {
+		t.Fatalf("RollbackTransaction failed: %v", err)
+	}
+}
+
+// Benchmark tests
+func BenchmarkFoundationDBStore_InsertEntry(b *testing.B) {
+	store := createBenchmarkStore(b)
+	defer store.Shutdown()
+
+	ctx := context.Background()
+	entry := &filer.Entry{
+		FullPath: "/benchmark/file.txt",
+		Attr: filer.Attr{
+			Mode:  0644,
+			Uid:   1000,
+			Gid:   1000,
+			Mtime: time.Now(),
+		},
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		entry.FullPath = util.NewFullPath("/benchmark", fmt.Sprintf("%x", uint64(i))+".txt")
+		err := store.InsertEntry(ctx, entry)
+		if err != nil {
+			b.Fatalf("InsertEntry failed: %v", err)
+		}
+	}
+}
+
+func BenchmarkFoundationDBStore_FindEntry(b *testing.B) {
+	store := createBenchmarkStore(b)
+	defer store.Shutdown()
+
+	ctx := context.Background()
+
+	// Pre-populate with test entries
+	numEntries := 1000
+	for i := 0; i < numEntries; i++ {
+		entry := &filer.Entry{
+			FullPath: util.NewFullPath("/benchmark", fmt.Sprintf("%x", uint64(i))+".txt"),
+			Attr: filer.Attr{
+				Mode:  0644,
+				Uid:   1000,
+				Gid:   1000,
+				Mtime: time.Now(),
+			},
+		}
+		err := store.InsertEntry(ctx, entry)
+		if err != nil {
+			b.Fatalf("Pre-population InsertEntry failed: %v", err)
+		}
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		path := util.NewFullPath("/benchmark", fmt.Sprintf("%x", uint64(i%numEntries))+".txt")
+		_, err := store.FindEntry(ctx, path)
+		if err != nil {
+			b.Fatalf("FindEntry failed: %v", err)
+		}
+	}
+}
+
+func BenchmarkFoundationDBStore_KvOperations(b *testing.B) {
+	store := createBenchmarkStore(b)
+	defer store.Shutdown()
+
+	ctx := context.Background()
+	key := []byte("benchmark_key")
+	value := []byte("benchmark_value")
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// Put
+		err := store.KvPut(ctx, key, value)
+		if err != nil {
+			b.Fatalf("KvPut failed: %v", err)
+		}
+
+		// Get
+		_, err = store.KvGet(ctx, key)
+		if err != nil {
+			b.Fatalf("KvGet failed: %v", err)
+		}
+	}
+}
+
+// BenchmarkFoundationDBStore_InsertEntry_NoBatch benchmarks insert performance
+// with batching disabled (direct commit mode - optimal for S3 PUT latency)
+func BenchmarkFoundationDBStore_InsertEntry_NoBatch(b *testing.B) {
+	store := createBenchmarkStoreWithBatching(b, false, 100, 1*time.Millisecond)
+	defer store.Shutdown()
+
+	ctx := context.Background()
+	entry := &filer.Entry{
+		FullPath: "/benchmark_nobatch/file.txt",
+		Attr: filer.Attr{
+			Mode:  0644,
+			Uid:   1000,
+			Gid:   1000,
+			Mtime: time.Now(),
+		},
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		entry.FullPath = util.NewFullPath("/benchmark_nobatch", fmt.Sprintf("%x", uint64(i))+".txt")
+		err := store.InsertEntry(ctx, entry)
+		if err != nil {
+			b.Fatalf("InsertEntry failed: %v", err)
+		}
+	}
+}
+
+// BenchmarkFoundationDBStore_InsertEntry_WithBatch benchmarks insert performance
+// with batching enabled (higher throughput for bulk ingestion)
+func BenchmarkFoundationDBStore_InsertEntry_WithBatch(b *testing.B) {
+	store := createBenchmarkStoreWithBatching(b, true, 100, 1*time.Millisecond)
+	defer store.Shutdown()
+
+	ctx := context.Background()
+	entry := &filer.Entry{
+		FullPath: "/benchmark_batch/file.txt",
+		Attr: filer.Attr{
+			Mode:  0644,
+			Uid:   1000,
+			Gid:   1000,
+			Mtime: time.Now(),
+		},
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		entry.FullPath = util.NewFullPath("/benchmark_batch", fmt.Sprintf("%x", uint64(i))+".txt")
+		err := store.InsertEntry(ctx, entry)
+		if err != nil {
+			b.Fatalf("InsertEntry failed: %v", err)
+		}
+	}
+}
+
+// BenchmarkFoundationDBStore_ConcurrentInsert_NoBatch benchmarks concurrent inserts
+// with batching disabled (simulates S3 PUT concurrency)
+func BenchmarkFoundationDBStore_ConcurrentInsert_NoBatch(b *testing.B) {
+	store := createBenchmarkStoreWithBatching(b, false, 100, 1*time.Millisecond)
+	defer store.Shutdown()
+
+	var counter atomic.Uint64
+
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		ctx := context.Background()
+		for pb.Next() {
+			n := counter.Add(1)
+			entry := &filer.Entry{
+				FullPath: util.NewFullPath("/benchmark_concurrent_nobatch", fmt.Sprintf("%d.txt", n)),
+				Attr: filer.Attr{
+					Mode:  0644,
+					Uid:   1000,
+					Gid:   1000,
+					Mtime: time.Now(),
+				},
+			}
+			err := store.InsertEntry(ctx, entry)
+			if err != nil {
+				b.Fatalf("InsertEntry failed: %v", err)
+			}
+		}
+	})
+}
+
+// BenchmarkFoundationDBStore_ConcurrentInsert_WithBatch benchmarks concurrent inserts
+// with batching enabled (tests batch efficiency under concurrent load)
+func BenchmarkFoundationDBStore_ConcurrentInsert_WithBatch(b *testing.B) {
+	store := createBenchmarkStoreWithBatching(b, true, 100, 1*time.Millisecond)
+	defer store.Shutdown()
+
+	var counter atomic.Uint64
+
+	b.ResetTimer()
+	b.RunParallel(func(pb *testing.PB) {
+		ctx := context.Background()
+		for pb.Next() {
+			n := counter.Add(1)
+			entry := &filer.Entry{
+				FullPath: util.NewFullPath("/benchmark_concurrent_batch", fmt.Sprintf("%d.txt", n)),
+				Attr: filer.Attr{
+					Mode:  0644,
+					Uid:   1000,
+					Gid:   1000,
+					Mtime: time.Now(),
+				},
+			}
+			err := store.InsertEntry(ctx, entry)
+			if err != nil {
+				b.Fatalf("InsertEntry failed: %v", err)
+			}
+		}
+	})
+}
+
+// Helper functions
+func getTestClusterFile() string {
+	clusterFile := os.Getenv("FDB_CLUSTER_FILE")
+	if clusterFile == "" {
+		clusterFile = "/var/fdb/config/fdb.cluster"
+	}
+	return clusterFile
+}
+
+func createBenchmarkStore(b *testing.B) *FoundationDBStore {
+	clusterFile := getTestClusterFile()
+	if _, err := os.Stat(clusterFile); os.IsNotExist(err) {
+		b.Skip("FoundationDB cluster file not found, skipping benchmark")
+	}
+
+	store := &FoundationDBStore{}
+	err := store.initialize(clusterFile, 740)
+	if err != nil {
+		b.Skipf("Failed to initialize FoundationDB store: %v", err)
+	}
+
+	return store
+}
+
+// createBenchmarkStoreWithBatching creates a store with specific batching configuration
+// for comparing performance between batched and non-batched modes
+func createBenchmarkStoreWithBatching(b *testing.B, batchEnabled bool, batchSize int, batchInterval time.Duration) *FoundationDBStore {
+	clusterFile := getTestClusterFile()
+	if _, err := os.Stat(clusterFile); os.IsNotExist(err) {
+		b.Skip("FoundationDB cluster file not found, skipping benchmark")
+	}
+
+	store := &FoundationDBStore{
+		batchEnabled:    batchEnabled,
+		batchSize:       batchSize,
+		batchInterval:   batchInterval,
+		directoryPrefix: "benchmark",
+		timeout:         5 * time.Second,
+		maxRetryDelay:   1 * time.Second,
+	}
+	err := store.initialize(clusterFile, 740)
+	if err != nil {
+		b.Skipf("Failed to initialize FoundationDB store: %v", err)
+	}
+
+	// Note: initialize() already creates the batcher if batchEnabled is true
+
+	return store
+}
+
+func getTestStore(t *testing.T) *FoundationDBStore {
+	t.Helper()
+
+	clusterFile := getTestClusterFile()
+	if _, err := os.Stat(clusterFile); os.IsNotExist(err) {
+		t.Skip("FoundationDB cluster file not found, skipping test")
+	}
+
+	store := &FoundationDBStore{}
+	if err := store.initialize(clusterFile, 740); err != nil {
+		t.Skipf("Failed to initialize FoundationDB store: %v", err)
+	}
+
+	return store
+}
+
+func containsString(s, substr string) bool {
+	return strings.Contains(s, substr)
+}
+
+func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) {
+	// This test validates that DeleteFolderChildren always uses batching
+	// to safely handle large directories, regardless of transaction context
+
+	store := getTestStore(t)
+	defer store.Shutdown()
+
+	ctx := context.Background()
+	testDir := util.FullPath(fmt.Sprintf("/test_batch_delete_%d", time.Now().UnixNano()))
+
+	// Create a large directory (> 100 entries to trigger batching)
+	const NUM_ENTRIES = 250
+
+	t.Logf("Creating %d test entries...", NUM_ENTRIES)
+	for i := 0; i < NUM_ENTRIES; i++ {
+		entry := &filer.Entry{
+			FullPath: util.NewFullPath(string(testDir), fmt.Sprintf("file_%04d.txt", i)),
+			Attr: filer.Attr{
+				Mode:  0644,
+				Uid:   1000,
+				Gid:   1000,
+				Mtime: time.Now(),
+			},
+		}
+		if err := store.InsertEntry(ctx, entry); err != nil {
+			t.Fatalf("Failed to insert test entry %d: %v", i, err)
+		}
+	}
+
+	// Test 1: DeleteFolderChildren outside transaction should succeed
+	t.Run("OutsideTransaction", func(t *testing.T) {
+		testDir1 := util.FullPath(fmt.Sprintf("/test_batch_1_%d", time.Now().UnixNano()))
+
+		// Create entries
+		for i := 0; i < NUM_ENTRIES; i++ {
+			entry := &filer.Entry{
+				FullPath: util.NewFullPath(string(testDir1), fmt.Sprintf("file_%04d.txt", i)),
+				Attr: filer.Attr{
+					Mode:  0644,
+					Uid:   1000,
+					Gid:   1000,
+					Mtime: time.Now(),
+				},
+			}
+			store.InsertEntry(ctx, entry)
+		}
+
+		// Delete with batching
+		err := store.DeleteFolderChildren(ctx, testDir1)
+		if err != nil {
+			t.Errorf("DeleteFolderChildren outside transaction should succeed, got error: %v", err)
+		}
+
+		// Verify all entries deleted
+		var count int
+		store.ListDirectoryEntries(ctx, testDir1, "", true, 1000, func(entry *filer.Entry) (bool, error) {
+			count++
+			return true, nil
+		})
+		if count != 0 {
+			t.Errorf("Expected all entries to be deleted, found %d", count)
+		}
+	})
+
+	// Test 2: DeleteFolderChildren with transaction context - uses its own batched transactions
+	t.Run("WithTransactionContext", func(t *testing.T) {
+		testDir2 := util.FullPath(fmt.Sprintf("/test_batch_2_%d", time.Now().UnixNano()))
+
+		// Create entries
+		for i := 0; i < NUM_ENTRIES; i++ {
+			entry := &filer.Entry{
+				FullPath: util.NewFullPath(string(testDir2), fmt.Sprintf("file_%04d.txt", i)),
+				Attr: filer.Attr{
+					Mode:  0644,
+					Uid:   1000,
+					Gid:   1000,
+					Mtime: time.Now(),
+				},
+			}
+			store.InsertEntry(ctx, entry)
+		}
+
+		// Start a transaction (DeleteFolderChildren will ignore it and use its own batching)
+		txCtx, err := store.BeginTransaction(ctx)
+		if err != nil {
+			t.Fatalf("BeginTransaction failed: %v", err)
+		}
+
+		// Delete large directory - should succeed with batching
+		err = store.DeleteFolderChildren(txCtx, testDir2)
+		if err != nil {
+			t.Errorf("DeleteFolderChildren should succeed with batching even when transaction context present, got: %v", err)
+		}
+
+		// Rollback transaction (DeleteFolderChildren used its own transactions, so this doesn't affect deletions)
+		store.RollbackTransaction(txCtx)
+
+		// Verify entries are still deleted (because DeleteFolderChildren managed its own transactions)
+		var count int
+		store.ListDirectoryEntries(ctx, testDir2, "", true, 1000, func(entry *filer.Entry) (bool, error) {
+			count++
+			return true, nil
+		})
+
+		if count != 0 {
+			t.Errorf("Expected all entries to be deleted, found %d (DeleteFolderChildren uses its own transactions)", count)
+		}
+	})
+
+	// Test 3: Nested directories with batching
+	t.Run("NestedDirectories", func(t *testing.T) {
+		testDir3 := util.FullPath(fmt.Sprintf("/test_batch_3_%d", time.Now().UnixNano()))
+
+		// Create nested structure
+		for i := 0; i < 50; i++ {
+			// Files in root
+			entry := &filer.Entry{
+				FullPath: util.NewFullPath(string(testDir3), fmt.Sprintf("file_%02d.txt", i)),
+				Attr: filer.Attr{
+					Mode:  0644,
+					Uid:   1000,
+					Gid:   1000,
+					Mtime: time.Now(),
+				},
+			}
+			store.InsertEntry(ctx, entry)
+
+			// Subdirectory
+			subDir := &filer.Entry{
+				FullPath: util.NewFullPath(string(testDir3), fmt.Sprintf("dir_%02d", i)),
+				Attr: filer.Attr{
+					Mode:  0755 | os.ModeDir,
+					Uid:   1000,
+					Gid:   1000,
+					Mtime: time.Now(),
+				},
+			}
+			store.InsertEntry(ctx, subDir)
+
+			// Files in subdirectory
+			for j := 0; j < 3; j++ {
+				subEntry := &filer.Entry{
+					FullPath: util.NewFullPath(string(testDir3)+"/"+fmt.Sprintf("dir_%02d", i), fmt.Sprintf("subfile_%02d.txt", j)),
+					Attr: filer.Attr{
+						Mode:  0644,
+						Uid:   1000,
+						Gid:   1000,
+						Mtime: time.Now(),
+					},
+				}
+				store.InsertEntry(ctx, subEntry)
+			}
+		}
+
+		// Delete all with batching
+		err := store.DeleteFolderChildren(ctx, testDir3)
+		if err != nil {
+			t.Errorf("DeleteFolderChildren should handle nested directories, got: %v", err)
+		}
+
+		// Verify all deleted
+		var count int
+		store.ListDirectoryEntries(ctx, testDir3, "", true, 1000, func(entry *filer.Entry) (bool, error) {
+			count++
+			return true, nil
+		})
+		if count != 0 {
+			t.Errorf("Expected all nested entries to be deleted, found %d", count)
+		}
+	})
+
+	// Cleanup
+	store.DeleteFolderChildren(ctx, testDir)
+}

weed/filer/foundationdb/value_size.go

@@ -0,0 +1,16 @@
+package foundationdb
+
+import "fmt"
+
+// FDB_VALUE_SIZE_LIMIT is Apple's documented maximum value size: 100,000
+// bytes, not 100 KiB. A value over it is rejected by FoundationDB itself with
+// error 2103, so the store checks it and names it.
+const FDB_VALUE_SIZE_LIMIT = 100 * 1000
+
+func errIfValueTooLarge(name string, value []byte) error {
+	if len(value) > FDB_VALUE_SIZE_LIMIT {
+		return fmt.Errorf("entry %s exceeds FoundationDB value size limit (%d > %d bytes)",
+			name, len(value), FDB_VALUE_SIZE_LIMIT)
+	}
+	return nil
+}

weed/filer/foundationdb/value_size_test.go

@@ -0,0 +1,32 @@
+package foundationdb
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestFDBValueSizeLimitConstant(t *testing.T) {
+	if FDB_VALUE_SIZE_LIMIT != 100*1000 {
+		t.Fatalf("FDB_VALUE_SIZE_LIMIT = %d, want 100000", FDB_VALUE_SIZE_LIMIT)
+	}
+}
+
+func TestErrIfValueTooLarge(t *testing.T) {
+	if err := errIfValueTooLarge("/p", make([]byte, FDB_VALUE_SIZE_LIMIT)); err != nil {
+		t.Fatalf("100000-byte value should pass, got %v", err)
+	}
+	err := errIfValueTooLarge("/buckets/b/o", make([]byte, FDB_VALUE_SIZE_LIMIT+1))
+	if err == nil {
+		t.Fatal("100001-byte value should fail")
+	}
+	msg := err.Error()
+	if !strings.Contains(msg, "value size limit") {
+		t.Fatalf("error should name value size limit, got %q", msg)
+	}
+	if strings.Contains(msg, "transaction size limit") {
+		t.Fatalf("error must not mention transaction size limit, got %q", msg)
+	}
+	if !strings.Contains(msg, "100001") || !strings.Contains(msg, "100000") {
+		t.Fatalf("error should include both sizes, got %q", msg)
+	}
+}

weed/filer/gateway_upload.go

@@ -0,0 +1,163 @@
+package filer
+
+import (
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// GatewayChunkUploader is the minimum surface the shared chunk-upload helper
+// needs from a concrete uploader. It is satisfied by *operation.Uploader and
+// can be mocked in tests without pulling in the weed/operation package.
+type GatewayChunkUploader interface {
+	UploadWithRetry(
+		filerClient filer_pb.FilerClient,
+		assignRequest *filer_pb.AssignVolumeRequest,
+		uploadOption *operation.UploadOption,
+		reader io.Reader,
+	) (fileId string, uploadResult *operation.UploadResult, err error, data []byte)
+}
+
+// GatewayChunkUploadRequest captures the inputs SaveGatewayDataAsChunk needs.
+// All fields except Reader and FilerClient are optional — empty values map to
+// sensible defaults that match what `weed filer`, `weed mount`, `weed nfs`,
+// and the WebDAV gateway produced before this helper was factored out.
+type GatewayChunkUploadRequest struct {
+	// FilerClient issues the AssignVolume RPC. Must be non-nil.
+	FilerClient filer_pb.FilerClient
+	// Uploader executes the HTTP upload. When nil, operation.NewUploader()
+	// is used so the common case needs no explicit wiring.
+	Uploader GatewayChunkUploader
+	// Reader supplies the bytes to upload. Must be non-nil; the helper
+	// reads it fully and relies on UploadWithRetry to report short reads.
+	Reader io.Reader
+
+	// Logical path of the target file on the filer. Used by AssignVolume
+	// for placement policies (replication, collection, etc.).
+	FullPath string
+	// File name used in the upload form; defaults to the last path segment
+	// of FullPath when empty.
+	Filename string
+	// Offset of the data inside the logical file — copied into the
+	// returned FileChunk.Offset so the filer's chunk view layer can
+	// reconstruct the file correctly.
+	Offset int64
+	// TsNs is the per-chunk modification timestamp written onto the
+	// resulting FileChunk. Typically time.Now().UnixNano().
+	TsNs int64
+
+	// Assign-time placement options.
+	Collection  string
+	Replication string
+	TtlSec      int32
+	DiskType    string
+	DataCenter  string
+
+	// Upload-time options.
+	Cipher   bool
+	MimeType string
+	PairMap  map[string]string
+
+	// VolumeServerAccess selects how the filer generates the chunk URL.
+	// Supported values: "direct", "publicUrl", "filerProxy". When set to
+	// "filerProxy", FilerHTTPAddress must be non-empty.
+	VolumeServerAccess string
+	// FilerHTTPAddress is the filer host:port that proxies chunk URLs when
+	// VolumeServerAccess == "filerProxy".
+	FilerHTTPAddress string
+}
+
+// SaveGatewayDataAsChunk uploads the bytes in `req.Reader` as a single chunk
+// on a volume server and returns a filer FileChunk describing it. It is the
+// shared chunk-upload path for the NFS, WebDAV, and future gateway servers;
+// mount has extra per-request caching and a pre-allocated file-id pool that
+// still live in weed/mount.
+//
+// Semantics:
+//
+//   - AssignVolume is driven by the filer client in `req.FilerClient`.
+//   - The chunk URL is built by the standard operation.UploadOption machinery
+//     and then optionally rewritten when VolumeServerAccess == "filerProxy".
+//   - The returned chunk's Offset is `req.Offset` and its ModifiedTsNs is
+//     `req.TsNs`. Callers that append to an existing file are responsible
+//     for installing the chunk into the entry's chunk list (typically via a
+//     filer UpdateEntry).
+func SaveGatewayDataAsChunk(req GatewayChunkUploadRequest) (*filer_pb.FileChunk, error) {
+	if req.FilerClient == nil {
+		return nil, errors.New("SaveGatewayDataAsChunk: nil filer client")
+	}
+	if req.Reader == nil {
+		return nil, errors.New("SaveGatewayDataAsChunk: nil reader")
+	}
+
+	uploader := req.Uploader
+	if uploader == nil {
+		realUploader, err := operation.NewUploader()
+		if err != nil {
+			return nil, fmt.Errorf("SaveGatewayDataAsChunk: new uploader: %w", err)
+		}
+		uploader = realUploader
+	}
+
+	filename := req.Filename
+	if filename == "" {
+		if slash := lastSlashIndex(req.FullPath); slash >= 0 && slash+1 < len(req.FullPath) {
+			filename = req.FullPath[slash+1:]
+		} else {
+			filename = req.FullPath
+		}
+	}
+
+	uploadOption := &operation.UploadOption{
+		Filename:          filename,
+		Cipher:            req.Cipher,
+		IsInputCompressed: false,
+		MimeType:          req.MimeType,
+		PairMap:           req.PairMap,
+	}
+
+	if req.VolumeServerAccess == "filerProxy" && req.FilerHTTPAddress != "" {
+		uploadOption.GenUploadUrl = operation.GenUploadUrlProxy(req.FilerHTTPAddress)
+	}
+
+	assignRequest := &filer_pb.AssignVolumeRequest{
+		Count:       1,
+		Replication: req.Replication,
+		Collection:  req.Collection,
+		TtlSec:      req.TtlSec,
+		DiskType:    req.DiskType,
+		DataCenter:  req.DataCenter,
+		Path:        req.FullPath,
+	}
+
+	fileID, uploadResult, uploadErr, _ := uploader.UploadWithRetry(
+		req.FilerClient,
+		assignRequest,
+		uploadOption,
+		req.Reader,
+	)
+	if uploadErr != nil {
+		return nil, fmt.Errorf("upload data: %w", uploadErr)
+	}
+	if uploadResult == nil {
+		return nil, errors.New("upload data: missing upload result")
+	}
+	if uploadResult.Error != "" {
+		return nil, fmt.Errorf("upload result: %s", uploadResult.Error)
+	}
+	return uploadResult.ToPbFileChunk(fileID, req.Offset, req.TsNs), nil
+}
+
+// lastSlashIndex returns the index of the last '/' in s, or -1 if none.
+// Intentionally local so this file has no new test-only imports.
+func lastSlashIndex(s string) int {
+	for i := len(s) - 1; i >= 0; i-- {
+		if s[i] == '/' {
+			return i
+		}
+	}
+	return -1
+}

weed/filer/hbase/hbase_store.go

@@ -0,0 +1,238 @@
+package hbase
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/tsuna/gohbase"
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &HbaseStore{})
+}
+
+type HbaseStore struct {
+	Client    gohbase.Client
+	table     []byte
+	cfKv      string
+	cfMetaDir string
+	column    string
+}
+
+func (store *HbaseStore) GetName() string {
+	return "hbase"
+}
+
+func (store *HbaseStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	return store.initialize(
+		configuration.GetString(prefix+"zkquorum"),
+		configuration.GetString(prefix+"table"),
+	)
+}
+
+func (store *HbaseStore) initialize(zkquorum, table string) (err error) {
+	store.Client = gohbase.NewClient(zkquorum)
+	store.table = []byte(table)
+	store.cfKv = "kv"
+	store.cfMetaDir = "meta"
+	store.column = "a"
+
+	// check table exists
+	key := "whatever"
+	headers := map[string][]string{store.cfMetaDir: nil}
+	get, err := hrpc.NewGet(context.Background(), store.table, []byte(key), hrpc.Families(headers))
+	if err != nil {
+		return fmt.Errorf("NewGet returned an error: %w", err)
+	}
+	_, err = store.Client.Get(get)
+	if err != gohbase.TableNotFound {
+		return nil
+	}
+
+	// create table
+	adminClient := gohbase.NewAdminClient(zkquorum)
+	cFamilies := []string{store.cfKv, store.cfMetaDir}
+	cf := make(map[string]map[string]string, len(cFamilies))
+	for _, f := range cFamilies {
+		cf[f] = nil
+	}
+	ct := hrpc.NewCreateTable(context.Background(), []byte(table), cf)
+	if err := adminClient.CreateTable(ct); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (store *HbaseStore) InsertEntry(ctx context.Context, entry *filer.Entry) error {
+	value, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
+	}
+	if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+		value = util.MaybeGzipData(value)
+	}
+
+	return store.doPut(ctx, store.cfMetaDir, []byte(entry.FullPath), value, entry.TtlSec)
+}
+
+func (store *HbaseStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *HbaseStore) FindEntry(ctx context.Context, path util.FullPath) (entry *filer.Entry, err error) {
+	value, err := store.doGet(ctx, store.cfMetaDir, []byte(path))
+	if err != nil {
+		if err == filer.ErrKvNotFound {
+			return nil, filer_pb.ErrNotFound
+		}
+		return nil, err
+	}
+
+	entry = &filer.Entry{
+		FullPath: path,
+	}
+	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData(value))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+	return entry, nil
+}
+
+func (store *HbaseStore) DeleteEntry(ctx context.Context, path util.FullPath) (err error) {
+	return store.doDelete(ctx, store.cfMetaDir, []byte(path))
+}
+
+func (store *HbaseStore) DeleteFolderChildren(ctx context.Context, path util.FullPath) (err error) {
+
+	family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}}
+	expectedPrefix := []byte(path.Child(""))
+	scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family))
+	if err != nil {
+		return err
+	}
+
+	scanner := store.Client.Scan(scan)
+	defer scanner.Close()
+	for {
+		res, err := scanner.Next()
+		if err != nil {
+			break
+		}
+		if len(res.Cells) == 0 {
+			continue
+		}
+		cell := res.Cells[0]
+
+		if !bytes.HasPrefix(cell.Row, expectedPrefix) {
+			break
+		}
+		fullpath := util.FullPath(cell.Row)
+		dir, _ := fullpath.DirAndName()
+		if dir != string(path) {
+			continue
+		}
+
+		err = store.doDelete(ctx, store.cfMetaDir, cell.Row)
+		if err != nil {
+			break
+		}
+
+	}
+	return
+}
+
+func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (string, error) {
+	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc)
+}
+
+func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}}
+	expectedPrefix := []byte(dirPath.Child(prefix))
+	scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family))
+	if err != nil {
+		return lastFileName, err
+	}
+
+	scanner := store.Client.Scan(scan)
+	defer scanner.Close()
+	for {
+		res, scanErr := scanner.Next()
+		if scanErr == io.EOF {
+			break
+		}
+		if scanErr != nil {
+			return lastFileName, scanErr
+		}
+		if len(res.Cells) == 0 {
+			continue
+		}
+		cell := res.Cells[0]
+
+		if !bytes.HasPrefix(cell.Row, expectedPrefix) {
+			break
+		}
+
+		fullpath := util.FullPath(cell.Row)
+		dir, fileName := fullpath.DirAndName()
+		if dir != string(dirPath) {
+			continue
+		}
+
+		value := cell.Value
+
+		if fileName == startFileName && !includeStartFile {
+			continue
+		}
+
+		limit--
+		if limit < 0 {
+			break
+		}
+
+		lastFileName = fileName
+
+		entry := &filer.Entry{
+			FullPath: fullpath,
+		}
+		if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(value)); decodeErr != nil {
+			err = decodeErr
+			glog.V(0).InfofCtx(ctx, "list %s : %v", entry.FullPath, err)
+			break
+		}
+
+		resEachEntryFunc, resEachEntryFuncErr := eachEntryFunc(entry)
+		if resEachEntryFuncErr != nil {
+			return lastFileName, fmt.Errorf("failed to process eachEntryFunc for entry %q: %w", entry.FullPath, resEachEntryFuncErr)
+		}
+
+		if !resEachEntryFunc {
+			break
+		}
+	}
+
+	return lastFileName, err
+}
+
+func (store *HbaseStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+
+func (store *HbaseStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *HbaseStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *HbaseStore) Shutdown() {
+	store.Client.Close()
+}

weed/filer/hbase/hbase_store_kv.go

@@ -0,0 +1,77 @@
+package hbase
+
+import (
+	"context"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+const (
+	COLUMN_NAME = "a"
+)
+
+func (store *HbaseStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+	return store.doPut(ctx, store.cfKv, key, value, 0)
+}
+
+func (store *HbaseStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+	return store.doGet(ctx, store.cfKv, key)
+}
+
+func (store *HbaseStore) KvDelete(ctx context.Context, key []byte) (err error) {
+	return store.doDelete(ctx, store.cfKv, key)
+}
+
+func (store *HbaseStore) doPut(ctx context.Context, cf string, key, value []byte, ttlSecond int32) (err error) {
+	if ttlSecond > 0 {
+		return store.doPutWithOptions(ctx, cf, key, value, hrpc.Durability(hrpc.AsyncWal), hrpc.TTL(time.Duration(ttlSecond)*time.Second))
+	}
+	return store.doPutWithOptions(ctx, cf, key, value, hrpc.Durability(hrpc.AsyncWal))
+}
+
+func (store *HbaseStore) doPutWithOptions(ctx context.Context, cf string, key, value []byte, options ...func(hrpc.Call) error) (err error) {
+	values := map[string]map[string][]byte{cf: map[string][]byte{}}
+	values[cf][COLUMN_NAME] = value
+	putRequest, err := hrpc.NewPut(ctx, store.table, key, values, options...)
+	if err != nil {
+		return err
+	}
+	_, err = store.Client.Put(putRequest)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (store *HbaseStore) doGet(ctx context.Context, cf string, key []byte) (value []byte, err error) {
+	family := map[string][]string{cf: {COLUMN_NAME}}
+	getRequest, err := hrpc.NewGet(context.Background(), store.table, key, hrpc.Families(family))
+	if err != nil {
+		return nil, err
+	}
+	getResp, err := store.Client.Get(getRequest)
+	if err != nil {
+		return nil, err
+	}
+	if len(getResp.Cells) == 0 {
+		return nil, filer.ErrKvNotFound
+	}
+
+	return getResp.Cells[0].Value, nil
+}
+
+func (store *HbaseStore) doDelete(ctx context.Context, cf string, key []byte) (err error) {
+	values := map[string]map[string][]byte{cf: map[string][]byte{}}
+	values[cf][COLUMN_NAME] = nil
+	deleteRequest, err := hrpc.NewDel(ctx, store.table, key, values, hrpc.Durability(hrpc.AsyncWal))
+	if err != nil {
+		return err
+	}
+	_, err = store.Client.Delete(deleteRequest)
+	if err != nil {
+		return err
+	}
+	return nil
+}

weed/filer/interval_list.go

@@ -0,0 +1,259 @@
+package filer
+
+import (
+	"math"
+	"sync"
+)
+
+type IntervalValue interface {
+	SetStartStop(start, stop int64)
+	Clone() IntervalValue
+}
+
+type Interval[T IntervalValue] struct {
+	StartOffset int64
+	StopOffset  int64
+	TsNs        int64
+	Value       T
+	Prev        *Interval[T]
+	Next        *Interval[T]
+}
+
+func (interval *Interval[T]) Size() int64 {
+	return interval.StopOffset - interval.StartOffset
+}
+
+// IntervalList mark written intervals within one page chunk
+type IntervalList[T IntervalValue] struct {
+	head *Interval[T]
+	tail *Interval[T]
+	Lock sync.RWMutex
+}
+
+func NewIntervalList[T IntervalValue]() *IntervalList[T] {
+	list := &IntervalList[T]{
+		head: &Interval[T]{
+			StartOffset: -1,
+			StopOffset:  -1,
+		},
+		tail: &Interval[T]{
+			StartOffset: math.MaxInt64,
+			StopOffset:  math.MaxInt64,
+		},
+	}
+	return list
+}
+
+func (list *IntervalList[T]) Front() (interval *Interval[T]) {
+	return list.head.Next
+}
+
+func (list *IntervalList[T]) AppendInterval(interval *Interval[T]) {
+	list.Lock.Lock()
+	defer list.Lock.Unlock()
+
+	if list.head.Next == nil {
+		list.head.Next = interval
+	}
+	interval.Prev = list.tail.Prev
+	if list.tail.Prev != nil {
+		list.tail.Prev.Next = interval
+	}
+	list.tail.Prev = interval
+}
+
+func (list *IntervalList[T]) Overlay(startOffset, stopOffset, tsNs int64, value T) {
+	if startOffset >= stopOffset {
+		return
+	}
+	interval := &Interval[T]{
+		StartOffset: startOffset,
+		StopOffset:  stopOffset,
+		TsNs:        tsNs,
+		Value:       value,
+	}
+
+	list.Lock.Lock()
+	defer list.Lock.Unlock()
+
+	list.overlayInterval(interval)
+}
+
+func (list *IntervalList[T]) InsertInterval(startOffset, stopOffset, tsNs int64, value T) {
+	interval := &Interval[T]{
+		StartOffset: startOffset,
+		StopOffset:  stopOffset,
+		TsNs:        tsNs,
+		Value:       value,
+	}
+
+	list.Lock.Lock()
+	defer list.Lock.Unlock()
+
+	value.SetStartStop(startOffset, stopOffset)
+	list.insertInterval(interval)
+}
+
+func (list *IntervalList[T]) insertInterval(interval *Interval[T]) {
+	prev := list.head
+	next := prev.Next
+
+	for interval.StartOffset < interval.StopOffset {
+		if next == nil {
+			// add to the end
+			list.insertBetween(prev, interval, list.tail)
+			break
+		}
+
+		// interval is ahead of the next
+		if interval.StopOffset <= next.StartOffset {
+			list.insertBetween(prev, interval, next)
+			break
+		}
+
+		// interval is after the next
+		if next.StopOffset <= interval.StartOffset {
+			prev = next
+			next = next.Next
+			continue
+		}
+
+		// intersecting next and interval
+		if interval.TsNs >= next.TsNs {
+			// interval is newer
+			if next.StartOffset < interval.StartOffset {
+				// left side of next is ahead of interval
+				t := &Interval[T]{
+					StartOffset: next.StartOffset,
+					StopOffset:  interval.StartOffset,
+					TsNs:        next.TsNs,
+					Value:       next.Value.Clone().(T),
+				}
+				t.Value.SetStartStop(t.StartOffset, t.StopOffset)
+				list.insertBetween(prev, t, interval)
+				next.StartOffset = interval.StartOffset
+				next.Value.SetStartStop(next.StartOffset, next.StopOffset)
+				prev = t
+			}
+			if interval.StopOffset < next.StopOffset {
+				// right side of next is after interval
+				next.StartOffset = interval.StopOffset
+				next.Value.SetStartStop(next.StartOffset, next.StopOffset)
+				list.insertBetween(prev, interval, next)
+				break
+			} else {
+				// next is covered
+				prev.Next = interval
+				next = next.Next
+			}
+		} else {
+			// next is newer
+			if interval.StartOffset < next.StartOffset {
+				// left side of interval is ahead of next
+				t := &Interval[T]{
+					StartOffset: interval.StartOffset,
+					StopOffset:  next.StartOffset,
+					TsNs:        interval.TsNs,
+					Value:       interval.Value.Clone().(T),
+				}
+				t.Value.SetStartStop(t.StartOffset, t.StopOffset)
+				list.insertBetween(prev, t, next)
+				interval.StartOffset = next.StartOffset
+				interval.Value.SetStartStop(interval.StartOffset, interval.StopOffset)
+			}
+			if next.StopOffset < interval.StopOffset {
+				// right side of interval is after next
+				interval.StartOffset = next.StopOffset
+				interval.Value.SetStartStop(interval.StartOffset, interval.StopOffset)
+			} else {
+				// interval is covered
+				break
+			}
+		}
+
+	}
+}
+
+func (list *IntervalList[T]) insertBetween(a, interval, b *Interval[T]) {
+	a.Next = interval
+	b.Prev = interval
+	if a != list.head {
+		interval.Prev = a
+	}
+	if b != list.tail {
+		interval.Next = b
+	}
+}
+
+func (list *IntervalList[T]) overlayInterval(interval *Interval[T]) {
+
+	//t := list.head
+	//for ; t.Next != nil; t = t.Next {
+	//	if t.TsNs > interval.TsNs {
+	//		println("writes is out of order", t.TsNs-interval.TsNs, "ns")
+	//	}
+	//}
+
+	p := list.head
+	for ; p.Next != nil && p.Next.StopOffset <= interval.StartOffset; p = p.Next {
+	}
+	q := list.tail
+	for ; q.Prev != nil && q.Prev.StartOffset >= interval.StopOffset; q = q.Prev {
+	}
+
+	// left side
+	// interval after p.Next start
+	if p.Next != nil && p.Next.StartOffset < interval.StartOffset {
+		t := &Interval[T]{
+			StartOffset: p.Next.StartOffset,
+			StopOffset:  interval.StartOffset,
+			TsNs:        p.Next.TsNs,
+			Value:       p.Next.Value,
+		}
+		p.Next = t
+		if p != list.head {
+			t.Prev = p
+		}
+		t.Next = interval
+		interval.Prev = t
+	} else {
+		p.Next = interval
+		if p != list.head {
+			interval.Prev = p
+		}
+	}
+
+	// right side
+	// interval ends before p.Prev
+	if q.Prev != nil && interval.StopOffset < q.Prev.StopOffset {
+		t := &Interval[T]{
+			StartOffset: interval.StopOffset,
+			StopOffset:  q.Prev.StopOffset,
+			TsNs:        q.Prev.TsNs,
+			Value:       q.Prev.Value,
+		}
+		q.Prev = t
+		if q != list.tail {
+			t.Next = q
+		}
+		interval.Next = t
+		t.Prev = interval
+	} else {
+		q.Prev = interval
+		if q != list.tail {
+			interval.Next = q
+		}
+	}
+
+}
+
+func (list *IntervalList[T]) Len() int {
+	list.Lock.RLock()
+	defer list.Lock.RUnlock()
+
+	var count int
+	for t := list.head; t != nil; t = t.Next {
+		count++
+	}
+	return count - 1
+}

weed/filer/interval_list_test.go

@@ -0,0 +1,328 @@
+package filer
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type IntervalInt int
+
+func (i IntervalInt) SetStartStop(start, stop int64) {
+}
+func (i IntervalInt) Clone() IntervalValue {
+	return i
+}
+
+func TestIntervalList_Overlay(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.Overlay(0, 100, 1, 1)
+	list.Overlay(50, 150, 2, 2)
+	list.Overlay(200, 250, 3, 3)
+	list.Overlay(225, 250, 4, 4)
+	list.Overlay(175, 210, 5, 5)
+	list.Overlay(0, 25, 6, 6)
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 6, list.Len())
+	println()
+	list.Overlay(50, 150, 7, 7)
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 6, list.Len())
+}
+
+func TestIntervalList_Overlay2(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.Overlay(50, 100, 1, 1)
+	list.Overlay(0, 50, 2, 2)
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+}
+
+func TestIntervalList_Overlay3(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.Overlay(50, 100, 1, 1)
+	assert.Equal(t, 1, list.Len())
+
+	list.Overlay(0, 60, 2, 2)
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 2, list.Len())
+}
+
+func TestIntervalList_Overlay4(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.Overlay(50, 100, 1, 1)
+	list.Overlay(0, 100, 2, 2)
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 1, list.Len())
+}
+
+func TestIntervalList_Overlay5(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.Overlay(50, 100, 1, 1)
+	list.Overlay(0, 110, 2, 2)
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 1, list.Len())
+}
+
+func TestIntervalList_Overlay6(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.Overlay(50, 100, 1, 1)
+	list.Overlay(50, 110, 2, 2)
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 1, list.Len())
+}
+
+func TestIntervalList_Overlay7(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.Overlay(50, 100, 1, 1)
+	list.Overlay(50, 90, 2, 2)
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 2, list.Len())
+}
+
+func TestIntervalList_Overlay8(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.Overlay(50, 100, 1, 1)
+	list.Overlay(60, 90, 2, 2)
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 3, list.Len())
+}
+
+func TestIntervalList_Overlay9(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.Overlay(50, 100, 1, 1)
+	list.Overlay(60, 100, 2, 2)
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 2, list.Len())
+}
+
+func TestIntervalList_Overlay10(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.Overlay(50, 100, 1, 1)
+	list.Overlay(60, 110, 2, 2)
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 2, list.Len())
+}
+
+func TestIntervalList_Overlay11(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.Overlay(0, 100, 1, 1)
+	list.Overlay(100, 110, 2, 2)
+	list.Overlay(0, 90, 3, 3)
+	list.Overlay(0, 80, 4, 4)
+	list.Overlay(0, 90, 5, 5)
+	list.Overlay(90, 90, 6, 6)
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 3, list.Len())
+}
+
+func TestIntervalList_insertInterval1(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.InsertInterval(50, 150, 2, 2)
+	list.InsertInterval(200, 250, 3, 3)
+
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 2, list.Len())
+}
+
+func TestIntervalList_insertInterval2(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.InsertInterval(50, 150, 2, 2)
+	list.InsertInterval(0, 25, 3, 3)
+
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 2, list.Len())
+}
+
+func TestIntervalList_insertInterval3(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.InsertInterval(50, 150, 2, 2)
+	list.InsertInterval(200, 250, 4, 4)
+
+	list.InsertInterval(0, 75, 3, 3)
+
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 3, list.Len())
+}
+
+func TestIntervalList_insertInterval4(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.InsertInterval(200, 250, 4, 4)
+
+	list.InsertInterval(0, 225, 3, 3)
+
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 2, list.Len())
+}
+
+func TestIntervalList_insertInterval5(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+	list.InsertInterval(200, 250, 4, 4)
+
+	list.InsertInterval(0, 225, 5, 5)
+
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 2, list.Len())
+}
+
+func TestIntervalList_insertInterval6(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+
+	list.InsertInterval(50, 150, 2, 2)
+	list.InsertInterval(200, 250, 4, 4)
+
+	list.InsertInterval(0, 275, 1, 1)
+
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 5, list.Len())
+}
+
+func TestIntervalList_insertInterval7(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+
+	list.InsertInterval(50, 150, 2, 2)
+	list.InsertInterval(200, 250, 4, 4)
+
+	list.InsertInterval(75, 275, 1, 1)
+
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 4, list.Len())
+}
+
+func TestIntervalList_insertInterval8(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+
+	list.InsertInterval(50, 150, 2, 2)
+	list.InsertInterval(200, 250, 4, 4)
+
+	list.InsertInterval(75, 275, 3, 3)
+
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 4, list.Len())
+}
+
+func TestIntervalList_insertInterval9(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+
+	list.InsertInterval(50, 150, 2, 2)
+	list.InsertInterval(200, 250, 4, 4)
+
+	list.InsertInterval(50, 150, 3, 3)
+
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 2, list.Len())
+}
+
+func TestIntervalList_insertInterval10(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+
+	list.InsertInterval(50, 100, 2, 2)
+
+	list.InsertInterval(200, 300, 4, 4)
+
+	list.InsertInterval(100, 200, 5, 5)
+
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 3, list.Len())
+}
+
+func TestIntervalList_insertInterval11(t *testing.T) {
+	list := NewIntervalList[IntervalInt]()
+
+	list.InsertInterval(0, 64, 1, 1)
+
+	list.InsertInterval(72, 136, 3, 3)
+
+	list.InsertInterval(64, 128, 2, 2)
+
+	list.InsertInterval(68, 72, 4, 4)
+
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 4, list.Len())
+}
+
+type IntervalStruct struct {
+	x     int
+	start int64
+	stop  int64
+}
+
+func newIntervalStruct(i int) IntervalStruct {
+	return IntervalStruct{
+		x: i,
+	}
+}
+
+func (i IntervalStruct) SetStartStop(start, stop int64) {
+	i.start, i.stop = start, stop
+}
+func (i IntervalStruct) Clone() IntervalValue {
+	return &IntervalStruct{
+		x:     i.x,
+		start: i.start,
+		stop:  i.stop,
+	}
+}
+
+func TestIntervalList_insertIntervalStruct(t *testing.T) {
+	list := NewIntervalList[IntervalStruct]()
+
+	list.InsertInterval(0, 64, 1, newIntervalStruct(1))
+
+	list.InsertInterval(64, 72, 2, newIntervalStruct(2))
+
+	list.InsertInterval(72, 136, 3, newIntervalStruct(3))
+
+	list.InsertInterval(64, 68, 4, newIntervalStruct(4))
+
+	for p := list.Front(); p != nil; p = p.Next {
+		fmt.Printf("[%d,%d) %d %d\n", p.StartOffset, p.StopOffset, p.TsNs, p.Value)
+	}
+	assert.Equal(t, 4, list.Len())
+}

weed/filer/leveldb/leveldb_store.go

@@ -0,0 +1,300 @@
+package leveldb
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	leveldb_errors "github.com/syndtr/goleveldb/leveldb/errors"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	weed_util "github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const (
+	DIR_FILE_SEPARATOR = byte(0x00)
+)
+
+var (
+	_ = filer.Debuggable(&LevelDBStore{})
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &LevelDBStore{})
+}
+
+type LevelDBStore struct {
+	db *leveldb.DB
+}
+
+func (store *LevelDBStore) GetName() string {
+	return "leveldb"
+}
+
+func (store *LevelDBStore) Initialize(configuration weed_util.Configuration, prefix string) (err error) {
+	dir := configuration.GetString(prefix + "dir")
+	return store.initialize(dir)
+}
+
+func (store *LevelDBStore) initialize(dir string) (err error) {
+	glog.V(0).Infof("filer store dir: %s", dir)
+	os.MkdirAll(dir, 0755)
+	if err := weed_util.TestFolderWritable(dir); err != nil {
+		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
+	}
+
+	opts := &opt.Options{
+		BlockCacheCapacity: 32 * 1024 * 1024,         // default value is 8MiB
+		WriteBuffer:        16 * 1024 * 1024,         // default value is 4MiB
+		Filter:             filter.NewBloomFilter(8), // false positive rate 0.02
+	}
+
+	if store.db, err = leveldb.OpenFile(dir, opts); err != nil {
+		if leveldb_errors.IsCorrupted(err) {
+			store.db, err = leveldb.RecoverFile(dir, opts)
+		}
+		if err != nil {
+			glog.Infof("filer store open dir %s: %v", dir, err)
+			return
+		}
+	}
+	return
+}
+
+func (store *LevelDBStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *LevelDBStore) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *LevelDBStore) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *LevelDBStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	key := genKey(entry.DirAndName())
+
+	value, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
+	}
+
+	if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+		value = weed_util.MaybeGzipData(value)
+	}
+
+	err = store.db.Put(key, value, nil)
+
+	if err != nil {
+		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
+	}
+
+	// println("saved", entry.FullPath, "chunks", len(entry.GetChunks()))
+
+	return nil
+}
+
+func (store *LevelDBStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	return store.InsertEntry(ctx, entry)
+}
+
+// BatchInsertEntries inserts multiple entries in a single LevelDB batch write.
+// This is more efficient than inserting entries one by one as it reduces
+// the number of write operations and syncs to disk.
+func (store *LevelDBStore) BatchInsertEntries(ctx context.Context, entries []*filer.Entry) error {
+	if len(entries) == 0 {
+		return nil
+	}
+
+	batch := new(leveldb.Batch)
+
+	for _, entry := range entries {
+		key := genKey(entry.DirAndName())
+
+		value, err := entry.EncodeAttributesAndChunks()
+		if err != nil {
+			return fmt.Errorf("encoding %s %+v: %w", entry.FullPath, entry.Attr, err)
+		}
+
+		if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+			value = weed_util.MaybeGzipData(value)
+		}
+
+		batch.Put(key, value)
+	}
+
+	err := store.db.Write(batch, nil)
+	if err != nil {
+		return fmt.Errorf("batch write: %w", err)
+	}
+
+	return nil
+}
+
+func (store *LevelDBStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) {
+	key := genKey(fullpath.DirAndName())
+
+	data, err := store.db.Get(key, nil)
+
+	if err == leveldb.ErrNotFound {
+		return nil, filer_pb.ErrNotFound
+	}
+	if err != nil {
+		return nil, fmt.Errorf("get %s : %v", fullpath, err)
+	}
+
+	entry = &filer.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(weed_util.MaybeDecompressData((data)))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	// println("read", entry.FullPath, "chunks", len(entry.GetChunks()), "data", len(data), string(data))
+
+	return entry, nil
+}
+
+func (store *LevelDBStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
+	key := genKey(fullpath.DirAndName())
+
+	err = store.db.Delete(key, nil)
+	if err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *LevelDBStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
+
+	batch := new(leveldb.Batch)
+
+	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
+	iter := store.db.NewIterator(&leveldb_util.Range{Start: directoryPrefix}, nil)
+	for iter.Next() {
+		key := iter.Key()
+		if !bytes.HasPrefix(key, directoryPrefix) {
+			break
+		}
+		fileName := getNameFromKey(key)
+		if fileName == "" {
+			continue
+		}
+		batch.Delete([]byte(genKey(string(fullpath), fileName)))
+	}
+	iter.Release()
+
+	err = store.db.Write(batch, nil)
+
+	if err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *LevelDBStore) ListDirectoryEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc)
+}
+
+func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+
+	directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix)
+	lastFileStart := directoryPrefix
+	if startFileName != "" {
+		lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
+	}
+
+	iter := store.db.NewIterator(&leveldb_util.Range{Start: lastFileStart}, nil)
+	for iter.Next() {
+		key := iter.Key()
+		if !bytes.HasPrefix(key, directoryPrefix) {
+			break
+		}
+		fileName := getNameFromKey(key)
+		if fileName == "" {
+			continue
+		}
+		if fileName == startFileName && !includeStartFile {
+			continue
+		}
+		limit--
+		if limit < 0 {
+			break
+		}
+		lastFileName = fileName
+		entry := &filer.Entry{
+			FullPath: weed_util.NewFullPath(string(dirPath), fileName),
+		}
+		if decodeErr := filer.DecodeListedEntry(ctx, entry, weed_util.MaybeDecompressData(iter.Value())); decodeErr != nil {
+			err = decodeErr
+			glog.V(0).InfofCtx(ctx, "list %s : %v", entry.FullPath, err)
+			break
+		}
+
+		resEachEntryFunc, resEachEntryFuncErr := eachEntryFunc(entry)
+		if resEachEntryFuncErr != nil {
+			err = fmt.Errorf("failed to process eachEntryFunc: %w", resEachEntryFuncErr)
+			break
+		}
+
+		if !resEachEntryFunc {
+			break
+		}
+	}
+	iter.Release()
+
+	return lastFileName, err
+}
+
+func genKey(dirPath, fileName string) (key []byte) {
+	key = []byte(dirPath)
+	key = append(key, DIR_FILE_SEPARATOR)
+	key = append(key, []byte(fileName)...)
+	return key
+}
+
+func genDirectoryKeyPrefix(fullpath weed_util.FullPath, startFileName string) (keyPrefix []byte) {
+	keyPrefix = []byte(string(fullpath))
+	keyPrefix = append(keyPrefix, DIR_FILE_SEPARATOR)
+	if len(startFileName) > 0 {
+		keyPrefix = append(keyPrefix, []byte(startFileName)...)
+	}
+	return keyPrefix
+}
+
+func getNameFromKey(key []byte) string {
+
+	sepIndex := len(key) - 1
+	for sepIndex >= 0 && key[sepIndex] != DIR_FILE_SEPARATOR {
+		sepIndex--
+	}
+
+	return string(key[sepIndex+1:])
+
+}
+
+func (store *LevelDBStore) Shutdown() {
+	store.db.Close()
+}
+
+func (store *LevelDBStore) Debug(writer io.Writer) {
+	iter := store.db.NewIterator(&leveldb_util.Range{}, nil)
+	for iter.Next() {
+		key := iter.Key()
+		fullName := bytes.Replace(key, []byte{DIR_FILE_SEPARATOR}, []byte{' '}, 1)
+		fmt.Fprintf(writer, "%v\n", string(fullName))
+	}
+	iter.Release()
+}

weed/filer/leveldb/leveldb_store_kv.go

@@ -0,0 +1,62 @@
+package leveldb
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/syndtr/goleveldb/leveldb"
+	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
+)
+
+func (store *LevelDBStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+
+	err = store.db.Put(key, value, nil)
+
+	if err != nil {
+		return fmt.Errorf("kv put: %w", err)
+	}
+
+	return nil
+}
+
+func (store *LevelDBStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+
+	value, err = store.db.Get(key, nil)
+
+	if err == leveldb.ErrNotFound {
+		return nil, filer.ErrKvNotFound
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("kv get: %w", err)
+	}
+
+	return
+}
+
+// VisitKvPrefix calls visit for every key-value pair whose key starts with
+// prefix, in key order. Returning an error from visit stops the scan.
+func (store *LevelDBStore) VisitKvPrefix(ctx context.Context, prefix []byte, visit func(key []byte, value []byte) error) error {
+	iter := store.db.NewIterator(leveldb_util.BytesPrefix(prefix), nil)
+	defer iter.Release()
+	for iter.Next() {
+		key := append([]byte(nil), iter.Key()...)
+		value := append([]byte(nil), iter.Value()...)
+		if err := visit(key, value); err != nil {
+			return err
+		}
+	}
+	return iter.Error()
+}
+
+func (store *LevelDBStore) KvDelete(ctx context.Context, key []byte) (err error) {
+
+	err = store.db.Delete(key, nil)
+
+	if err != nil {
+		return fmt.Errorf("kv delete: %w", err)
+	}
+
+	return nil
+}

weed/filer/leveldb/leveldb_store_test.go

@@ -0,0 +1,112 @@
+package leveldb
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestCreateAndFind(t *testing.T) {
+	testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
+	dir := t.TempDir()
+	store := &LevelDBStore{}
+	store.initialize(dir)
+	testFiler.SetStore(store)
+
+	fullpath := util.FullPath("/home/chris/this/is/one/file1.jpg")
+
+	ctx := context.Background()
+
+	entry1 := &filer.Entry{
+		FullPath: fullpath,
+		Attr: filer.Attr{
+			Mode: 0440,
+			Uid:  1234,
+			Gid:  5678,
+		},
+	}
+
+	if err := testFiler.CreateEntry(ctx, entry1, nil, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {
+		t.Errorf("create entry %v: %v", entry1.FullPath, err)
+		return
+	}
+
+	entry, err := testFiler.FindEntry(ctx, fullpath)
+
+	if err != nil {
+		t.Errorf("find entry: %v", err)
+		return
+	}
+
+	if entry.FullPath != entry1.FullPath {
+		t.Errorf("find wrong entry: %v", entry.FullPath)
+		return
+	}
+
+	// checking one upper directory
+	entries, _, _ := testFiler.ListDirectoryEntries(ctx, util.FullPath("/home/chris/this/is/one"), "", false, 100, "", "", "")
+	if len(entries) != 1 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+	// checking one upper directory
+	entries, _, _ = testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "", "")
+	if len(entries) != 1 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+}
+
+func TestEmptyRoot(t *testing.T) {
+	testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
+	dir := t.TempDir()
+	store := &LevelDBStore{}
+	store.initialize(dir)
+	testFiler.SetStore(store)
+
+	ctx := context.Background()
+
+	// checking one upper directory
+	entries, _, err := testFiler.ListDirectoryEntries(ctx, util.FullPath("/"), "", false, 100, "", "", "")
+	if err != nil {
+		t.Errorf("list entries: %v", err)
+		return
+	}
+	if len(entries) != 0 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+}
+
+func BenchmarkInsertEntry(b *testing.B) {
+	testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
+	dir := b.TempDir()
+	store := &LevelDBStore{}
+	store.initialize(dir)
+	testFiler.SetStore(store)
+
+	ctx := context.Background()
+
+	b.ReportAllocs()
+
+	for i := 0; i < b.N; i++ {
+		entry := &filer.Entry{
+			FullPath: util.FullPath(fmt.Sprintf("/file%d.txt", i)),
+			Attr: filer.Attr{
+				Crtime: time.Now(),
+				Mtime:  time.Now(),
+				Mode:   os.FileMode(0644),
+			},
+		}
+		store.InsertEntry(ctx, entry)
+	}
+}

weed/filer/leveldb/object_size_metric_test.go

@@ -0,0 +1,75 @@
+package leveldb
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	dto "github.com/prometheus/client_model/go"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func histogramState(t *testing.T) (count uint64, perBucket map[float64]uint64) {
+	t.Helper()
+	m := &dto.Metric{}
+	if err := stats.FilerObjectSizeBytesHistogram.Write(m); err != nil {
+		t.Fatalf("write histogram: %v", err)
+	}
+	perBucket = make(map[float64]uint64)
+	for _, b := range m.GetHistogram().GetBucket() {
+		perBucket[b.GetUpperBound()] = b.GetCumulativeCount()
+	}
+	return m.GetHistogram().GetSampleCount(), perBucket
+}
+
+func TestCreateEntryRecordsObjectSize(t *testing.T) {
+	testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
+	store := &LevelDBStore{}
+	if err := store.initialize(t.TempDir()); err != nil {
+		t.Fatalf("init store: %v", err)
+	}
+	testFiler.SetStore(store)
+	defer testFiler.Shutdown()
+	ctx := context.Background()
+
+	file := func(path string, size uint64) *filer.Entry {
+		return &filer.Entry{
+			FullPath: util.FullPath(path),
+			Attr:     filer.Attr{Mode: 0640, FileSize: size},
+		}
+	}
+
+	before, _ := histogramState(t)
+
+	if err := testFiler.CreateEntry(ctx, file("/data/small.txt", 500), nil, false, false, nil, false, 255); err != nil {
+		t.Fatalf("create small: %v", err)
+	}
+	if err := testFiler.CreateEntry(ctx, file("/data/big.bin", 5*1024*1024), nil, false, false, nil, false, 255); err != nil {
+		t.Fatalf("create big: %v", err)
+	}
+	dir := &filer.Entry{FullPath: util.FullPath("/data/sub"), Attr: filer.Attr{Mode: os.ModeDir | 0755}}
+	if err := testFiler.CreateEntry(ctx, dir, nil, false, false, nil, false, 255); err != nil {
+		t.Fatalf("create dir: %v", err)
+	}
+	// overwrite is an update, not a new object
+	if err := testFiler.CreateEntry(ctx, file("/data/small.txt", 800), nil, false, false, nil, false, 255); err != nil {
+		t.Fatalf("overwrite small: %v", err)
+	}
+
+	after, buckets := histogramState(t)
+
+	// only the two new files are sampled, into their respective ranges
+	if got := after - before; got != 2 {
+		t.Fatalf("expected 2 sampled objects, got %d", got)
+	}
+	if buckets[1024] < 1 {
+		t.Errorf("expected the 500-byte object in the <=1024 bucket, buckets=%v", buckets)
+	}
+	if buckets[104857600]-buckets[1048576] < 1 {
+		t.Errorf("expected the 5MB object in the (1MB,100MB] range, buckets=%v", buckets)
+	}
+}